robots.txt

# This is a demonstration staging server and should not be indexed by search engines.
# The canonical site to crawl is https://data.gov

User-agent: * # match all bots
Disallow: / # keep them out

User-Agent: Googlebot
Disallow: /
 
User-Agent: Googlebot-Mobile
Disallow: /
 
User-Agent: msnbot
Disallow: /
 
User-Agent: bingbot
Disallow: /
 
# Adsense
User-Agent: Mediapartners-Google
Disallow: /
 
# Blekko
User-Agent: ScoutJet
Disallow: /
 
User-Agent: Yandex
Disallow: /
 
User-agent: baiduspider
Disallow: /
 
User-agent: DuckDuckBot
Disallow: /
 
# CommonCrawl
User-agent: ccbot
Disallow: /