# robots.txt
User-Agent: sogou spider
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: UbiCrawler
Disallow: /
User-agent:Mediapartners-Google* 
Disallow:
User-agent: *
#Crawl-delay: 100