User-agent: Slurp
Disallow: /

User-agent: Yahoo! Slurp
Disallow: /


# Block wiederfreibot as it is just noise
User-agent: wiederfreibot/1.0
Disallow: /

# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /

# Block MJ12bot as it is just noise
User-agent: MJ12bot
Disallow: /

# Block dotbot as it cannot parse base urls properly
User-agent: dotbot/1.0
Disallow: /

# Block Gigabot
User-agent: Gigabot
Disallow: /

# Block AhrefsBot
User-agent: AhrefsBot
Disallow: /

# Block YandexBot
User-agent: YandexBot
Disallow: /

# Block Yandex
User-agent: Yandex
Disallow: /

# Block Exabot
User-agent: Exabot
Disallow: /

# Block PathDefender
User-agent: PathDefender
Disallow: /

# Block Goodzer
User-agent: Goodzer
Disallow: /

# Block  MegaIndex.ru
User-agent: MegaIndex.ru/2.0
Disallow: /

# Block Ezooms
User-agent: Ezooms
Disallow: /

# Block Baiduspider
User-agent: Baiduspider
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Vagabondo
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: BSpider
Disallow: /

# Block common cralwers:
User-agent: voltron
Disallow: /

# Some weird backlink checker (webmeup.com)
User-agent: BLEXBot
Disallow: /

# Another backlink crawler
User-agent: linkdexbot/2.0
Disallow: /

# Another backlink crawler (https://ahrefs.com/robot)
User-Agent: AhrefsBot
Disallow: /

# Another one.
User-Agent: GetintentCrawler
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: OSG_BOT
Disallow: /

User-agent: VegeBot
Disallow: /
Crawl-delay: 60

User-agent: "Vegi bot (we follow your robots.txt settings before crawling, you can slow down the bot by change the Crawl-Delay parameter in the settings.if you have an enquiry, please email to: abuse-report@terrykyleseoagency.com)"
Disallow: /
Crawl-delay: 60

User-agent: adbeat_bot
Disallow: /

User-agent: Jooblebot
Disallow: /

# BEGIN W3TC ROBOTS
User-agent: *
Disallow: /wp-content/cache/
# END W3TC ROBOTS

Sitemap: https://www.niemanlab.org/sitemap.xml