# For Mediapartners-Google, Googlebot, and Slurp set the crawl delay to 1 second
User-agent: Bingbot
Disallow: /r/candidates/
Disallow: /r/jobs/applicant.cfm
Disallow: /c/apply/
Disallow: /emails/
Crawl-delay: 1

User-agent: Mediapartners-Google
Disallow: /r/candidates/
Disallow: /btrp/
Disallow: /r/jobs/applicant.cfm
Disallow: /c/apply/
Disallow: /emails/
Crawl-delay: 1

User-agent: Googlebot
Disallow: /r/candidates/
Disallow: /btrp/
Disallow: /r/jobs/applicant.cfm
Disallow: /c/apply/
Disallow: /emails/
Crawl-delay: 1

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: Slurp
Disallow: /r/candidates/
Disallow: /btrp/
Disallow: /r/jobs/applicant.cfm
Disallow: /c/apply/
Disallow: /emails/
Crawl-delay: 1

User-agent: Facebot
Disallow: /r/candidates/
Disallow: /r/jobs/applicant.cfm
Disallow: /c/apply/
Disallow: /emails/
Crawl-delay: 1

User-agent: facebookexternalhit/1.0
Disallow: /r/candidates/
Disallow: /r/jobs/applicant.cfm
Disallow: /c/apply/
Disallow: /emails/
Crawl-delay: 1

User-agent: facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)
Disallow: /r/candidates/
Disallow: /r/jobs/applicant.cfm
Disallow: /c/apply/
Disallow: /emails/
Crawl-delay: 1

User-agent: facebookexternalhit/1.1
Disallow: /r/candidates/
Disallow: /r/jobs/applicant.cfm
Disallow: /c/apply/
Disallow: /emails/
Crawl-delay: 1

User-agent: facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)
Disallow: /r/candidates/
Disallow: /r/jobs/applicant.cfm
Disallow: /c/apply/
Disallow: /emails/
Crawl-delay: 1

# Bad Bots:
# gsa-crawler, and MJ12bot - Disallow from root 
User-agent: gsa-crawler
Disallow: /

User-agent: MJ12bot
Disallow: /

# For all user-agent types, disallow candidates, companies, and bot-trap directories.
# Force the crawl delay for all bots to be one hour
User-agent: *
Disallow: /r/candidates/
Disallow: /btrp/
Disallow: /r/jobs/applicant.cfm
Disallow: /c/apply/
Disallow: /emails/
Crawl-delay: 3600