#                               
#         &&&&&                 
#         &&&&&                 
#          @@@                  
#          @@@                  
#          @@@                  
#          @@@                  
#          @@@                  
#          @@@                  
#          @@@                  
#         @@@                   
#        @@@                    
#                               
#                               

# www.robotstxt.org/
# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449

Crawl-delay: 5

User-agent: *

Disallow: /search/getjob
Disallow: /i/*
Disallow: /cd/*
Disallow: /private/search
Disallow: *fo=1
Disallow: /feed-wel/
Disallow: /ads/
Disallow: *&orgn=113
Disallow: /api/*
Disallow: /before-redirect

User-agent: Mediapartners-Google
User-agent: AdIdxBot
Allow: *

User-agent: 008
Disallow: /

User-agent: grapeshot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Baiduspider
User-agent: 360Spider
User-agent: Yisouspider
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Sogou inst spider
Disallow: /

User-agent: Timpibot
Disallow: /