# Tries to stop all robots # derived from Nexor's robots.txt # See http://web.nexor.co.uk/users/mak/doc/robots/norobots.html # Original by Martijn Koster , # User-agent: spider name (Ultraseek, * for all) # Disallow: /directory/ (e.g. /icons/) User-agent: BecomeBot Crawl-Delay: 50 User-Agent: msnbot Crawl-Delay: 120 User-Agent: Yahoo! Slurp China Crawl-Delay: 50 User-agent: Slurp Crawl-delay: 50 User-agent: Twiceler Crawl-delay: 50 User-agent: baiduspider Crawl-delay: 50 User-Agent: Slurp Disallow: /ParticipatingParish.php User-Agent: Charlotte Disallow: / User-Agent: * Disallow: /audio/ Disallow: /cgi-bin/ Disallow: /common/ Disallow: /cron/ Disallow: /css/ Disallow: /documents/ Disallow: /dotproject-DEAD/ Disallow: /error_log Disallow: /images/ Disallow: /images_source/ Disallow: /mentors/ Disallow: /ParticipatingParish.php Disallow: /pdf/ Disallow: /photos/ Disallow: /php_scripts/ Disallow: /RSS/ Disallow: /Tasks.php