User-agent: 
Disallow: /
User-agent: !Susie (http://www.sync2it.com/susie)  
Disallow: /
User-agent: ( Robots.txt Validator http://www.searchengineworld.com/cgi-bin/robotcheck.cgi )  
Disallow: /
User-agent: (DreamPassport/3.0; isao/MyDiGiRabi)  
Disallow: /
User-agent: (Privoxy/1.0)  
Disallow: /
User-agent: */Nutch-0.9-dev  
Disallow: /
User-agent: +SitiDi.net/SitiDiBot/1.0 (+Have Good Day)  
Disallow: /
User-agent: -DIE-KRAEHE- META-SEARCH-ENGINE/1.1 http://www.die-kraehe.de  
Disallow: /
User-agent: 123spider-Bot (Version: 1.02, powered by www.123spider.de  
Disallow: /
User-agent: 192.comAgent  
Disallow: /
User-agent: 1st ZipCommander (Net) - http://www.zipcommander.com/  
Disallow: /
User-agent: 2Bone_LinkChecker/1.0 libwww-perl/5.64  
Disallow: /
User-agent: 4anything.com LinkChecker v2.0  
Disallow: /
User-agent: 8484 Boston Project v 1.0  
Disallow: /
User-agent: :robot/1.0 (linux) ( admin e-mail: undefined http://www.neofonie.de/loesungen/search/robot.html )  
Disallow: /
User-agent: <a href='http://www.unchaos.com/'> UnChaos </a> From Chaos To Order Hybrid Web Search Engine.(vadim_gonchar@unchaos.com)  
Disallow: /
User-agent: <a href='http://www.unchaos.com/'> UnChaos Bot Hybrid Web Search Engine. </a> (vadim_gonchar@unchaos.com)  
Disallow: /
User-agent: <b> UnChaosBot From Chaos To Order UnChaos Hybrid Web Search Engine at www.unchaos.com </b> (info@unchaos.com)  
Disallow: /
User-agent: <http://www.sygol.com/> http://www.sygol.com  
Disallow: /
User-agent: A-Online Search  
Disallow: /
User-agent: A1 Keyword Research/1.0.2 (+http://www.micro-sys.dk/products/keyword-research/) miggibot/2007.03.27  
Disallow: /
User-agent: A1 Sitemap Generator/1.0 (+http://www.micro-sys.dk/products/sitemap-generator/) miggibot/2006.01.24  
Disallow: /
User-agent: ABCdatos BotLink/5.xx.xxx#BBL  
Disallow: /
User-agent: AESOP_com_SpiderMan  
Disallow: /
User-agent: AIBOT/2.1 By +(www.21seek.com A Real artificial intelligence search engine China)  
Disallow: /
User-agent: ANTFresco/x.xx  
Disallow: /
User-agent: ASAHA Search Engine Turkey V.001 (http://www.asaha.com/)  
Disallow: /
User-agent: ASPSeek/1.2.5  
Disallow: /
User-agent: ASPSeek/1.2.x  
Disallow: /
User-agent: ASPSeek/1.2.xa  
Disallow: /
User-agent: ASPSeek/1.2.xxpre  
Disallow: /
User-agent: ASPseek/1.2.9d  
Disallow: /
User-agent: ASPseek/1.2.xx  
Disallow: /
User-agent: ASSORT/0.10  
Disallow: /
User-agent: AU-MIC/2.0 MMP/2.0  
Disallow: /
User-agent: AUDIOVOX-SMT5600  
Disallow: /
User-agent: AV Fetch 1.0  
Disallow: /
User-agent: AVSearch-1.0(peter.turney@nrc.ca)  
Disallow: /
User-agent: AVSearch-2.0-fusionIdx-14-CompetitorWebSites  
Disallow: /
User-agent: AVSearch-3.0(AltaVista/AVC)  
Disallow: /
User-agent: AWeb  
Disallow: /
User-agent: AbachoBOT  
Disallow: /
User-agent: AbachoBOT (Mozilla compatible)  
Disallow: /
User-agent: Aberja Checkomat  
Disallow: /
User-agent: About/0.1libwww-perl/5.47  
Disallow: /
User-agent: Accelatech RSSCrawler/0.4  
Disallow: /
User-agent: Accoona-AI-Agent/1.1.1 (crawler at accoona dot com)  
Disallow: /
User-agent: Accoona-AI-Agent/1.1.2 (aicrawler at accoonabot dot com)  
Disallow: /
User-agent: Ace Explorer  
Disallow: /
User-agent: Ack (http://www.ackerm.com/)  
Disallow: /
User-agent: AcoiRobot  
Disallow: /
User-agent: Acoon Robot v1.50.001  
Disallow: /
User-agent: Acoon Robot v1.52 (http://www.acoon.de)  
Disallow: /
User-agent: Acoon-Robot 4.0.x.[xx] (http://www.acoon.de)  
Disallow: /
User-agent: Acoon-Robot v3.xx (http://www.acoon.de and http://www.acoon.com)  
Disallow: /
User-agent: Acorn/Nutch-0.9 (Non-Profit Search Engine; acorn.isara.org; acorn at isara dot org)  
Disallow: /
User-agent: ActiveBookmark 1.x  
Disallow: /
User-agent: ActiveWorlds/3.xx (xxx)  
Disallow: /
User-agent: Activeworlds  
Disallow: /
User-agent: Ad Muncher v4.xx.x  
Disallow: /
User-agent: Ad Muncher v4x Build xxxxx  
Disallow: /
User-agent: Adaxas Spider (http://www.adaxas.net/)  
Disallow: /
User-agent: Advanced Browser (http://www.avantbrowser.com)  
Disallow: /
User-agent: Agent-SharewarePlazaFileCheckBot/2.0+(+http://www.SharewarePlaza.com)  
Disallow: /
User-agent: AgentName/0.1 libwww-perl/5.48  
Disallow: /
User-agent: AideRSS/1.0 (aiderss.com)  
Disallow: /
User-agent: Aladin/3.324  
Disallow: /
User-agent: Alcatel-BG3/1.0 UP.Browser/5.0.3.1.2  
Disallow: /
User-agent: Aleksika Spider/1.0 (+http://www.aleksika.com/)  
Disallow: /
User-agent: AlkalineBOT/1.3  
Disallow: /
User-agent: AlkalineBOT/1.4 (1.4.0326.0 RTM)  
Disallow: /
User-agent: Allesklar/0.1 libwww-perl/5.46  
Disallow: /
User-agent: Alligator 1.31 (www.nearsoftware.com)  
Disallow: /
User-agent: AltaVista Intranet V2.0 AVS EVAL search@freeit.com  
Disallow: /
User-agent: AltaVista Intranet V2.0 Compaq Altavista Eval sveand@altavista.net  
Disallow: /
User-agent: AltaVista Intranet V2.0 evreka.com crawler@evreka.com  
Disallow: /
User-agent: AltaVista V2.0B crawler@evreka.com  
Disallow: /
User-agent: AmfibiBOT  
Disallow: /
User-agent: Amfibibot/0.06 (Amfibi Web Search; http://www.amfibi.com; agent@amfibi.com)  
Disallow: /
User-agent: Amfibibot/0.07 (Amfibi Robot; http://www.amfibi.com; agent@amfibi.com)  
Disallow: /
User-agent: AmiTCP Miami (AmigaOS 2.04)  
Disallow: /
User-agent: Amiga-AWeb/3.4.167SE  
Disallow: /
User-agent: AmigaVoyager/3.4.4 (MorphOS/PPC native)  
Disallow: /
User-agent: AnnoMille spider 0.1 alpha - http://www.annomille.it  
Disallow: /
User-agent: Anonymized by ProxyOS: http://www.megaproxy.com  
Disallow: /
User-agent: Anonymizer/1.1  
Disallow: /
User-agent: AnswerBus (http://www.answerbus.com/)  
Disallow: /
User-agent: AnswerChase PROve x.0  
Disallow: /
User-agent: AnswerChase x.0  
Disallow: /
User-agent: AnzwersCrawl/2.0 (anzwerscrawl@anzwers.com.au;Engine)  
Disallow: /
User-agent: Apexoo Spider 1.x  
Disallow: /
User-agent: Aplix HTTP/1.0.1  
Disallow: /
User-agent: Aplix_SANYO_browser/1.x (Japanese)  
Disallow: /
User-agent: Aplix_SEGASATURN_browser/1.x (Japanese)  
Disallow: /
User-agent: Aport  
Disallow: /
User-agent: Apple iPhone v1.1.4 CoreMedia v1.0.0.4A102  
Disallow: /
User-agent: ArabyBot (compatible; Mozilla/5.0; GoogleBot; FAST Crawler 6.4; http://www.araby.com;)  
Disallow: /
User-agent: Arachnoidea (arachnoidea@euroseek.com)  
Disallow: /
User-agent: ArchitextSpider  
Disallow: /
User-agent: Argus/1.1 (Nutch; http://www.simpy.com/bot.html; feedback at simpy dot com)  
Disallow: /
User-agent: Arikus_Spider  
Disallow: /
User-agent: Arquivo-web-crawler (compatible; heritrix/1.12.1 +http://arquivo-web.fccn.pt)  
Disallow: /
User-agent: Asahina-Antenna/1.x  
Disallow: /
User-agent: Asahina-Antenna/1.x (libhina.pl/x.x ; libtime.pl/x.x)  
Disallow: /
User-agent: AskAboutOil/0.06-rcp (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@askaboutoil.com)  
Disallow: /
User-agent: AtlocalBot/1.1 +(http://www.atlocal.com/local-web-site-owner.html)  
Disallow: /
User-agent: Atomic_Email_Hunter/4.0  
Disallow: /
User-agent: Atomz/1.0  
Disallow: /
User-agent: Attentio/Nutch-0.9-dev (Attentio's beta blog crawler; www.attentio.com; info@attentio.com)  
Disallow: /
User-agent: Avant Browser (http://www.avantbrowser.com)  
Disallow: /
User-agent: AxmoRobot - Crawling your site for better indexing on www.axmo.com search engine.  
Disallow: /
User-agent: Azureus 2.x.x.x  
Disallow: /
User-agent: BDFetch  
Disallow: /
User-agent: BDNcentral Crawler v2.3 [en] (http://www.bdncentral.com/robot.html) (X11; I; Linux 2.0.44 i686)  
Disallow: /
User-agent: BIGLOTRON (Beta 2;GNU/Linux)  
Disallow: /
User-agent: BMCLIENT  
Disallow: /
User-agent: BMLAUNCHER  
Disallow: /
User-agent: BPImageWalker/2.0 (www.bdbrandprotect.com)  
Disallow: /
User-agent: BSDSeek/1.0  
Disallow: /
User-agent: BStop.BravoBrian.it Agent Detector  
Disallow: /
User-agent: BTWebClient/180B(9704)  
Disallow: /
User-agent: BTbot/0.x (+http://www.btbot.com/btbot.html)  
Disallow: /
User-agent: BW-C-2.0  
Disallow: /
User-agent: BaboomBot/1.x.x (+http://www.baboom.us)  
Disallow: /
User-agent: BackStreet Browser 3.x  
Disallow: /
User-agent: BaiDuSpider  
Disallow: /
User-agent: BaiduImagespider+(+http://www.baidu.jp/search/s308.html)  
Disallow: /
User-agent: Baiduspider+(+http://help.baidu.jp/system/05.html)  
Disallow: /
User-agent: Baiduspider+(+http://www.baidu.com/search/spider.htm)  
Disallow: /
User-agent: Baiduspider+(+http://www.baidu.com/search/spider_jp.html)  
Disallow: /
User-agent: Balihoo/Nutch-1.0-dev (Crawler for Balihoo.com search engine - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)  
Disallow: /
User-agent: BanBots/1.2 (spider@banbots.com)  
Disallow: /
User-agent: Barca/2.0.xxxx  
Disallow: /
User-agent: BarcaPro/1.4.xxxx  
Disallow: /
User-agent: BarraHomeCrawler (albertof@barrahome.org)  
Disallow: /
User-agent: BeamMachine/0.5 (dead link remover of www.beammachine.net)  
Disallow: /
User-agent: BebopBot/2.5.1 ( crawler http://www.apassion4jazz.net/bebopbot.html )  
Disallow: /
User-agent: BeebwareDirectory/v0.01  
Disallow: /
User-agent: Big Brother (http://pauillac.inria.fr/~fpottier/)  
Disallow: /
User-agent: Big Fish v1.0  
Disallow: /
User-agent: BigBrother/1.6e  
Disallow: /
User-agent: BigCliqueBOT/1.03-dev (bigclicbot; http://www.bigclique.com; bot@bigclique.com)  
Disallow: /
User-agent: Bigsearch.ca/Nutch-x.x-dev (Bigsearch.ca Internet Spider; http://www.bigsearch.ca/; info@enhancededge.com)  
Disallow: /
User-agent: Bilbo/2.3b-UNIX  
Disallow: /
User-agent: BilgiBetaBot/0.8-dev (bilgi.com (Beta) ; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)  
Disallow: /
User-agent: BilgiBot/1.0(beta) (http://www.bilgi.com/; bilgi at bilgi dot com)  
Disallow: /
User-agent: Bitacle Robot (V:1.0;) (http://www.bitacle.com)  
Disallow: /
User-agent: Bitacle bot/1.1  
Disallow: /
User-agent: Biyubi/x.x (Sistema Fenix; G11; Familia Toledo; es-mx)  
Disallow: /
User-agent: BlackBerry7520/4.0.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/5.0.3.3 UP.Link/5.1.2.12 (Google WAP Proxy/1.0)  
Disallow: /
User-agent: BlackWidow  
Disallow: /
User-agent: Blaiz-Bee/1.0 (+http://www.blaiz.net)  
Disallow: /
User-agent: Blaiz-Bee/2.00.8222 (BE Internet Search Engine http://www.rawgrunt.com)  
Disallow: /
User-agent: Blaiz-Bee/2.00.xxxx (+http://www.blaiz.net)  
Disallow: /
User-agent: BlitzBOT@tricus.net  
Disallow: /
User-agent: BlitzBOT@tricus.net (Mozilla compatible)  
Disallow: /
User-agent: BlockNote.Net  
Disallow: /
User-agent: BlogBot/1.x  
Disallow: /
User-agent: BlogBridge 2.13 (http://www.blogbridge.com/)  
Disallow: /
User-agent: BlogMap (http://www.feedmap.net)  
Disallow: /
User-agent: BlogPulseLive (support@blogpulse.com)  
Disallow: /
User-agent: BlogSearch/1.x +http://www.icerocket.com/  
Disallow: /
User-agent: BlogVibeBot-v1.1 (spider@blogvibe.nl)  
Disallow: /
User-agent: Bloglines Title Fetch/1.0 (http://www.bloglines.com)  
Disallow: /
User-agent: Bloglines-Images/0.1 (http://www.bloglines.com)  
Disallow: /
User-agent: Blogpulse (info@blogpulse.com)  
Disallow: /
User-agent: BlogsNowBot, V 2.01 (+http://www.blogsnow.com/)  
Disallow: /
User-agent: BlogzIce/1.0 (+http://icerocket.com; rhodes@icerocket.com)  
Disallow: /
User-agent: BlogzIce/1.0 +http://www.icerocket.com/  
Disallow: /
User-agent: BloobyBot  
Disallow: /
User-agent: Bloodhound/Nutch-0.9 (Testing Crawler for Research - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)  
Disallow: /
User-agent: Bobby/4.0.x RPT-HTTPClient/0.3-3E  
Disallow: /
User-agent: Bookdog/x.x  
Disallow: /
User-agent: Bookmark Buddy bookmark checker (http://www.bookmarkbuddy.net/)  
Disallow: /
User-agent: Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/]  
Disallow: /
User-agent: Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/] (Version 2.0beta)  
Disallow: /
User-agent: BookmarkBase(2/;http://bookmarkbase.com)  
Disallow: /
User-agent: Bot mailto:craftbot@yahoo.com  
Disallow: /
User-agent: BravoBrian SpiderEngine MarcoPolo  
Disallow: /
User-agent: BravoBrian bstop.bravobrian.it  
Disallow: /
User-agent: BrightCrawler (http://www.brightcloud.com/brightcrawler.asp)  
Disallow: /
User-agent: BruinBot (+http://webarchive.cs.ucla.edu/bruinbot.html)   
Disallow: /
User-agent: BuildCMS crawler (http://www.buildcms.com/crawler)  
Disallow: /
User-agent: Bulkfeeds/r1752 (http://bulkfeeds.net/)  
Disallow: /
User-agent: BullsEye  
Disallow: /
User-agent: BunnySlippers  
Disallow: /
User-agent: BurstFindCrawler/1.1 (crawler.burstfind.com; http://crawler.burstfind.com; crawler@burstfind.com)  
Disallow: /
User-agent: Buscaplus Robi/1.0 (http://www.buscaplus.com/robi/)  
Disallow: /
User-agent: CCBot/1.0 (+http://www.commoncrawl.org/bot.html)  
Disallow: /
User-agent: CDR/1.7.1 Simulator/0.7(+http://timewe.net) Profile/MIDP-1.0 Configuration/CLDC-1.0  
Disallow: /
User-agent: CE-Preload  
Disallow: /
User-agent: CFNetwork/x.x  
Disallow: /
User-agent: CHttpClient by Open Text Corporation  
Disallow: /
User-agent: CJ Spider/  
Disallow: /
User-agent: CJB.NET Proxy  
Disallow: /
User-agent: COAST WebMaster Pro/4.x.x.xx (Windows NT)  
Disallow: /
User-agent: CSE HTML Validator Professional (http://www.htmlvalidator.com/)  
Disallow: /
User-agent: Cabot/Nutch-0.9 (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)  
Disallow: /
User-agent: Cabot/Nutch-1.0-dev (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)  
Disallow: /
User-agent: CamelHttpStream/1.0  
Disallow: /
User-agent: Cancer Information and Support International;  
Disallow: /
User-agent: Carnegie_Mellon_University_Research_WebBOT-->PLEASE READ-->http://www.andrew.cmu.edu/~brgordon/webbot/index.html http://www.andrew.cmu.edu/~brgordon/webbot/index.html  
Disallow: /
User-agent: Carnegie_Mellon_University_WebCrawler http://www.andrew.cmu.edu/~brgordon/webbot/index.html  
Disallow: /
User-agent: Catall Spider  
Disallow: /
User-agent: CazoodleBot/CazoodleBot-0.1 (CazoodleBot Crawler; http://www.cazoodle.com/cazoodlebot; cazoodlebot@cazoodle.com)  
Disallow: /
User-agent: CentiverseBot  
Disallow: /
User-agent: CentiverseBot - investigator  
Disallow: /
User-agent: CentiverseBot/3.0 (http://www.centiverse-project.net)  
Disallow: /
User-agent: Ceramic Tile Installation Guide (http://www.floorstransformed.com)  
Disallow: /
User-agent: Charon/1.x (Amiga)  
Disallow: /
User-agent: CheckLinks/1.x.x  
Disallow: /
User-agent: CheckUrl  
Disallow: /
User-agent: CheckWeb  
Disallow: /
User-agent: Checkbot/1.xx LWP/5.xx  
Disallow: /
User-agent: Chilkat/1.0.0 (+http://www.chilkatsoft.com/ChilkatHttpUA.asp)  
Disallow: /
User-agent: China Local Browse 2.6  
Disallow: /
User-agent: Chitika ContentHit 1.0  
Disallow: /
User-agent: ChristCRAWLER 2.0  
Disallow: /
User-agent: CipinetBot (http://www.cipinet.com/bot.html)  
Disallow: /
User-agent: Cityreview Robot (+http://www.cityreview.org/crawler/)  
Disallow: /
User-agent: ClariaBot/1.0  
Disallow: /
User-agent: Claymont.com  
Disallow: /
User-agent: CloakDetect/0.9 (+http://fulltext.seznam.cz/)  
Disallow: /
User-agent: Clushbot/2.x (+http://www.clush.com/bot.html)  
Disallow: /
User-agent: Clushbot/3.x-BinaryFury (+http://www.clush.com/bot.html)  
Disallow: /
User-agent: Clushbot/3.xx-Ajax (+http://www.clush.com/bot.html)  
Disallow: /
User-agent: Clushbot/3.xx-Hector (+http://www.clush.com/bot.html)  
Disallow: /
User-agent: Clushbot/3.xx-Peleus (+http://www.clush.com/bot.html)  
Disallow: /
User-agent: CoBITSProbe  
Disallow: /
User-agent: Cocoal.icio.us/1.0 (v36) (Mac OS X; http://www.scifihifi.com/cocoalicious)  
Disallow: /
User-agent: ColdFusion  
Disallow: /
User-agent: ColdFusion (BookmarkTracker.com)  
Disallow: /
User-agent: Combine/2.0 http://combine.it.lth.se/  
Disallow: /
User-agent: Combine/3 http://combine.it.lth.se/  
Disallow: /
User-agent: Combine/x.0  
Disallow: /
User-agent: Commerce Browser Center  
Disallow: /
User-agent: Computer_and_Automation_Research_Institute_Crawler crawler@ilab.sztaki.hu  
Disallow: /
User-agent: Comrite/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)  
Disallow: /
User-agent: Contact  
Disallow: /
User-agent: ContactBot/0.2  
Disallow: /
User-agent: ContentSmartz  
Disallow: /
User-agent: Convera Internet Spider V6.x  
Disallow: /
User-agent: ConveraCrawler/0.2  
Disallow: /
User-agent: ConveraCrawler/0.9d (+http://www.authoritativeweb.com/crawl)  
Disallow: /
User-agent: ConveraMultiMediaCrawler/0.1 (+http://www.authoritativeweb.com/crawl)  
Disallow: /
User-agent: CoolBot  
Disallow: /
User-agent: CoralWebPrx/0.1.1x (See http://coralcdn.org/)  
Disallow: /
User-agent: CoteoNutchCrawler/Nutch-0.9 (info [at] coteo [dot] com)  
Disallow: /
User-agent: CougarSearch/0.x (+http://www.cougarsearch.com/faq.shtml)  
Disallow: /
User-agent: Covac TexAs Arachbot  
Disallow: /
User-agent: Cowbot-0.1 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)  
Disallow: /
User-agent: Cowbot-0.1.x (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)  
Disallow: /
User-agent: CrawlConvera0.1 (CrawlConvera@yahoo.com)  
Disallow: /
User-agent: Crawler  
Disallow: /
User-agent: Crawler (cometsearch@cometsystems.com)  
Disallow: /
User-agent: Crawler V 0.2.x admin@crawler.de  
Disallow: /
User-agent: Crawler admin@crawler.de  
Disallow: /
User-agent: CrawlerBoy Pinpoint.com  
Disallow: /
User-agent: Crawllybot/0.1 (Crawllybot; +http://www.crawlly.com; crawler@crawlly.com)  
Disallow: /
User-agent: CreativeCommons/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)  
Disallow: /
User-agent: CrocCrawler vx.3 [en] (http://www.croccrawler.com) (X11; I; Linux 2.0.44 i686)  
Disallow: /
User-agent: Cuam Ver0.050bx  
Disallow: /
User-agent: Cuasarbot/0.9b http://www.cuasar.com/spider_beta/   
Disallow: /
User-agent: CurryGuide SiteScan 1.1  
Disallow: /
User-agent: Custo x.x (www.netwu.com)  
Disallow: /
User-agent: Custom Spider www.bisnisseek.com /1.0  
Disallow: /
User-agent: CyberSpyder Link Test/2.1.12 (admin@mspennyworth.com)  
Disallow: /
User-agent: Cyberdog/2.0 (Macintosh; 68k)  
Disallow: /
User-agent: CydralSpider/1.x (Cydral Web Image Search; http://www.cydral.com)  
Disallow: /
User-agent: CydralSpider/3.0 (Cydral Image Search; http://www.cydral.com)  
Disallow: /
User-agent: DA 3.5 (www.lidan.com)  
Disallow: /
User-agent: DA 4.0  
Disallow: /
User-agent: DA 4.0 (www.downloadaccelerator.com)  
Disallow: /
User-agent: DA 5.0  
Disallow: /
User-agent: DA 7.0  
Disallow: /
User-agent: DBrowse 1.4b  
Disallow: /
User-agent: DBrowse 1.4d  
Disallow: /
User-agent: DC-Sakura/x.xx  
Disallow: /
User-agent: DDD  
Disallow: /
User-agent: DIIbot/1.2  
Disallow: /
User-agent: DISCo Pump x.x  
Disallow: /
User-agent: DNSRight.com WebBot Link Ckeck Tool. Report abuse to: dnsr@dnsright.com  
Disallow: /
User-agent: DSurf15a 01  
Disallow: /
User-agent: DSurf15a 71  
Disallow: /
User-agent: DSurf15a 81  
Disallow: /
User-agent: DSurf15a VA  
Disallow: /
User-agent: DTAAgent  
Disallow: /
User-agent: Dart Communications PowerTCP  
Disallow: /
User-agent: DataCha0s/2.0  
Disallow: /
User-agent: DataFountains/DMOZ Downloader  
Disallow: /
User-agent: DataFountains/DMOZ Feature Vector Corpus Creator (http://ivia.ucr.edu/useragents.shtml)  
Disallow: /
User-agent: DataFountains/Dmoz Downloader (http://ivia.ucr.edu/useragents.shtml)  
Disallow: /
User-agent: DataSpear/1.0 (Spider; http://www.dataspear.com/spider.html; spider@dataspear.com)  
Disallow: /
User-agent: DataSpearSpiderBot/0.2 (DataSpear Spider Bot; http://dssb.dataspear.com/bot.html; dssb@dataspear.com)  
Disallow: /
User-agent: DataparkSearch/4.47 (+http://dataparksearch.org/bot)  
Disallow: /
User-agent: DataparkSearch/4.xx (http://www.dataparksearch.org/)  
Disallow: /
User-agent: DatenBot( http://www.sicher-durchs-netz.de/bot.html)  
Disallow: /
User-agent: DaviesBot/1.7 (www.wholeweb.net)  
Disallow: /
User-agent: DeadLinkCheck/0.4.0 libwww-perl/5.xx  
Disallow: /
User-agent: Deep Link Calculator v1.0  
Disallow: /
User-agent: DeepIndex  
Disallow: /
User-agent: DeepIndex ( http://www.zetbot.com )  
Disallow: /
User-agent: DeepIndex (www.en.deepindex.com)  
Disallow: /
User-agent: DeepIndexer.ca  
Disallow: /
User-agent: DeleGate/9.0.5-fix1  
Disallow: /
User-agent: Demo Bot DOT 16b  
Disallow: /
User-agent: Demo Bot Z 16b  
Disallow: /
User-agent: Denmex websearch (http://search.denmex.com)  
Disallow: /
User-agent: Der gro�e BilderSauger 2.00u  
Disallow: /
User-agent: DevComponents.com HtmlDocument Object  
Disallow: /
User-agent: DiaGem/1.1 (http://www.skyrocket.gr.jp/diagem.html)  
Disallow: /
User-agent: Diamond/x.0  
Disallow: /
User-agent: DiamondBot  
Disallow: /
User-agent: DigOut4U  
Disallow: /
User-agent: Digger/1.0 JDK/1.3.0rc3  
Disallow: /
User-agent: Dillo/0.8.5-i18n-misc  
Disallow: /
User-agent: Dillo/0.x.x  
Disallow: /
User-agent: DittoSpyder  
Disallow: /
User-agent: DoCoMo/1.0/Nxxxi/c10  
Disallow: /
User-agent: DoCoMo/1.0/Nxxxi/c10/TB  
Disallow: /
User-agent: DoCoMo/1.0/P502i/c10 (Google CHTML Proxy/1.0)  
Disallow: /
User-agent: DoCoMo/2.0 P900iV(c100;TB;W24H11)   
Disallow: /
User-agent: DoCoMo/2.0 SH901iS(c100;TB;W24H12),gzip(gfe) (via translate.google.com)  
Disallow: /
User-agent: DoCoMo/2.0 SH902i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)  
Disallow: /
User-agent: DoCoMo/2.0/SO502i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)  
Disallow: /
User-agent: DocZilla/1.0 (Windows; U; WinNT4.0; en-US; rv:1.0.0) Gecko/20020804  
Disallow: /
User-agent: DonutP; Windows98SE  
Disallow: /
User-agent: Doubanbot/1.0 (bot@douban.com http://www.douban.com)  
Disallow: /
User-agent: Download Demon/3.x.x.x  
Disallow: /
User-agent: Download Druid 2.x  
Disallow: /
User-agent: Download Express 1.0  
Disallow: /
User-agent: Download Master  
Disallow: /
User-agent: Download Ninja 3.0  
Disallow: /
User-agent: Download Wonder  
Disallow: /
User-agent: Download-Tipp Linkcheck (http://download-tipp.de/)  
Disallow: /
User-agent: Download.exe(1.1) (+http://www.sql-und-xml.de/freeware-tools/)  
Disallow: /
User-agent: DownloadDirect.1.0  
Disallow: /
User-agent: Dr.Web (R) online scanner: http://online.drweb.com/  
Disallow: /
User-agent: Dragonfly File Reader  
Disallow: /
User-agent: Drecombot/1.0 (http://career.drecom.jp/bot.html)  
Disallow: /
User-agent: Drupal (+http://drupal.org/)  
Disallow: /
User-agent: Dual Proxy  
Disallow: /
User-agent: DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)  
Disallow: /
User-agent: Dumbot(version 0.1 beta - dumbfind.com)  
Disallow: /
User-agent: Dumbot(version 0.1 beta - http://www.dumbfind.com/dumbot.html)  
Disallow: /
User-agent: Dumbot(version 0.1 beta)  
Disallow: /
User-agent: EARTHCOM.info/1.x [www.earthcom.info]  
Disallow: /
User-agent: EARTHCOM.info/1.xbeta [www.earthcom.info]  
Disallow: /
User-agent: EBrowse 1.4b  
Disallow: /
User-agent: ELI/20070402:2.0 (DAUM RSS Robot, Daum Communications Corp.; +http://ws.daum.net/aboutkr.html)  
Disallow: /
User-agent: ELinks (0.x.x; Linux 2.4.20 i586; 132x60)  
Disallow: /
User-agent: ELinks/0.x.x (textmode; NetBSD 1.6.2 sparc; 132x43)  
Disallow: /
User-agent: EMPAS_ROBOT  
Disallow: /
User-agent: ES.NET_Crawler/2.0 (http://search.innerprise.net/)  
Disallow: /
User-agent: ESISmartSpider  
Disallow: /
User-agent: ESurf15a 15  
Disallow: /
User-agent: EasyDL/3.xx  
Disallow: /
User-agent: EasyDL/3.xx http://keywen.com/Encyclopedia/Bot  
Disallow: /
User-agent: EchO!/2.0  
Disallow: /
User-agent: Educate Search VxB  
Disallow: /
User-agent: EgotoBot/4.8 (+http://www.egoto.com/about.htm)  
Disallow: /
User-agent: EldoS TimelyWeb/3.x  
Disallow: /
User-agent: EmailSiphon  
Disallow: /
User-agent: EmailSpider  
Disallow: /
User-agent: EmailWolf 1.00  
Disallow: /
User-agent: EmeraldShield.com WebBot  
Disallow: /
User-agent: EmeraldShield.com WebBot (http://www.emeraldshield.com/webbot.aspx)  
Disallow: /
User-agent: EnaBot/1.x (http://www.enaball.com/crawler.html)  
Disallow: /
User-agent: Enfish Tracker  
Disallow: /
User-agent: Enterprise_Search/1.0  
Disallow: /
User-agent: Enterprise_Search/1.0.xxx  
Disallow: /
User-agent: Enterprise_Search/1.00.xxx;MSSQL (http://www.innerprise.net/es-spider.asp)  
Disallow: /
User-agent: EroCrawler  
Disallow: /
User-agent: EuripBot/0.x (+http://www.eurip.com) GetFile  
Disallow: /
User-agent: EuripBot/0.x (+http://www.eurip.com) GetRobots  
Disallow: /
User-agent: EuripBot/0.x (+http://www.eurip.com) PreCheck  
Disallow: /
User-agent: Eurobot/1.0 (http://www.ayell.eu)  
Disallow: /
User-agent: EvaalSE - bot@evaal.com  
Disallow: /
User-agent: Everest-Vulcan Inc./0.1 (R&D project; host=e-1-24; http://everest.vulcan.com/crawlerhelp)  
Disallow: /
User-agent: Everest-Vulcan Inc./0.1 (R&D project; http://everest.vulcan.com/crawlerhelp)  
Disallow: /
User-agent: Exabot-Images/1.0  
Disallow: /
User-agent: Exabot-Test/1.0  
Disallow: /
User-agent: Exabot/2.0  
Disallow: /
User-agent: Exabot/3.0  
Disallow: /
User-agent: ExactSearch  
Disallow: /
User-agent: ExactSeek Crawler/0.1  
Disallow: /
User-agent: Exalead NG/MimeLive Client (convert/http/0.120)  
Disallow: /
User-agent: Excalibur Internet Spider V6.5.4  
Disallow: /
User-agent: Execrawl/1.0 (Execrawl; http://www.execrawl.com/; bot@execrawl.com)  
Disallow: /
User-agent: ExperimentalHenrytheMiragoRobot  
Disallow: /
User-agent: Expired Domain Sleuth  
Disallow: /
User-agent: Express WebPictures (www.express-soft.com)  
Disallow: /
User-agent: ExtractorPro  
Disallow: /
User-agent: Extreme Picture Finder  
Disallow: /
User-agent: EyeCatcher (Download-tipp.de)/1.0  
Disallow: /
User-agent: FANGCrawl/0.01  
Disallow: /
User-agent: FARK.com link verifier  
Disallow: /
User-agent: FAST Enterprise Crawler 6 (Experimental)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Cobra Development (admin@fastsearch.com)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Comperio AS (sts@comperio.no)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by FAST (FAST)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Pages Jaunes (pvincent@pagesjaunes.fr)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by Singapore Press Holdings (crawler@sphsearch.sg)  
Disallow: /
User-agent: FAST Enterprise Crawler 6 used by WWU (wardi@uni-muenster.de)  
Disallow: /
User-agent: FAST Enterprise Crawler/6 (www.fastsearch.com)  
Disallow: /
User-agent: FAST Enterprise Crawler/6.4 (helpdesk at fast.no)  
Disallow: /
User-agent: FAST FirstPage retriever (compatible; MSIE 5.5; Mozilla/4.0)  
Disallow: /
User-agent: FAST MetaWeb Crawler (helpdesk at fastsearch dot com)  
Disallow: /
User-agent: FAST-WebCrawler/2.2.10 (Multimedia Search) (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)  
Disallow: /
User-agent: FAST-WebCrawler/2.2.6 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)  
Disallow: /
User-agent: FAST-WebCrawler/2.2.7 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no  
Disallow: /
User-agent: FAST-WebCrawler/2.2.8 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no  
Disallow: /
User-agent: FAST-WebCrawler/3.2 test  
Disallow: /
User-agent: FAST-WebCrawler/3.3 (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)  
Disallow: /
User-agent: FAST-WebCrawler/3.4/Nirvana (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)  
Disallow: /
User-agent: FAST-WebCrawler/3.4/PartnerSite (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)  
Disallow: /
User-agent: FAST-WebCrawler/3.5 (atw-crawler at fast dot no; http://fast.no/support.php?c=faqs/crawler)  
Disallow: /
User-agent: FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)  
Disallow: /
User-agent: FAST-WebCrawler/3.6/FirstPage (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)  
Disallow: /
User-agent: FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)  
Disallow: /
User-agent: FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)  
Disallow: /
User-agent: FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)  
Disallow: /
User-agent: FAST-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)  
Disallow: /
User-agent: FAST-WebCrawler/3.x Multimedia  
Disallow: /
User-agent: FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no)  
Disallow: /
User-agent: FDM 1.x  
Disallow: /
User-agent: FDM 2.x  
Disallow: /
User-agent: FFC Trap Door Spider  
Disallow: /
User-agent: FLATARTS_FAVICO  
Disallow: /
User-agent: FSurf15a 01  
Disallow: /
User-agent: FaEdit/2.0.x  
Disallow: /
User-agent: Factbot 1.09 (see http://www.factbites.com/webmasters.php)  
Disallow: /
User-agent: FairAd Client  
Disallow: /
User-agent: Fast Crawler Gold Edition  
Disallow: /
User-agent: Fast PartnerSite Crawler  
Disallow: /
User-agent: FastBug http://www.ay-up.com  
Disallow: /
User-agent: FastCrawler 3.0.1 (crawler@1klik.dk)  
Disallow: /
User-agent: FastSearch Web Crawler for Verizon SuperPages (kevin.watters@fastsearch.com)  
Disallow: /
User-agent: FavIconizer  
Disallow: /
User-agent: FavOrg  
Disallow: /
User-agent: Favcollector/2.0 (info@favcollector.com http://www.favcollector.com/)  
Disallow: /
User-agent: Favorites Checking (http://campulka.net)  
Disallow: /
User-agent: Favorites Sweeper v.2.03  
Disallow: /
User-agent: Faxobot/1.0  
Disallow: /
User-agent: Feed Seeker Bot (RSS Feed Seeker http://www.MyNewFavoriteThing.com/fsb.php)  
Disallow: /
User-agent: Feed24.com  
Disallow: /
User-agent: Feed::Find/0.0x  
Disallow: /
User-agent: FeedChecker/0.01  
Disallow: /
User-agent: FeedDemon/2.7 (http://www.newsgator.com/; Microsoft Windows XP)  
Disallow: /
User-agent: FeedForAll rss2html.php v2  
Disallow: /
User-agent: FeedHub FeedDiscovery/1.0 (http://www.feedhub.com)  
Disallow: /
User-agent: FeedHub MetaDataFetcher/1.0 (http://www.feedhub.com)  
Disallow: /
User-agent: FeedZcollector v1.x (Platinum) http://www.feeds4all.com/feedzcollector  
Disallow: /
User-agent: Feedable/0.1 (compatible; MSIE 6.0; Windows NT 5.1)  
Disallow: /
User-agent: Feedfetcher-Google-iGoogleGadgets; (+http://www.google.com/feedfetcher.html)  
Disallow: /
User-agent: Feedfetcher-Google; (+http://www.google.com/feedfetcher.html)  
Disallow: /
User-agent: Feedreader 3.xx (Powered by Newsbrain)  
Disallow: /
User-agent: Feedshow/x.0 (http://www.feedshow.com; 1 subscriber)  
Disallow: /
User-agent: FeedshowOnline (http://www.feedshow.com)  
Disallow: /
User-agent: Feedster Crawler/3.0; Feedster, Inc.  
Disallow: /
User-agent: Felix - Mixcat Crawler (+http://mixcat.com)  
Disallow: /
User-agent: Filangy/0.01-beta (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)  
Disallow: /
User-agent: Filangy/1.0x (Filangy; http://www.filangy.com/filangyinfo.jsp?inc=robots.jsp; filangy-agent@filangy.com)  
Disallow: /
User-agent: Filangy/1.0x (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)  
Disallow: /
User-agent: FileHound x.x  
Disallow: /
User-agent: Filtrbox/1.0  
Disallow: /
User-agent: FindAnISP.com_ISP_Finder_v99a  
Disallow: /
User-agent: Findexa Crawler (http://www.findexa.no/gulesider/article26548.ece)  
Disallow: /
User-agent: FineBot  
Disallow: /
User-agent: Finjan-prefetch  
Disallow: /
User-agent: Firefly/1.0  
Disallow: /
User-agent: Firefly/1.0 (compatible; Mozilla 4.0; MSIE 5.5)  
Disallow: /
User-agent: Firefox (kastaneta03@hotmail.com)  
Disallow: /
User-agent: Firefox_1.0.6 (kasparek@naparek.cz)  
Disallow: /
User-agent: FirstGov.gov Search - POC:firstgov.webmasters@gsa.gov  
Disallow: /
User-agent: Flapbot/0.7.2 (Flaptor Crawler; http://www.flaptor.com; crawler at flaptor period com)  
Disallow: /
User-agent: FlashGet  
Disallow: /
User-agent: Flexum spider  
Disallow: /
User-agent: Flexum/2.0  
Disallow: /
User-agent: FlickBot 2.0 RPT-HTTPClient/0.3-3  
Disallow: /
User-agent: FocusedSampler/1.0  
Disallow: /
User-agent: Folkd.com Spider/0.1 beta 1 (www.folkd.com)  
Disallow: /
User-agent: Fooky.com/ScorpionBot/ScoutOut; http://www.fooky.com/scorpionbots  
Disallow: /
User-agent: Francis/1.0 (francis@neomo.de http://www.neomo.de/)  
Disallow: /
User-agent: Franklin Locator 1.8  
Disallow: /
User-agent: FreeFind.com-SiteSearchEngine/1.0 (http://freefind.com; spiderinfo@freefind.com)  
Disallow: /
User-agent: FreshDownload/x.xx  
Disallow: /
User-agent: FreshNotes crawler< report problems to crawler-at-freshnotes-dot-com  
Disallow: /
User-agent: Full Web Bot 0416B  
Disallow: /
User-agent: Full Web Bot 0516B  
Disallow: /
User-agent: Full Web Bot 2816B  
Disallow: /
User-agent: FuseBulb.Com  
Disallow: /
User-agent: FyberSpider (+http://www.fybersearch.com/fyberspider.php)
Disallow: /
User-agent: GAIS Robot/1.0B2  
Disallow: /
User-agent: GNODSPIDER (www.gnod.net)  
Disallow: /
User-agent: GOFORITBOT ( http://www.goforit.com/about/ )  
Disallow: /
User-agent: GSiteCrawler/v1.xx rev. xxx (http://gsitecrawler.com/)  
Disallow: /
User-agent: Gagglebot  
Disallow: /
User-agent: Gaisbot/3.0 (indexer@gais.cs.ccu.edu.tw; http://gais.cs.ccu.edu.tw/robot.php)  
Disallow: /
User-agent: Gaisbot/3.0+(robot06@gais.cs.ccu.edu.tw;+http://gais.cs.ccu.edu.tw/robot.php)  
Disallow: /
User-agent: GalaxyBot/1.0 (http://www.galaxy.com/galaxybot.html)  
Disallow: /
User-agent: Gallent Search Spider v1.4 Robot 2 (http://robot.GallentSearch.com)  
Disallow: /
User-agent: Gamespy_Arcade  
Disallow: /
User-agent: GammaSpider/1.0  
Disallow: /
User-agent: Generic Mobile Phone (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)  
Disallow: /
User-agent: GeoBot/1.0  
Disallow: /
User-agent: GeoURLBot 1.0 (http://geourl.org)  
Disallow: /
User-agent: GeonaBot 1.x; http://www.geona.com/  
Disallow: /
User-agent: GetBot  
Disallow: /
User-agent: GetRight/3.x.x  
Disallow: /
User-agent: GetRight/4.5xx  
Disallow: /
User-agent: GetRight/4.x  
Disallow: /
User-agent: GetRight/4.x[a-e]  
Disallow: /
User-agent: GetRight/6.1 (Pro)  
Disallow: /
User-agent: GetRightPro/6.0beta2  
Disallow: /
User-agent: GetWeb/0.1 libwww-perl/5.16  
Disallow: /
User-agent: GhostRouteHunter/20021130 (https://www.sixxs.net/tools/grh/; info@sixxs.net)  
Disallow: /
User-agent: Gigabot/2.0 (gigablast.com)  
Disallow: /
User-agent: Gigabot/2.0/gigablast.com/spider.html  
Disallow: /
User-agent: Gigabot/2.0; http://www.gigablast.com/spider.html  
Disallow: /
User-agent: Gigabot/2.0att  
Disallow: /
User-agent: Gigabot/3.0 (http://www.gigablast.com/spider.html)  
Disallow: /
User-agent: Gigabot/x.0  
Disallow: /
User-agent: GigabotSiteSearch/2.0 (sitesearch.gigablast.com)  
Disallow: /
User-agent: Go!Zilla 3.x (www.gozilla.com)  
Disallow: /
User-agent: Go!Zilla/4.x.x.xx  
Disallow: /
User-agent: Go-Ahead-Got-It/1.1  
Disallow: /
User-agent: GoForIt.com  
Disallow: /
User-agent: GoGuides.Org Link Check  
Disallow: /
User-agent: Goblin/0.9 (http://www.goguides.org/)  
Disallow: /
User-agent: Goblin/0.9.x (http://www.goguides.org/goblin-info.html)  
Disallow: /
User-agent: GoldenFeed Spider 1.0 (http://www.goldenfeed.com)  
Disallow: /
User-agent: Goldfire Server  
Disallow: /
User-agent: Goofer/0.2  
Disallow: /
User-agent: GrapeFX/0.3 libwww/5.4.0  
Disallow: /
User-agent: GreatNews/1.0  
Disallow: /
User-agent: GreenBrowser  
Disallow: /
User-agent: GrigorBot 0.8 (http://www.grigor.biz/bot.html)  
Disallow: /
User-agent: Gromit/1.0  
Disallow: /
User-agent: Guestbook Auto Submitter  
Disallow: /
User-agent: Gulliver/1.3  
Disallow: /
User-agent: Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)  
Disallow: /
User-agent: Gungho/0.08004 (http://code.google.com/p/gungho-crawler/wiki/Index)  
Disallow: /
User-agent: GurujiBot/1.0 (+http://www.guruji.com/WebmasterFAQ.html)  
Disallow: /
User-agent: GurujiImageBot/1.0 (+http://www.guruji.com/en/WebmasterFAQ.html)  
Disallow: /
User-agent: HLoader  
Disallow: /
User-agent: HPL/Nutch-0.9 -  
Disallow: /
User-agent: HTML2JPG Blackbox, http://www.html2jpg.com  
Disallow: /
User-agent: HTML2JPG Enterprise  
Disallow: /
User-agent: HTMLParser/1.x  
Disallow: /
User-agent: HTTP Retriever  
Disallow: /
User-agent: HTTP::Lite/2.x.x  
Disallow: /
User-agent: HTTPEyes  
Disallow: /
User-agent: HTTPResume v. 1.x  
Disallow: /
User-agent: HappyFunBot/1.1  
Disallow: /
User-agent: Harvest-NG/1.0.2  
Disallow: /
User-agent: Haste/0.12 (HOME: http://haste.kytoon.com/)  
Disallow: /
User-agent: Hatena Antenna/0.4 (http://a.hatena.ne.jp/help#robot)  
Disallow: /
User-agent: Hatena Mobile Gateway/1.0  
Disallow: /
User-agent: Hatena Pagetitle Agent/1.0  
Disallow: /
User-agent: Hatena RSS/0.3 (http://r.hatena.ne.jp)  
Disallow: /
User-agent: HatenaScreenshot/1.0 (checker)  
Disallow: /
User-agent: HeinrichderMiragoRobot  
Disallow: /
User-agent: HeinrichderMiragoRobot (http://www.miragorobot.com/scripts/deinfo.asp)  
Disallow: /
User-agent: Helix/1.x ( http://www.sitesearch.ca/helix/)  
Disallow: /
User-agent: HenriLeRobotMirago (http://www.miragorobot.com/scripts/frinfo.asp)  
Disallow: /
User-agent: HenryTheMiragoRobot (http://www.miragorobot.com/scripts/mrinfo.asp)  
Disallow: /
User-agent: HenrytheMiragoRobot  
Disallow: /
User-agent: Hi! I'm CsCrawler my homepage: http://www.kde.cs.uni-kassel.de/lehre/ss2005/googlespam/crawler.html RPT-HTTPClient/0.3-3  
Disallow: /
User-agent: HiDownload  
Disallow: /
User-agent: Hippias/0.9 Beta  
Disallow: /
User-agent: HitList  
Disallow: /
User-agent: Hitwise Spider v1.0 http://www.hitwise.com  
Disallow: /
User-agent: HomePageSearch(hpsearch.uni-trier.de)  
Disallow: /
User-agent: Homerbot: www.homerweb.com  
Disallow: /
User-agent: Honda-Search/0.7.2 (Nutch; http://lucene.apache.org/nutch/bot.html; search@honda-search.com)  
Disallow: /
User-agent: HooWWWer/2.1.3 (debugging run) (+http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-info<at>hiit.fi)  
Disallow: /
User-agent: HooWWWer/2.1.x ( http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-info<at>hiit.fi)  
Disallow: /
User-agent: HotJava/1.0.1/JRE1.1.x  
Disallow: /
User-agent: Hotzonu/x.0  
Disallow: /
User-agent: Html Link Validator (www.lithopssoft.com)  
Disallow: /
User-agent: Hybrid/1.2 [en] (OS Independent)  
Disallow: /
User-agent: HyperEstraier/1.x.xx  
Disallow: /
User-agent: IAArchiver-1.0  
Disallow: /
User-agent: IBrowse/2.2 (AmigaOS 3.5)  
Disallow: /
User-agent: IBrowse/2.2 (Windows 3.1)  
Disallow: /
User-agent: ICC-Crawler(Mozilla-compatible; http://kc.nict.go.jp/icc/crawl.html; icc-crawl(at)ml(dot)nict(dot)go(dot)jp)  
Disallow: /
User-agent: ICC-Crawler(Mozilla-compatible;http://kc.nict.go.jp/icc/crawl.html;icc-crawl-contact(at)ml(dot)nict(dot)go(dot)jp)  
Disallow: /
User-agent: ICCrawler - ICjobs (http://www.icjobs.de/bot.htm)  
Disallow: /
User-agent: ICE Browser/5.05 (Java 1.4.0; Windows 2000 5.0 x86)  
Disallow: /
User-agent: ICOO Loader v.x.x.x  
Disallow: /
User-agent: ICRA_label_spider/x.0  
Disallow: /
User-agent: IDA  
Disallow: /
User-agent: IEFav172Free  
Disallow: /
User-agent: IIITBOT/1.1 (Indian Language Web Search Engine; http://webkhoj.iiit.net; pvvpr at iiit dot ac dot in)  
Disallow: /
User-agent: INFOMINE/8.0 Adders  
Disallow: /
User-agent: INFOMINE/8.0 RemoteServices  
Disallow: /
User-agent: INFOMINE/8.0 VLCrawler (http://infomine.ucr.edu/useragents)  
Disallow: /
User-agent: INGRID/3.0 MT (webcrawler@NOSPAMexperimental.net; http://webmaster.ilse.nl/jsp/webmaster.jsp)  
Disallow: /
User-agent: IP*Works! V5 HTTP/S Component - by /n software - www.nsoftware.com  
Disallow: /
User-agent: IP2LocationBot/1.0 <a href="http://www.ip2location.com">http://www.ip2location.com</a>  
Disallow: /
User-agent: IP2MapBot/1.1 <a href=http://www.ip2map.com>http://www.ip2map.com</a>  
Disallow: /
User-agent: IPiumBot laurion(dot)com  
Disallow: /
User-agent: IRLbot/1.0 ( http://irl.cs.tamu.edu/crawler)  
Disallow: /
User-agent: IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler/)  
Disallow: /
User-agent: ISC Systems iRc Search 2.1  
Disallow: /
User-agent: IUPUI Research Bot v 1.9a  
Disallow: /
User-agent: IWAgent/ 1.0 - www.brandprotect.com  
Disallow: /
User-agent: IconSurf/2.0 favicon finder (see http://iconsurf.com/robot.html)  
Disallow: /
User-agent: IconSurf/2.0 favicon monitor (see http://iconsurf.com/robot.html)  
Disallow: /
User-agent: IlTrovatore-Setaccio ( http://www.iltrovatore.it)  
Disallow: /
User-agent: IlTrovatore-Setaccio/1.2 ( http://www.iltrovatore.it/aiuto/faq.html)  
Disallow: /
User-agent: IlTrovatore/1.2 (IlTrovatore; http://www.iltrovatore.it/bot.html; bot@iltrovatore.it)  
Disallow: /
User-agent: IlseBot/1.x  
Disallow: /
User-agent: Iltrovatore-Setaccio/0.3-dev (Indexing; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)  
Disallow: /
User-agent: Iltrovatore-Setaccio/1.2 (It-bot; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)  
Disallow: /
User-agent: ImageVisu/v4.x.x  
Disallow: /
User-agent: ImageWalker/2.0 (www.bdbrandprotect.com)  
Disallow: /
User-agent: Incutio HttpClient v0.x  
Disallow: /
User-agent: IncyWincy data gatherer(webmaster@loopimprovements.com  
Disallow: /
User-agent: IncyWincy page crawler(webmaster@loopimprovements.com  
Disallow: /
User-agent: IncyWincy(http://www.look.com)  
Disallow: /
User-agent: IncyWincy(http://www.loopimprovements.com/robot.html)  
Disallow: /
User-agent: IncyWincy/2.1(loopimprovements.com/robot.html)  
Disallow: /
User-agent: IndexTheWeb.com Crawler7  
Disallow: /
User-agent: Industry Program 1.0.x  
Disallow: /
User-agent: Inet library  
Disallow: /
User-agent: InetURL/1.0  
Disallow: /
User-agent: InfoFly/1.0 (http://www.versions-project.org/)  
Disallow: /
User-agent: InfoLink/1.x  
Disallow: /
User-agent: InfoNaviRobot(F107)  
Disallow: /
User-agent: Inktomi Search  
Disallow: /
User-agent: InnerpriseBot/1.0 (http://www.innerprise.com/)  
Disallow: /
User-agent: Insitor.com search and find world wide!  
Disallow: /
User-agent: Insitornaut  
Disallow: /
User-agent: InstallShield DigitalWizard  
Disallow: /
User-agent: Intelix/0.x (cs; http://www.microton.cz/intelix/; microton@@microton.cz)  
Disallow: /
User-agent: Interarchy/x.x.x (InterarchyCrawler)  
Disallow: /
User-agent: Internet Ninja x.0  
Disallow: /
User-agent: InternetArchive/0.8-dev(Nutch;http://lucene.apache.org/nutch/bot.html;nutch-agent@lucene.apache  
Disallow: /
User-agent: InternetLinkAgent/3.1  
Disallow: /
User-agent: InternetSeer.com  
Disallow: /
User-agent: IpselonBot/0.xx-beta (Ipselon; http://www.ipselon.com; ipselonbot@ipselon.com)  
Disallow: /
User-agent: Iria/1.xxa  
Disallow: /
User-agent: IrssiUrlLog/0.2  
Disallow: /
User-agent: Irvine/1.x.x  
Disallow: /
User-agent: J-PHONE/3.0/J-SH07  
Disallow: /
User-agent: JBH Agent 2.0  
Disallow: /
User-agent: JCheckLinks/0.1 RPT-HTTPClient/0.3-1  
Disallow: /
User-agent: JDK/1.1  
Disallow: /
User-agent: JOC Web Spider  
Disallow: /
User-agent: JRTS Check Favorites Utility  
Disallow: /
User-agent: JRTwine Software Check Favorites Utility  
Disallow: /
User-agent: Jabot/6.x (http://odin.ingrid.org/)  
Disallow: /
User-agent: Jabot/7.x.x (http://odin.ingrid.org/)  
Disallow: /
User-agent: Jack  
Disallow: /
User-agent: Jakarta Commons-HttpClient/2.0xxx  
Disallow: /
User-agent: Jakarta Commons-HttpClient/3.0-rcx  
Disallow: /
User-agent: Jambot/0.1.x (Jambot; http://www.jambot.com/blog; crawler@jambot.com)  
Disallow: /
User-agent: Jambot/0.2.1 (Jambot; http://www.jambot.com/blog/static.php?page=webmaster-robot; crawler@jambot.com)  
Disallow: /
User-agent: Java 1.1  
Disallow: /
User-agent: Java/1.4.1_01  
Disallow: /
User-agent: Java1.0.21.0  
Disallow: /
User-agent: Java1.1.xx.x  
Disallow: /
User-agent: Java1.3.0rc1  
Disallow: /
User-agent: Java1.3.x  
Disallow: /
User-agent: Java1.4.0  
Disallow: /
User-agent: Jayde Crawler. http://www.jayde.com  
Disallow: /
User-agent: Jeode/1.x.x  
Disallow: /
User-agent: JetBrains Omea Reader 1.0.x (http://www.jetbrains.com/omea_reader/)  
Disallow: /
User-agent: JetBrains Omea Reader 2.0 Release Candidate 1 (http://www.jetbrains.com/omea_reader/)  
Disallow: /
User-agent: JetCar  
Disallow: /
User-agent: Jetbot/1.0  
Disallow: /
User-agent: Jigsaw/2.2.0 W3C_CSS_Validator_JFouffa/2.0  
Disallow: /
User-agent: JoBo/1.x (http://www.matuschek.net/jobo.html)  
Disallow: /
User-agent: JoBo/@JOBO_VERSION@(http://www.matuschek.net/jobo.html)  
Disallow: /
User-agent: JobSpider_BA/1.1  
Disallow: /
User-agent: JordoMedia/1.0 RSS File Reader (http://www.jordomedia.com)  
Disallow: /
User-agent: Journster [alpha] (http://journster.com/)  
Disallow: /
User-agent: Journster.com RSS/Atom aggregator 0.5 (http://www.journster.com/bot.phtml)  
Disallow: /
User-agent: Jyxobot/x  
Disallow: /
User-agent: K-Meleon/0.6 (Windows; U; Windows NT 5.1; en-US; rv:0.9.5) Gecko/20011011  
Disallow: /
User-agent: KAIST AITrc Crawler  
Disallow: /
User-agent: KDDI-SN22 UP.Browser/6.0.7 (GUI) MMP/1.1 (Google WAP Proxy/1.0)  
Disallow: /
User-agent: KE_1.0/2.0 libwww/5.2.8  
Disallow: /
User-agent: KFSW-Bot (Version: 1.01 powered by KFSW www.kfsw.de)  
Disallow: /
User-agent: KIT-Fireball/2.0  
Disallow: /
User-agent: KIT-Fireball/2.0 (compatible; Mozilla 4.0; MSIE 5.5)  
Disallow: /
User-agent: KSbot/1.0 (KnowledgeStorm crawler; http://www.knowledgestorm.com/resources/content/crawler/index.html; crawleradmin@knowledgestorm.com)  
Disallow: /
User-agent: KakleBot - www.kakle.com/0.1 (KakleBot - www.kakle.com; http:// www.kakle.com/bot.html; support@kakle.com)  
Disallow: /
User-agent: Kapere (http://www.kapere.com)  
Disallow: /
User-agent: Kazehakase/0.x.x.[x]  
Disallow: /
User-agent: Kenjin Spider  
Disallow: /
User-agent: Kevin http://dznet.com/kevin/  
Disallow: /
User-agent: Kevin http://websitealert.net/kevin/  
Disallow: /
User-agent: Klondike/1.50 (WSP Win32) (Google WAP Proxy/1.0)  
Disallow: /
User-agent: KnowItAll(knowitall@cs.washington.edu)  
Disallow: /
User-agent: Knowledge.com/0.x  
Disallow: /
User-agent: Kontiki Client x.xx  
Disallow: /
User-agent: Krugle/Krugle,Nutch/0.8+ (Krugle web crawler; http://www.krugle.com/crawler/info.html; webcrawler@krugle.com)  
Disallow: /
User-agent: KummHttp/1.1 (compatible; KummClient; Linux rulez)  
Disallow: /
User-agent: LARBIN-EXPERIMENTAL (efp@gmx.net)  
Disallow: /
User-agent: LECodeChecker/3.0 libgetdoc/1.0  
Disallow: /
User-agent: LEIA/2.90  
Disallow: /
User-agent: LEIA/3.01pr (LEIAcrawler; [SNIP])  
Disallow: /
User-agent: LG/U8138/v1.0  
Disallow: /
User-agent: LMQueueBot/0.2  
Disallow: /
User-agent: LNSpiderguy  
Disallow: /
User-agent: LTI/LemurProject Nutch Spider/Nutch-1.0-dev (Research spider using Nutch; http://www.lemurproject.org; mhoy@cs.cmu.edu)  
Disallow: /
User-agent: LTI/LemurProject Nutch Spider/Nutch-1.0-dev (lti crawler for CMU; http://www.lti.cs.cmu.edu; changkuk at cmu dot edu)  
Disallow: /
User-agent: LWP::Simple/5.22  
Disallow: /
User-agent: LWP::Simple/5.36  
Disallow: /
User-agent: LWP::Simple/5.48  
Disallow: /
User-agent: LWP::Simple/5.50  
Disallow: /
User-agent: LWP::Simple/5.51  
Disallow: /
User-agent: LWP::Simple/5.53  
Disallow: /
User-agent: LWP::Simple/5.63  
Disallow: /
User-agent: LWP::Simple/5.803  
Disallow: /
User-agent: Lachesis  
Disallow: /
User-agent: LapozzBot/1.4 ( http://robot.lapozz.com)  
Disallow: /
User-agent: LapozzBot/1.5 (+http://robot.lapozz.hu)  
Disallow: /
User-agent: LeapTag/0.8.1.beta081.r3750 (compatible; Mozilla 4.0; MSIE 5.5; robot@yoriwa.com)  
Disallow: /
User-agent: LeechGet 200x (www.leechget.de)  
Disallow: /
User-agent: LetsCrawl.com/1.0 +http://letscrawl.com/  
Disallow: /
User-agent: LexiBot/1.00  
Disallow: /
User-agent: Libby_1.1/libwww-perl/5.47  
Disallow: /
User-agent: LibertyW (+http://www.lw01.com)  
Disallow: /
User-agent: Liferea/0.x.x (Linux; en_US.UTF-8; http://liferea.sf.net/)  
Disallow: /
User-agent: Liferea/1.x.x (Linux; es_ES.UTF-8; http://liferea.sf.net/)  
Disallow: /
User-agent: LightningDownload/1.0beta2  
Disallow: /
User-agent: LightningDownload/1.x.x  
Disallow: /
User-agent: LightningDownload/1.x.x [Accelerated x]  
Disallow: /
User-agent: LijitSpider/Nutch-0.9 (Reports crawler; http://www.lijit.com/; info(a)lijit(d)com)  
Disallow: /
User-agent: Lincoln State Web Browser  
Disallow: /
User-agent: Link Valet Online 1.x  
Disallow: /
User-agent: LinkAlarm/2.x  
Disallow: /
User-agent: LinkCheck (linkcheck@inter7.com http://www.inter7.com/linkcheck)  
Disallow: /
User-agent: LinkLint-checkonly/2.x.x  
Disallow: /
User-agent: LinkLint-spider/2.x.x  
Disallow: /
User-agent: LinkPimpin v1.0  
Disallow: /
User-agent: LinkProver 2.1  
Disallow: /
User-agent: LinkScan/11.0beta2 UnixShareware robot from Elsop.com (used by Indiafocus/Indiainfo)  
Disallow: /
User-agent: LinkScan/9.0g Unix  
Disallow: /
User-agent: LinkScan/x.x Unix  
Disallow: /
User-agent: LinkSonar/1.35  
Disallow: /
User-agent: LinkSweeper/1.x  
Disallow: /
User-agent: LinkWalker  
Disallow: /
User-agent: Linkbot  
Disallow: /
User-agent: Linkbot x.0  
Disallow: /
User-agent: Links (0.9x; Linux 2.4.7-10 i686)  
Disallow: /
User-agent: Links (0.9xpre12; Linux 2.2.14-5.0 i686; 80x24)  
Disallow: /
User-agent: Links (2.xpre7; Linux 2.4.18 i586; x)  
Disallow: /
User-agent: Links - http://gossamer-threads.com/scripts/links/  
Disallow: /
User-agent: Links 2.0 (http://gossamer-threads.com/scripts/links/)  
Disallow: /
User-agent: Links SQL (http://gossamer-threads.com/scripts/links-sql/)  
Disallow: /
User-agent: Links4US-Crawler, (+http://links4us.com/)  
Disallow: /
User-agent: LinksManager.com (http://linksmanager.com/linkchecker.html)  
Disallow: /
User-agent: ListBidBot (freelance job spider http://listbid.com)<a href=http://listbid.com>Freelance</a>  
Disallow: /
User-agent: LiveTrans/Nutch-0.9 (maintainer: cobain at iis dot sinica dot edu dot tw; http://wkd.iis.sinica.edu.tw/LiveTrans/)  
Disallow: /
User-agent: Llaut/1.0 (http://mnm.uib.es/~gallir/llaut/bot.html)  
Disallow: /
User-agent: LocalBot/1.0 ( http://www.localbot.co.uk/)  
Disallow: /
User-agent: LocalcomBot/1.2.x ( http://www.local.com/bot.htm)  
Disallow: /
User-agent: Lockstep Spider/1.0  
Disallow: /
User-agent: Look.com  
Disallow: /
User-agent: Lotus-Notes/4.5 ( Windows-NT )  
Disallow: /
User-agent: LotusDiscovery/x.0 (compatible; Mozilla 4.0; MSIE 4.01; Windows NT)  
Disallow: /
User-agent: Lovel as 1.0 ( +http://www.everatom.com)  
Disallow: /
User-agent: Lunascape  
Disallow: /
User-agent: Lycos_Spider_(T-Rex)  
Disallow: /
User-agent: Lycos_Spider_(modspider)  
Disallow: /
User-agent: Lynx/2-4-2 (Bobcat/0.5 [DOS] Jp Beta04)  
Disallow: /
User-agent: Lynx/2.6 libwww-FM/2.14  
Disallow: /
User-agent: Lynx/2.8 (;http://seebot.org)  
Disallow: /
User-agent: Lynx/2.8.3dev.9 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6  
Disallow: /
User-agent: Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6c (human-guided@lerly.net)  
Disallow: /
User-agent: MARTINI  
Disallow: /
User-agent: MDbot/1.0 (+http://www.megadownload.net/bot.html)  
Disallow: /
User-agent: MFC Foundation Class Library 4.0  
Disallow: /
User-agent: MFC_Tear_Sample  
Disallow: /
User-agent: MFHttpScan  
Disallow: /
User-agent: MIIxpc/4.2  
Disallow: /
User-agent: MJ12bot/vx.x.x (http://majestic12.co.uk/bot.php?+)  
Disallow: /
User-agent: MJ12bot/vx.x.x (http://www.majestic12.co.uk/projects/dsearch/mj12bot.php)  
Disallow: /
User-agent: MJBot (SEO assessment)  
Disallow: /
User-agent: MLBot (www.metadatalabs.com)  
Disallow: /
User-agent: MQBOT/Nutch-0.9-dev (MQBOT Nutch Crawler; http://falcon.cs.uiuc.edu; mqbot@cs.uiuc.edu)  
Disallow: /
User-agent: MQbot metaquerier.cs.uiuc.edu/crawler  
Disallow: /
User-agent: MSFrontPage/4.0  
Disallow: /
User-agent: MSIE 4.0 (Win95)  
Disallow: /
User-agent: MSIE-5.13 (larbin@unspecified.mail)  
Disallow: /
User-agent: MVAClient  
Disallow: /
User-agent: MaSagool/1.0 (MaSagool; http://sagool.jp/; info@sagool.jp)  
Disallow: /
User-agent: Mac Finder 1.0.xx  
Disallow: /
User-agent: Mackster( http://www.ukwizz.com )  
Disallow: /
User-agent: Mag-Net  
Disallow: /
User-agent: MagicWML/1.0 (forcewml)  
Disallow: /
User-agent: MagpieRSS/0.7x (+http://magpierss.sf.net)  
Disallow: /
User-agent: Mahiti.Com/Mahiti Crawler-1.0 (Mahiti.Com; http://mahiti.com ; mahiti.com)  
Disallow: /
User-agent: Mail.Ru/1.0  
Disallow: /
User-agent: MantraAgent  
Disallow: /
User-agent: MapoftheInternet.com ( http://MapoftheInternet.com)  
Disallow: /
User-agent: Mariner/5.1b [de] (Win95; I ;Kolibri gncwebbot)  
Disallow: /
User-agent: Marketwave Hit List  
Disallow: /
User-agent: Martini  
Disallow: /
User-agent: Marvin v0.3  
Disallow: /
User-agent: Mass Downloader 2.x  
Disallow: /
User-agent: MasterSeek  
Disallow: /
User-agent: Mata Hari/2.00   
Disallow: /
User-agent: Matrix S.p.A. - FAST Enterprise Crawler 6 (Unknown admin e-mail address)  
Disallow: /
User-agent: McBot/5.001 (windows; U; NT4.0; en-us)  
Disallow: /
User-agent: Media Player Classic  
Disallow: /
User-agent: MediaCrawler-1.0 (Experimental)  
Disallow: /
User-agent: MediaSearch/0.1  
Disallow: /
User-agent: Mediapartners-Google/2.1 ( http://www.googlebot.com/bot.html)  
Disallow: /
User-agent: MegaSheep v1.0 (www.searchuk.com internet sheep)  
Disallow: /
User-agent: Megite2.0 (http://www.megite.com)  
Disallow: /
User-agent: Mercator-1.x  
Disallow: /
User-agent: Mercator-2.0  
Disallow: /
User-agent: Mercator-Scrub-1.1  
Disallow: /
User-agent: MetaGer-LinkChecker  
Disallow: /
User-agent: MetaGer_PreChecker0.1  
Disallow: /
User-agent: MetaProducts Download Express/1.x  
Disallow: /
User-agent: Metaeuro Web Crawler/0.2 (MetaEuro Web Search Clustering Engine; http://www.metaeuro.com; crawler at metaeuro dot com)  
Disallow: /
User-agent: MetagerBot/0.8-dev (MetagerBot; http://metager.de; )  
Disallow: /
User-agent: Metaspinner/0.01 (Metaspinner; http://www.meta-spinner.de/; support@meta-spinner.de/)  
Disallow: /
User-agent: MicroBaz  
Disallow: /
User-agent: Microsoft Data Access Internet Publishing Provider Cache Manager  
Disallow: /
User-agent: Microsoft Data Access Internet Publishing Provider DAV  
Disallow: /
User-agent: Microsoft Data Access Internet Publishing Provider Protocol Discovery  
Disallow: /
User-agent: Microsoft Log Parser 2.2  
Disallow: /
User-agent: Microsoft Small Business Indexer  
Disallow: /
User-agent: Microsoft URL Control - 6.00.8xxx  
Disallow: /
User-agent: MicrosoftPrototypeCrawler (How's my crawling? mailto:newbiecrawler@hotmail.com)  
Disallow: /
User-agent: Microsoft_Internet_Explorer_5.00.438 (fjones@isd.net)  
Disallow: /
User-agent: Mindjet MindManager  
Disallow: /
User-agent: MiracleAlphaTest  
Disallow: /
User-agent: Missauga Locate 1.0.0  
Disallow: /
User-agent: Missigua Locator 1.9  
Disallow: /
User-agent: Missouri College Browse  
Disallow: /
User-agent: Mister PiX version.dll  
Disallow: /
User-agent: Mister Pix II 2.02a  
Disallow: /
User-agent: Misterbot-Nutch/0.7.1 (Misterbot-Nutch; http://www.misterbot.fr; admin@misterbot.fr)  
Disallow: /
User-agent: Miva (AlgoFeedback@miva.com)  
Disallow: /
User-agent: Mizzu Labs 2.2  
Disallow: /
User-agent: MnogoSearch/3.2.xx  
Disallow: /
User-agent: Mo College 1.9  
Disallow: /
User-agent: MojeekBot/0.x (archi; http://www.mojeek.com/bot.html)  
Disallow: /
User-agent: MoonBrowser (version 0.41 Beta4)  
Disallow: /
User-agent: Moreoverbot/x.00 (+http://www.moreover.com)  
Disallow: /
User-agent: Morris - Mixcat Crawler ( http://mixcat.com)  
Disallow: /
User-agent: Motoricerca-Robots.txt-Checker/1.0 (http://tool.motoricerca.info/robots-checker.phtml)  
Disallow: /
User-agent: Motorola-V3m Obigo  
Disallow: /
User-agent: Mouse-House/7.4 (spider_monkey spider info at www.mobrien.com/sm.shtml)  
Disallow: /
User-agent: MovableType/x.x  
Disallow: /
User-agent: Mozi!  
Disallow: /
User-agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)  
Disallow: /
User-agent: Mulder, VCR-1.0  
Disallow: /
User-agent: MultiText/0.1  
Disallow: /
User-agent: MusicWalker2.0 ( http://www.somusical.com)  
Disallow: /
User-agent: My WinHTTP Connection  
Disallow: /
User-agent: MyGetRight/1.0.0  
Disallow: /
User-agent: MyGetRight/1.0b  
Disallow: /
User-agent: Mylinea.com Crawler 2.0  
Disallow: /
User-agent: NABOT/5.0  
Disallow: /
User-agent: NASA Search 1.0  
Disallow: /
User-agent: NCSA Beta 1 (http://vias.ncsa.uiuc.edu/viasarchivinginformation.html)  
Disallow: /
User-agent: NEC Research Agent -- compuman at research.nj.nec.com  
Disallow: /
User-agent: NEC-Hayek/1.0  
Disallow: /
User-agent: NETCOMplete/x.xx  
Disallow: /
User-agent: NG-Search/0.90 (NG-SearchBot; http://www.ng-search.com; )  
Disallow: /
User-agent: NG/1.0  
Disallow: /
User-agent: NG/4.0.1229  
Disallow: /
User-agent: NICO/1.0  
Disallow: /
User-agent: NITLE Blog Spider/0.01  
Disallow: /
User-agent: NP/0.1 (NP; http://www.nameprotect.com; npbot@nameprotect.com)  
Disallow: /
User-agent: NPBot (http://www.nameprotect.com/botinfo.html)  
Disallow: /
User-agent: NPBot-1/2.0  
Disallow: /
User-agent: NSPlayer/10.0.0.xxxx WMFSDK/10.0  
Disallow: /
User-agent: Naamah 1.0.1/Blogbot (http://blogbot.de/)  
Disallow: /
User-agent: Naamah 1.0a/Blogbot (http://blogbot.de/)  
Disallow: /
User-agent: NameOfAgent (CMS Spider)  
Disallow: /
User-agent: NationalDirectory-WebSpider/1.3  
Disallow: /
User-agent: NationalDirectoryAddURL/1.0  
Disallow: /
User-agent: NaverBot-1.0 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)  
Disallow: /
User-agent: NaverBot_dloader/1.5  
Disallow: /
User-agent: NavissoBot  
Disallow: /
User-agent: NavissoBot/1.7 (+http://navisso.com/)  
Disallow: /
User-agent: Nebullabot/2.2 (http://bot.nebulla.info)  
Disallow: /
User-agent: NetAnts/1.2x  
Disallow: /
User-agent: NetLookout/2.24  
Disallow: /
User-agent: NetMechanic Vx.0  
Disallow: /
User-agent: NetNewsWire/2.x (Mac OS X; http://ranchero.com/netnewswire/)  
Disallow: /
User-agent: NetNoseCrawler/v1.0  
Disallow: /
User-agent: NetPumper/x.xx  
Disallow: /
User-agent: NetResearchServer(http://www.look.com)  
Disallow: /
User-agent: NetResearchServer/x.x(loopimprovements.com/robot.html)  
Disallow: /
User-agent: NetSprint -- 2.0  
Disallow: /
User-agent: NetWhatCrawler/0.06-dev (NetWhatCrawler from NetWhat.com; http://www.netwhat.com; support@netwhat.com)  
Disallow: /
User-agent: NetZippy  
Disallow: /
User-agent: NetinfoBot/1.0 (http://netinfo.bg/netinfobot.html)  
Disallow: /
User-agent: Netluchs/0.8-dev ( ; http://www.netluchs.de/; ___don't___spam_me_@netluchs.de)  
Disallow: /
User-agent: Netprospector JavaCrawler  
Disallow: /
User-agent: NeuralBot/0.2  
Disallow: /
User-agent: NewsGator FetchLinks extension/0.2.0 (http://graemef.com)  
Disallow: /
User-agent: NewsGatorOnline/2.0 (http://www.newsgator.com; 1 subscribers)  
Disallow: /
User-agent: NextGenSearchBot 1 (for information visit http://www.eliyon.com/NextGenSearchBot)  
Disallow: /
User-agent: NextopiaBOT (+http://www.nextopia.com) distributed crawler client beta v0.x  
Disallow: /
User-agent: Nikita the Spider (http://NikitaTheSpider.com/)  
Disallow: /
User-agent: Nitro Downloader 1.x (www.klsofttools.com)  
Disallow: /
User-agent: Noago Spider  
Disallow: /
User-agent: Nocilla/1.0  
Disallow: /
User-agent: Nokia-WAPToolkit/1.2 googlebot(at)googlebot.com  
Disallow: /
User-agent: Nokia6610/1.0 (3.09) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible;YahooSeeker/M1A1-R2D2; http://help.yahoo.com/help/us/ysearch/crawling/crawling-01.html)  
Disallow: /
User-agent: Nokia7110/1.0 (05.01) (Google WAP Proxy/1.0)  
Disallow: /
User-agent: NokodoBot/1.x (+http://nokodo.com/bot.htm)  
Disallow: /
User-agent: Norbert the Spider(Burf.com)  
Disallow: /
User-agent: Nsauditor/1.x  
Disallow: /
User-agent: NuSearch Spider (compatible; MSIE 6.0)  
Disallow: /
User-agent: NuSearch Spider www.nusearch.com  
Disallow: /
User-agent: Nucleus SiteList LinkChecker/1.1  
Disallow: /
User-agent: Nutch  
Disallow: /
User-agent: Nutch crawler/Nutch-0.9 (picapage.com; admin@picapage.com)  
Disallow: /
User-agent: Nutch/Nutch-0.9 (Eurobot; http://www.ayell.eu )  
Disallow: /
User-agent: NutchCVS/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)  
Disallow: /
User-agent: NutchCVS/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)  
Disallow: /
User-agent: NutchCVS/0.7.1 (Nutch running at UW; http://www.nutch.org/docs/en/bot.html; sycrawl@cs.washington.edu)  
Disallow: /
User-agent: NutchEC2Test/Nutch-0.9-dev (Testing Nutch on Amazon EC2.; http://lucene.apache.org/nutch/bot.html; ec2test at lucene.com)  
Disallow: /
User-agent: NutchOrg/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)  
Disallow: /
User-agent: NutchVinegarCrawl/Nutch-0.8.1 (Vinegar; http://www.cs.washington.edu; eytanadar at gmail dot com)  
Disallow: /
User-agent: OPWV-SDK UP.Browser/7.0.2.3.119 (GUI) MMP/2.0 Push/PO  
Disallow: /
User-agent: OSSProxy 1.3.305.321 (Build 305.321 Win32 en-us)(Dec 21 2005 16:30:54)  
Disallow: /
User-agent: OWR_Crawler 0.1  
Disallow: /
User-agent: ObjectsSearch/0.01-dev (ObjectsSearch;http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)  
Disallow: /
User-agent: ObjectsSearch/0.0x (ObjectsSearch; http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)  
Disallow: /
User-agent: Ocelli/1.x (http://www.globalspec.com/Ocelli)  
Disallow: /
User-agent: Octopus  
Disallow: /
User-agent: Octora Beta - www.octora.com  
Disallow: /
User-agent: Octora Beta Bot - www.octora.com  
Disallow: /
User-agent: Offline Explorer 1.*  
Disallow: /
User-agent: OliverPerry  
Disallow: /
User-agent: OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Internet CategorizerOmniExplorer http://www.omni-explorer.com/ car & shopping search (64.62.175.xxx)  
Disallow: /
User-agent: OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Job Crawler  
Disallow: /
User-agent: OmniExplorer_Bot/1.1x (+http://www.omni-explorer.com) Torrent Crawler  
Disallow: /
User-agent: OmniExplorer_Bot/x.xx (+http://www.omni-explorer.com) WorldIndexer  
Disallow: /
User-agent: Onet.pl SA- http://szukaj.onet.pl  
Disallow: /
User-agent: Online24-Bot (Version: 1.0x, powered by www.online24.de)  
Disallow: /
User-agent: OntoSpider/1.0 libwww-perl/5.65  
Disallow: /
User-agent: OpenAcoon v4.0.x (www.openacoon.de)  
Disallow: /
User-agent: OpenISearch/1.x (www.openisearch.com)  
Disallow: /
User-agent: OpenTaggerBot (http://www.opentagger.com/opentaggerbot.htm)  
Disallow: /
User-agent: OpenTextSiteCrawler/2.9.2  
Disallow: /
User-agent: OpenWebSpider/0.x.x (http://www.openwebspider.org)  
Disallow: /
User-agent: OpenWebSpider/x  
Disallow: /
User-agent: Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)  
Disallow: /
User-agent: Openfind Robot/1.1A2  
Disallow: /
User-agent: Openfind data gatherer- Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)  
Disallow: /
User-agent: Opera/5.0 (Linux 2.0.38 i386; U) [en]  
Disallow: /
User-agent: Opera/5.11 (Windows ME; U) [ru]  
Disallow: /
User-agent: Opera/5.12 (Windows 98; U) [en]  
Disallow: /
User-agent: Opera/6.01 (larbin@unspecified.mail)  
Disallow: /
User-agent: Opera/6.x (Linux 2.4.8-26mdk i686; U) [en]  
Disallow: /
User-agent: Opera/6.x (Windows NT 4.0; U) [de]  
Disallow: /
User-agent: Opera/7.x (Windows NT 5.1; U) [en]  
Disallow: /
User-agent: Opera/8.xx (Windows NT 5.1; U; en)  
Disallow: /
User-agent: Opera/9.0 (Windows NT 5.1; U; en)  
Disallow: /
User-agent: Opera/9.00 (Windows NT 5.1; U; de)  
Disallow: /
User-agent: OpidooBOT (larbin2.6.3@unspecified.mail)  
Disallow: /
User-agent: Oracle Application Server Web Cache 10g  
Disallow: /
User-agent: Oracle Ultra Search  
Disallow: /
User-agent: Oracle iMTCrawler  
Disallow: /
User-agent: OrangeSpider  
Disallow: /
User-agent: Orbiter/T-2.0 (+http://www.dailyorbit.com/bot.htm)  
Disallow: /
User-agent: Orca Browser (http://www.orcabrowser.com)  
Disallow: /
User-agent: OutfoxBot/0.x (For internet experiments; http://; outfox.agent@gmail.com)  
Disallow: /
User-agent: OutfoxMelonBot/0.5 (for internet experiments; http://; outfoxbot@gmail.com)  
Disallow: /
User-agent: Overture-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)  
Disallow: /
User-agent: PADLibrary Spider  
Disallow: /
User-agent: PBrowse 1.4b  
Disallow: /
User-agent: PEAR HTTP_Request class ( http://pear.php.net/ )  
Disallow: /
User-agent: PEERbot www.peerbot.com  
Disallow: /
User-agent: PEval 1.4b  
Disallow: /
User-agent: PHP/3.x.xx  
Disallow: /
User-agent: PHP/4.0.4pl1  
Disallow: /
User-agent: PHP/4.0.6  
Disallow: /
User-agent: PHP/4.1.1  
Disallow: /
User-agent: PHP/4.1.2  
Disallow: /
User-agent: PJspider/3.0 (pjspider@portaljuice.com; http://www.portaljuice.com)  
Disallow: /
User-agent: POE-Component-Client-HTTP/0.64 (perl; N; POE; en; rv:0.640000)  
Disallow: /
User-agent: PRCrawler/Nutch-0.9 (data mining development project; crawler@projectrialto.com)  
Disallow: /
User-agent: PROve AnswerBot 4.0  
Disallow: /
User-agent: PSurf15a 11  
Disallow: /
User-agent: PSurf15a 51  
Disallow: /
User-agent: PSurf15a VA  
Disallow: /
User-agent: PWeBot/1.2 Inspector (http://www.programacionweb.net/robot.php)  
Disallow: /
User-agent: PageBitesHyperBot/600 (http://www.pagebites.com/)  
Disallow: /
User-agent: Pagebull http://www.pagebull.com/  
Disallow: /
User-agent: Pagestacker Bot  
Disallow: /
User-agent: PagmIEDownload  
Disallow: /
User-agent: ParaSite/1.0b (http://www.ianett.com/parasite/)  
Disallow: /
User-agent: Patwebbot (http://www.herz-power.de/technik.html)  
Disallow: /
User-agent: PeopleChat/Search_Engine  
Disallow: /
User-agent: PicoSearch/1.0  
Disallow: /
User-agent: Piffany_Web_Scraper_v0.x  
Disallow: /
User-agent: Piffany_Web_Spider_v0.x  
Disallow: /
User-agent: PigeonBot1.0 BETA  
Disallow: /
User-agent: PingALink Monitoring Services 1.0  
Disallow: /
User-agent: PingALink Monitoring Services 1.0 (http://www.pingalink.com)  
Disallow: /
User-agent: Pingdom GIGRIB (http://www.pingdom.com)  
Disallow: /
User-agent: Pita  
Disallow: /
User-agent: Pizilla++ ver 2.45  
Disallow: /
User-agent: Plagger/0.x.xx (http://plagger.org/)  
Disallow: /
User-agent: PlagiarBot/1.0  
Disallow: /
User-agent: PlantyNet_WebRobot_V1.9 dhkang@plantynet.com  
Disallow: /
User-agent: PluckFeedCrawler/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://www.pluck.com; 1 subscribers)  
Disallow: /
User-agent: Pluggd/Nutch-0.9 (automated crawler http://www.pluggd.com;support at pluggd dot com)  
Disallow: /
User-agent: Pockey-GetHTML/4.12.0 (Win32; GUI; ix86)  
Disallow: /
User-agent: Pockey-GetHTML/x.xx  
Disallow: /
User-agent: Pockey/x.x.x  
Disallow: /
User-agent: Pockey7.x.x(WIN32GUI)  
Disallow: /
User-agent: Poirot  
Disallow: /
User-agent: Pompos/1.x http://dir.com/pompos.html  
Disallow: /
User-agent: Pompos/1.x pompos@iliad.fr  
Disallow: /
User-agent: Popdexter/1.0  
Disallow: /
User-agent: Port Huron Labs  
Disallow: /
User-agent: PortalBSpider/2.0 (spider@portalb.com)  
Disallow: /
User-agent: PostFavorites  
Disallow: /
User-agent: PrivacyFinder Cache Bot v1.0  
Disallow: /
User-agent: PrivacyFinder/1.1  
Disallow: /
User-agent: Privoxy/3.0 (Anonymous)  
Disallow: /
User-agent: ProWebGuide Link Checker (http://www.prowebguide.com)  
Disallow: /
User-agent: Production Bot 0116B  
Disallow: /
User-agent: Production Bot 2016B  
Disallow: /
User-agent: Production Bot DOT 3016B  
Disallow: /
User-agent: Program Shareware 1.0.2  
Disallow: /
User-agent: Progressive Download  
Disallow: /
User-agent: Progressive Download HTTP check  
Disallow: /
User-agent: Project XP5 [2.03.07-111203]  
Disallow: /
User-agent: PubCrawl (pubcrawl.stanford.edu)  
Disallow: /
User-agent: PureSight  
Disallow: /
User-agent: PuxaRapido v1.0  
Disallow: /
User-agent: PycURL  
Disallow: /
User-agent: PycURL/7.xx.x  
Disallow: /
User-agent: Python-urllib/1.1x  
Disallow: /
User-agent: Python-urllib/2.0a1  
Disallow: /
User-agent: QEAVis Agent/Nutch-0.9 (Quantitative Evaluation of Academic Websites Visibility; http://nlp.uned.es/qeavis  
Disallow: /
User-agent: QPCreep Test Rig ( We are not indexing- just testing )  
Disallow: /
User-agent: Qango.com Web Directory (http://www.qango.com/)  
Disallow: /
User-agent: QuepasaCreep ( crawler@quepasacorp.com )  
Disallow: /
User-agent: QuepasaCreep v0.9.1x  
Disallow: /
User-agent: QueryN Metasearch  
Disallow: /
User-agent: QuickTime\xaa.7.0.4 (qtver=7.0.4;cpu=PPC;os=Mac 10.3.9)  
Disallow: /
User-agent: Quicksilver (Blacktree,MacOSX)  
Disallow: /
User-agent: QweeryBot/3.01 ( http://qweerybot.qweery.nl)  
Disallow: /
User-agent: Qweery_robot.txt_CheckBot/3.01 (http://qweerybot.qweery.com)  
Disallow: /
User-agent: R6_CommentReader_(www.radian6.com/crawler)  
Disallow: /
User-agent: R6_FeedFetcher_(www.radian6.com/crawler)  
Disallow: /
User-agent: RAMPyBot - www.giveRAMP.com/0.1 (RAMPyBot - www.giveRAMP.com; http://www.giveramp.com/bot.html; support@giveRAMP.com)  
Disallow: /
User-agent: RAMPyBot/0.8-dev (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)  
Disallow: /
User-agent: REAP-crawler Nutch/Nutch-1.0-dev (Reap Project; http://reap.cs.cmu.edu/REAP-crawler/; Reap Project)  
Disallow: /
User-agent: REBOL Core 2.x.x.x.x  
Disallow: /
User-agent: REBOL View 1.x.x.x.x  
Disallow: /
User-agent: REL Link Checker Lite x.x  
Disallow: /
User-agent: RMA/1.0 (compatible; RealMedia)  
Disallow: /
User-agent: RPT-HTTPClient/0.3-x  
Disallow: /
User-agent: RRC (crawler_admin@bigfoot.com)  
Disallow: /
User-agent: RSSMicro.com RSS/Atom Feed Robot  
Disallow: /
User-agent: RSSOwl/1.2.3 2006-11-26 (Windows; U; zhtw)  
Disallow: /
User-agent: RSSOwl/1.2.4 Preview Release 2007-04-15 (Windows; U; zhtw)  
Disallow: /
User-agent: RSurf15a 41  
Disallow: /
User-agent: RSurf15a 51  
Disallow: /
User-agent: RSurf15a 81  
Disallow: /
User-agent: RX Bar  
Disallow: /
User-agent: RaBot/1.0 Agent-admin/phortse@hanmail.net  
Disallow: /
User-agent: Rainbot1.1  
Disallow: /
User-agent: Rank Exec (rankexec.com) Reciprocal Link Manager 1.x/bot  
Disallow: /
User-agent: Rankivabot/3.2 (www.rankiva.com; 3.2; vzmxikn)  
Disallow: /
User-agent: Rational SiteCheck (Windows NT)  
Disallow: /
User-agent: ReadABlog Spider (compatible; 1.1; feed update; www.readablog.com)  
Disallow: /
User-agent: RealDownload/4.0.0.4x  
Disallow: /
User-agent: Reaper [2.03.10-031204] (http://www.sitesearch.ca/reaper/)  
Disallow: /
User-agent: Reaper/2.0x (+http://www.sitesearch.ca/reaper)  
Disallow: /
User-agent: RebusnetBot (+http://www.rebusnet.biz)  
Disallow: /
User-agent: RebusnetPADBot/1.5x (+http://www.rebusnet.biz)  
Disallow: /
User-agent: RedBot/redbot-1.0 (Rediff.com Crawler; redbot at rediff dot com)  
Disallow: /
User-agent: RedCarpet/1.2 (http://www.redcarpet-inc.com/robots.html)  
Disallow: /
User-agent: RedCell/0.1 (InfoSec Search Bot (Coming Soon); http://www.telegenetic.net/bot.html; lhall@telegenetic.net)  
Disallow: /
User-agent: RedCell/0.1 (RedCell; telegenetic.net/bot.html; lhall_at_telegenetic.net)  
Disallow: /
User-agent: RedKernel WWW-Spider 2/0 (+http://www-spider.redkernel-softwares.com/)  
Disallow: /
User-agent: RepoMonkey Bait & Tackle/v1.01  
Disallow: /
User-agent: Rewebber/1.2 libwww-perl/5.41  
Disallow: /
User-agent: RixBot (http://babelserver.org/rix)  
Disallow: /
User-agent: RoboCrawl (http://www.canadiancontent.net)  
Disallow: /
User-agent: RoboCrawl (www.canadiancontent.net)  
Disallow: /
User-agent: RoboPal (http://www.findpal.com/)  
Disallow: /
User-agent: Robot/www.pj-search.com  
Disallow: /
User-agent: Robot: NutchCrawler- Owner: wdavies@acm.org  
Disallow: /
User-agent: Robot@SuperSnooper.Com  
Disallow: /
User-agent: Robozilla/1.0  
Disallow: /
User-agent: Rome Client (http://tinyurl.com/64t5n) Ver: 0.9  
Disallow: /
User-agent: Rotondo/3.1 libwww/5.3.1  
Disallow: /
User-agent: RssBandit/1.5.0.10 (.NET CLR 1.1.4322.2407; WinNT 5.1.2600.0; http://www.rssbandit.org) (.NET CLR 1.1.4322.2407; WinNT 5.1.2600.0; )  
Disallow: /
User-agent: RssReader/1.0.xx.x (http://www.rssreader.com) Microsoft Windows NT 5.1.2600.0  
Disallow: /
User-agent: Rubbot/1.0 (+http://rubhub.com/)  
Disallow: /
User-agent: RufusBot (Rufus Web Miner; http://64.124.122.252/feedback.html)  
Disallow: /
User-agent: RufusBot (Rufus Web Miner; http://www.webaroo.com/rooSiteOwners.html)  
Disallow: /
User-agent: Rumours-Agent  
Disallow: /
User-agent: S&L Spider (http://search.hirners.com/)  
Disallow: /
User-agent: S.T.A.L.K.E.R. (http://www.seo-tools.net/en/bot.aspx)  
Disallow: /
User-agent: SBIder/0.7 (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)  
Disallow: /
User-agent: SBIder/0.8-dev (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)  
Disallow: /
User-agent: SBL-BOT (http://sbl.net)  
Disallow: /
User-agent: SQ Webscanner  
Disallow: /
User-agent: SSurf15a 11   
Disallow: /
User-agent: SURF   
Disallow: /
User-agent: SWB/V1.4 (HP)  
Disallow: /
User-agent: SWSBot-Images/1.2 http://www.smartwaresoft.com/swsbot12.html  
Disallow: /
User-agent: SafariBookmarkChecker (+http://www.coriolis.ch/)  
Disallow: /
User-agent: SandCrawler - Compatibility Testing  
Disallow: /
User-agent: ScanWeb  
Disallow: /
User-agent: ScholarUniverse/0.8 (Nutch;+http://scholaruniverse.com/bot.jsp; fetch-agent@scholaruniverse.com)  
Disallow: /
User-agent: Science Traveller International 1X/1.0  
Disallow: /
User-agent: ScollSpider/2.0 (+http://www.webwobot.com/ScollSpider.php)  
Disallow: /
User-agent: Scope (Mars+)  
Disallow: /
User-agent: ScoutAbout  
Disallow: /
User-agent: ScoutAnt/0.1; +http://www.ant.com/what_is_ant.com/  
Disallow: /
User-agent: Scrubby/2.x (http://www.scrubtheweb.com/)  
Disallow: /
User-agent: Scrubby/3.0 (+http://www.scrubtheweb.com/help/technology.html)  
Disallow: /
User-agent: Search+  
Disallow: /
User-agent: Search-Engine-Studio  
Disallow: /
User-agent: Search/1.0 (http://www.innerprise.net/es-spider.asp)  
Disallow: /
User-agent: SearchByUsa/2 (SearchByUsa; http://www.SearchByUsa.com/bot.html; info@SearchByUsa.com)  
Disallow: /
User-agent: SearchExpress Spider0.99  
Disallow: /
User-agent: SearchGuild/DMOZ/Experiment (searchguild@gmail.com)  
Disallow: /
User-agent: SearchGuild_DMOZ_Experiment (chris@searchguild.com)  
Disallow: /
User-agent: SearchSight/2.0 (http://SearchSight.com/)  
Disallow: /
User-agent: SearchSpider.com/1.1  
Disallow: /
User-agent: SearchTone2.0 - IDEARE  
Disallow: /
User-agent: SearchdayBot  
Disallow: /
User-agent: Searchit-Now Robot/2.2 (+http://www.searchit-now.co.uk)  
Disallow: /
User-agent: Searchmee! Spider v0.98a  
Disallow: /
User-agent: Searchspider/1.2 (SearchSpider; http://www.searchspider.com; webmaster@searchspider.com)  
Disallow: /
User-agent: Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/0.3  
Disallow: /
User-agent: Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.0 (XDF)  
Disallow: /
User-agent: Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2  
Disallow: /
User-agent: Seeker.lookseek.com  
Disallow: /
User-agent: Semager/1.1 (http://www.semager.de/blog/semager-bots/)  
Disallow: /
User-agent: Semager/1.x (http://www.semager.de)  
Disallow: /
User-agent: Sensis Web Crawler (search_comments\at\sensis\dot\com\dot\au)  
Disallow: /
User-agent: Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au)  
Disallow: /
User-agent: SeznamBot/1.0  
Disallow: /
User-agent: SeznamBot/1.0 (+http://fulltext.seznam.cz/)  
Disallow: /
User-agent: SeznamBot/2.0-test (+http://fulltext.sblog.cz/)  
Disallow: /
User-agent: ShablastBot 1.0  
Disallow: /
User-agent: Shareaza v1.x.x.xx  
Disallow: /
User-agent: SharewarePlazaFileCheckBot/1.0+(+http://www.SharewarePlaza.com)  
Disallow: /
User-agent: Shim Crawler  
Disallow: /
User-agent: Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)  
Disallow: /
User-agent: ShopWiki/1.0 ( +http://www.shopwiki.com/)  
Disallow: /
User-agent: ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)  
Disallow: /
User-agent: Shoula.com Crawler 2.0  
Disallow: /
User-agent: SietsCrawler/1.1 (+http://www.siets.biz)  
Disallow: /
User-agent: Sigram/Nutch-1.0-dev (Test agent for Nutch development; http://www.sigram.com/bot.html; bot at sigram dot com)  
Disallow: /
User-agent: Siigle Orumcex v.001 Turkey (http://www.siigle.com)  
Disallow: /
User-agent: SimpleFavPanel/1.2  
Disallow: /
User-agent: Simpy 1.x; http://www.simpy.com/  
Disallow: /
User-agent: Simpy/1.x (Simpy; http://www.simpy.com/?ref=bot; feedback at simpy dot com)  
Disallow: /
User-agent: Sirketcebot/v.01 (http://www.sirketce.com/bot.html)  
Disallow: /
User-agent: SiteBar/3.x.x (Bookmark Server; http://sitebar.org/)  
Disallow: /
User-agent: SiteBar/x.x  
Disallow: /
User-agent: SiteBar/x.x.x (Bookmark Server; http://sitebar.org/)  
Disallow: /
User-agent: SiteRecon+(xx)  
Disallow: /
User-agent: SiteSnagger  
Disallow: /
User-agent: SiteSpider +(http://www.SiteSpider.com/)  
Disallow: /
User-agent: SiteSucker/1.x.x  
Disallow: /
User-agent: SiteTaggerBot (http://www.sitetagger.com/bot.htm)  
Disallow: /
User-agent: SiteTruth.com site rating system  
Disallow: /
User-agent: SiteWinder  
Disallow: /
User-agent: SiteXpert  
Disallow: /
User-agent: Skampy/0.9.x (http://www.skaffe.com/skampy-info.html)  
Disallow: /
User-agent: Skimpy/0.x (http://www.skaffe.com/skampy-info.html)  
Disallow: /
User-agent: Skywalker/0.1 (Skywalker; anonymous; anonymous)  
Disallow: /
User-agent: Slarp/0.1  
Disallow: /
User-agent: Sleipnir  
Disallow: /
User-agent: Sleipnir Version 1.xx  
Disallow: /
User-agent: Sleipnir Version2.x  
Disallow: /
User-agent: Sleipnir/2.xx  
Disallow: /
User-agent: Slider_Search_v1-de  
Disallow: /
User-agent: SlimBrowser  
Disallow: /
User-agent: Slurp/2.0 (slurp@inktomi.com; http://www.inktomi.com/slurp.html)  
Disallow: /
User-agent: Slurp/2.0-KiteWeekly (slurp@inktomi.com; http://www.inktomi.com/slurp.html)  
Disallow: /
User-agent: Slurp/si (slurp@inktomi.com; http://www.inktomi.com/slurp.html)  
Disallow: /
User-agent: Slurpy Verifier/1.0  
Disallow: /
User-agent: SlySearch (slysearch@slysearch.com)  
Disallow: /
User-agent: SlySearch/1.0 http://www.plagiarism.org/crawler/robotinfo.html  
Disallow: /
User-agent: SlySearch/1.x http://www.slysearch.com  
Disallow: /
User-agent: SmartDownload/1.2.67 (Win32; Jan 12 1999)  
Disallow: /
User-agent: SmartDownload/1.2.77 (Win32; Feb 1 2000)  
Disallow: /
User-agent: SmartDownload/1.2.77 (Win32; Jun 19 2001)  
Disallow: /
User-agent: SmiffyDCMetaSpider/1.0  
Disallow: /
User-agent: Snapbot/1.0  
Disallow: /
User-agent: Snapbot/1.0 (Snap Shots, +http://www.snap.com)  
Disallow: /
User-agent: Snappy/1.1 ( http://www.urltrends.com/ )  
Disallow: /
User-agent: Snarfer/0.x.x (http://www.snarfware.com/)  
Disallow: /
User-agent: SnoopRob/x.x  
Disallow: /
User-agent: Snoopy v1.xx  
Disallow: /
User-agent: Snoopy v1.xx- : User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2)  
Disallow: /
User-agent: Snoopy_v0.xx  
Disallow: /
User-agent: SnykeBot/0.6 (http://www.snyke.com)  
Disallow: /
User-agent: SocSciBot ()  
Disallow: /
User-agent: SoftBank/1.0/812SH/SHJ001 Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1  
Disallow: /
User-agent: SoftHypermarketFileCheckBot/1.0+(+http://www.softhypermaket.com)  
Disallow: /
User-agent: Softizerbot (http://www.softizer.com)  
Disallow: /
User-agent: Sogou Orion spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)  
Disallow: /
User-agent: Sogou web spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)  
Disallow: /
User-agent: Sosospider+(+http://help.soso.com/webspider.htm)  
Disallow: /
User-agent: Space Bison/0.02 [fu] (Win67; X; SK)  
Disallow: /
User-agent: SpeedDownload/1.x  
Disallow: /
User-agent: Speedy Spider (Beta/x.x; speedy@entireweb.com)  
Disallow: /
User-agent: Speedy Spider (Entireweb; Beta/1.0; http://www.entireweb.com/about/search_tech/speedyspider/)  
Disallow: /
User-agent: Speedy_Spider (http://www.entireweb.com)  
Disallow: /
User-agent: Sphere Scout&v4.0 - scout at sphere dot com  
Disallow: /
User-agent: Sphider  
Disallow: /
User-agent: Spida/0.1  
Disallow: /
User-agent: Spider-Sleek/2.0 (+http://search-info.com/linktous.html)  
Disallow: /
User-agent: Spider.TerraNautic.net - v:1.04  
Disallow: /
User-agent: Spider/maxbot.com admin@maxbot.com  
Disallow: /
User-agent: SpiderKU/0.x  
Disallow: /
User-agent: SpiderMan  
Disallow: /
User-agent: SpiderMonkey/7.0x (SpiderMonkey.ca info at http://spidermonkey.ca/sm.shtml)  
Disallow: /
User-agent: Spinne/2.0  
Disallow: /
User-agent: Spinne/2.0 med  
Disallow: /
User-agent: Spinne/2.0 med_AH  
Disallow: /
User-agent: Spock Crawler (http://www.spock.com/crawler)  
Disallow: /
User-agent: Squid-Prefetch  
Disallow: /
User-agent: SquidClamAV_Redirector 1.x.x  
Disallow: /
User-agent: Sqworm/2.9.81-BETA (beta_release; 20011102-760; i686-pc-linux-gnu)  
Disallow: /
User-agent: Sqworm/2.9.85-BETA (beta_release; 20011115-775; i686-pc-linux-gnu)  
Disallow: /
User-agent: Sqworm/2.9.89-BETA (beta_release; 20020130-839; i686-pc-linux-gnu)   
Disallow: /
User-agent: StackRambler/x.x   
Disallow: /
User-agent: Stamina/1.4  
Disallow: /
User-agent: Star Downloader  
Disallow: /
User-agent: StarDownloader/1.xx  
Disallow: /
User-agent: Steeler/1.x (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)  
Disallow: /
User-agent: Steeler/3.3 (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)  
Disallow: /
User-agent: Strategic Board Bot (+http://www.strategicboard.com)  
Disallow: /
User-agent: Submission Spider at surfsafely.com  
Disallow: /
User-agent: Suchknecht.at-Robot  
Disallow: /
User-agent: Sunrise XP/2.x  
Disallow: /
User-agent: Sunrise/0.42g (Windows XP)  
Disallow: /
User-agent: SuperBot/x.x (Win32)  
Disallow: /
User-agent: SuperBot/x.x.x.xx (Windows XP)  
Disallow: /
User-agent: Superdownloads Spiderman  
Disallow: /
User-agent: SurfMaster  
Disallow: /
User-agent: SurferF3 1/0  
Disallow: /
User-agent: SurveyBot/2.2 <a href='http://www.whois.sc'>Whois Source</a>  
Disallow: /
User-agent: SurveyBot/2.3 (Whois Source)  
Disallow: /
User-agent: Swooglebot/2.0. (+http://swoogle.umbc.edu/swooglebot.htm)  
Disallow: /
User-agent: SygolBot http://www.sygol.net  
Disallow: /
User-agent: Sylera/1.2.x  
Disallow: /
User-agent: SyncBot  
Disallow: /
User-agent: SyncIT/x.x  
Disallow: /
User-agent: Syndirella/0.91pre  
Disallow: /
User-agent: SynoBot  
Disallow: /
User-agent: Syntryx ANT Scout Chassis Pheromone; Mozilla/4.0 compatible crawler  
Disallow: /
User-agent: Szukacz/1.x  
Disallow: /
User-agent: Szukacz/1.x (robot; www.szukacz.pl/jakdzialarobot.html; szukacz@proszynski.pl)  
Disallow: /
User-agent: T-Online Browser  
Disallow: /
User-agent: TAMU_CS_IRL_CRAWLER/1.0  
Disallow: /
User-agent: TCDBOT/Nutch-0.8 (PhD student research;"http://www.tcd.ie; mcgettrs at t c d dot IE)"  
Disallow: /
User-agent: TE  
Disallow: /
User-agent: TECOMAC-Crawler/0.x  
Disallow: /
User-agent: TJG/Spider  
Disallow: /
User-agent: TJvMultiHttpGrabber Component  
Disallow: /
User-agent: TOPOS robot/1.1 (http://www.topos.com.ua/)  
Disallow: /
User-agent: TSurf15a 11  
Disallow: /
User-agent: Tagword (http://tagword.com/dmoz_survey.php)  
Disallow: /
User-agent: Tagyu Agent/1.0  
Disallow: /
User-agent: Talkro Web-Shot/1.0 (E-mail: webshot@daumsoft.com- Home: http://222.122.15.190/webshot)  
Disallow: /
User-agent: TargetYourNews.com bot  
Disallow: /
User-agent: TeamSoft WinInet Component  
Disallow: /
User-agent: Tecomi Bot (http://www.tecomi.com/bot.htm)  
Disallow: /
User-agent: Teemer (NetSeer, Inc. is a Los Angeles based Internet startup company.; http://www.netseer.com/crawler.html; crawler@netseer.com)  
Disallow: /
User-agent: Teleport Pro/1.2x(.1xxx)  
Disallow: /
User-agent: Teoma MP  
Disallow: /
User-agent: Teradex Mapper; mapper@teradex.com; http://www.teradex.com  
Disallow: /
User-agent: TeragramCrawler  
Disallow: /
User-agent: TerrawizBot/1.0 (+http://www.terrawiz.com/bot.html)  
Disallow: /
User-agent: Test spider  
Disallow: /
User-agent: TestCrawler/Nutch-0.9 (Testing Crawler for Research ; http://balihoo.com/index.aspx; tgautier at balihoo dot com)  
Disallow: /
User-agent: The Expert HTML Source Viewer (http://www.expert-html.com)  
Disallow: /
User-agent: TheRarestParser/0.2a (http://therarestwords.com/)  
Disallow: /
User-agent: TheSuBot/0.1 (www.thesubot.de)  
Disallow: /
User-agent: TimelyWeb/4.1 ( <a href="http://www.timelyweb.com/">EldoS TimelyWeb 4.1</a> )  
Disallow: /
User-agent: TinEye/1.1 (http://tineye.com/crawler.html)  
Disallow: /
User-agent: Tkensaku/x.x(http://www.tkensaku.com/q.html)  
Disallow: /
User-agent: Topodia/1.2-dev (Topodia - Crawler for HTTP content indexing; http://www.topodia.com/; support@topodia.com)  
Disallow: /
User-agent: Toutatis x-xx.x (hoppa.com)  
Disallow: /
User-agent: Toutatis x.x (hoppa.com)  
Disallow: /
User-agent: Toutatis x.x-x  
Disallow: /
User-agent: Trailfire-bot/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)  
Disallow: /
User-agent: Trailfire-bot/0.7.1 (Trailfire page content analyzer; http://trailfire.com; info@trailfire.com)  
Disallow: /
User-agent: Trailfire/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)  
Disallow: /
User-agent: Trampelpfad-Spider  
Disallow: /
User-agent: Trampelpfad-Spider-v0.1  
Disallow: /
User-agent: TulipChain/5.x (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://java.sun.com/) Linux/2.4.17  
Disallow: /
User-agent: TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8  
Disallow: /
User-agent: Tumblr/1.0 RSS syndication (+http://www.tumblr.com/) (support@tumblr.com)  
Disallow: /
User-agent: TurnitinBot/x.x (http://www.turnitin.com/robot/crawlerinfo.html)  
Disallow: /
User-agent: Turnpike Emporium LinkChecker/0.1  
Disallow: /
User-agent: TutorGig/1.5 (+http://www.tutorgig.com/crawler)  
Disallow: /
User-agent: Tutorial Crawler 1.4 (http://www.tutorgig.com/crawler)  
Disallow: /
User-agent: Twiceler www.cuill.com/robots.html  
Disallow: /
User-agent: Twiceler-0.9 http://www.cuill.com/twiceler/robot.html  
Disallow: /
User-agent: Twisted PageGetter  
Disallow: /
User-agent: Twitturly / v0.x  
Disallow: /
User-agent: Twotrees Reactive Filter V2.0  
Disallow: /
User-agent: Tycoon Agent/Nutch-1.0-dev  
Disallow: /
User-agent: TygoBot  
Disallow: /
User-agent: TygoProwler  
Disallow: /
User-agent: UCMore Crawler App  
Disallow: /
User-agent: UCWEB5.1  
Disallow: /
User-agent: UCmore  
Disallow: /
User-agent: UDM  
Disallow: /
User-agent: UIowaCrawler/1.0  
Disallow: /
User-agent: UKWizz/Nutch-0.8.1 (UKWizz Nutch crawler; http://www.ukwizz.com/)  
Disallow: /
User-agent: UP.Browser/3.01-IG01 UP.Link/3.2.3.4  
Disallow: /
User-agent: UPG1 UP/4.0 (compatible; Blazer 1.0)  
Disallow: /
User-agent: URI::Fetch/0.06  
Disallow: /
User-agent: URL Spider Pro/x.xx (innerprise.net)  
Disallow: /
User-agent: URLBase/6.x  
Disallow: /
User-agent: URLBlaze  
Disallow: /
User-agent: URLGetFile  
Disallow: /
User-agent: URL_Spider_Pro/x.x  
Disallow: /
User-agent: URL_Spider_Pro/x.x+(http://www.innerprise.net/usp-spider.asp)  
Disallow: /
User-agent: USyd-NLP-Spider (http://www.it.usyd.edu.au/~vinci/bot.html)  
Disallow: /
User-agent: UdmSearch/3.1.x  
Disallow: /
User-agent: Ultraseek  
Disallow: /
User-agent: Under the Rainbow 2.2  
Disallow: /
User-agent: UofTDB_experiment (leehyun@cs.toronto.edu)  
Disallow: /
User-agent: UptimeBot(www.uptimebot.com)  
Disallow: /
User-agent: Uptimebot  
Disallow: /
User-agent: User-Agent: BoardReader Favicon Fetcher /1.0 info@boardreader.com  
Disallow: /
User-agent: User-Agent: BoardReader Image Fetcher /1.0 info@boardreader.com  
Disallow: /
User-agent: User-Agent: FileHeap! file downloader (http://www.fileheap.com)  
Disallow: /
User-agent: User-Agent: LjSEEK Picture-Bot /1.0 contact@ljseek.com  
Disallow: /
User-agent: User-Agent: Mozilla/4.0 (SKIZZLE! Distributed Internet Spider v1.0 - www.SKIZZLE.com)  
Disallow: /
User-agent: User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)  
Disallow: /
User-agent: UtilMind HTTPGet  
Disallow: /
User-agent: Utopia WebWasher 3.0  
Disallow: /
User-agent: VLC media player - version 0.8.5 Janus - (c) 1996-2006 the VideoLAN team  
Disallow: /
User-agent: VMBot/0.x.x (VMBot; http://www.VerticalMatch.com/; vmbot@tradedot.com)  
Disallow: /
User-agent: VSE/1.0 (testcrawler@hotmail.com)  
Disallow: /
User-agent: VSE/1.0 (testcrawler@vivisimo.com)  
Disallow: /
User-agent: VWBOT/Nutch-0.9-dev (VWBOT Nutch Crawler; http://vwbot.cs.uiuc.edu;+vwbot@cs.uiuc.edu  
Disallow: /
User-agent: VadixBot  
Disallow: /
User-agent: Vagabondo-WAP/2.0 (webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)/1.0 Profile  
Disallow: /
User-agent: Vagabondo/1.x MT (webagent@wise-guys.nl)  
Disallow: /
User-agent: Vagabondo/2.0 MT  
Disallow: /
User-agent: Vagabondo/2.0 MT (webagent at wise-guys dot nl)  
Disallow: /
User-agent: Vagabondo/2.0 MT (webagent@NOSPAMwise-guys.nl)  
Disallow: /
User-agent: Vagabondo/3.0 (webagent at wise-guys dot nl)  
Disallow: /
User-agent: Vakes/0.01 (Vakes; http://www.vakes.com/; search@vakes.com)  
Disallow: /
User-agent: VayalaCreep-v0.0.1 (haploid@haploid.com)  
Disallow: /
User-agent: Vayala|Creep-v0.0.1 (codepoet@wildties.com)  
Disallow: /
User-agent: Verticrawlbot  
Disallow: /
User-agent: VeryGoodSearch.com.DaddyLongLegs  
Disallow: /
User-agent: Verzamelgids/2.2 (http://www.verzamelgids.nl)  
Disallow: /
User-agent: Vespa Crawler  
Disallow: /
User-agent: VisBot/2.0 (Visvo.com Crawler; http://www.visvo.com/bot.html; bot@visvo.com)  
Disallow: /
User-agent: Visicom Toolbar  
Disallow: /
User-agent: Vision Research Lab image spider at vision.ece.ucsb.edu  
Disallow: /
User-agent: Vortex/2.2 (+http://marty.anstey.ca/robots/vortex/)  
Disallow: /
User-agent: W3C-WebCon/5.x.x libwww/5.x.x  
Disallow: /
User-agent: W3C-checklink/3.x.x.x libwww-perl/5.xx  
Disallow: /
User-agent: W3C-checklink/4.x [4.xx] libwww-perl/5.xxx  
Disallow: /
User-agent: W3CLineMode/5.4.0 libwww/5.x.x  
Disallow: /
User-agent: W3CRobot/5.4.0 libwww/5.4.0  
Disallow: /
User-agent: W3C_Validator/1.xxx libwww-perl/5.xx  
Disallow: /
User-agent: W3SiteSearch Crawler_v1.1 http://www.w3sitesearch.de  
Disallow: /
User-agent: WAVcheck 1.0.x (http://www.webbanalys.se/apps/WAVcheck/)  
Disallow: /
User-agent: WDG_Validator/1.1  
Disallow: /
User-agent: WEP Search 00  
Disallow: /
User-agent: WFARC  
Disallow: /
User-agent: WIRE/0.11 (Linux; i686; Bot,Robot,Spider,Crawler,aromano@cli.di.unipi.it)  
Disallow: /
User-agent: WIRE/0.x (Linux; i686; Bot,Robot,Spider,Crawler)  
Disallow: /
User-agent: WISEbot/1.0 (WISEbot@koreawisenut.com; http://wisebot.koreawisenut.com)  
Disallow: /
User-agent: WSB WebCrawler V1.0 (Beta)- cl@cs.uni-dortmund.de  
Disallow: /
User-agent: WSB, http://websearchbench.cs.uni-dortmund.de  
Disallow: /
User-agent: WWSBOT 1.x [--- http://www.analyzer.nu ---]  
Disallow: /
User-agent: WWW-Mechanize/1.1x  
Disallow: /
User-agent: WWWC/1.0x  
Disallow: /
User-agent: WWWOFFLE/2.x  
Disallow: /
User-agent: WWWeasel Robot v1.00 (http://wwweasel.de)  
Disallow: /
User-agent: WannaBe (Macintosh; PPC)  
Disallow: /
User-agent: WapOnWindows 1.0  
Disallow: /
User-agent: Watchfire WebXM 1.0  
Disallow: /
User-agent: Wavefire/0.8-dev (Wavefire; http://www.wavefire.com; info@wavefire.com)  
Disallow: /
User-agent: Waypath Scout v2.x - info at waypath dot com  
Disallow: /
User-agent: Waypath development crawler - info at waypath dot com  
Disallow: /
User-agent: WeBoX/0.xx  
Disallow: /
User-agent: Web Image Collector  
Disallow: /
User-agent: Web Link Validator 1.5  
Disallow: /
User-agent: Web Snooper  
Disallow: /
User-agent: Web-Bot V1.03  
Disallow: /
User-agent: Web-Robot/5.0 (en-US; web-robot.com/policy.html) Web-Robot Crawler/2.0.3  
Disallow: /
User-agent: WebAlta Crawler/1.2.1 (http://www.webalta.ru/bot.html)  
Disallow: /
User-agent: WebAuto/3.4xxx (WinNT; I)  
Disallow: /
User-agent: WebBug/5.x  
Disallow: /
User-agent: WebCompass 2.0  
Disallow: /
User-agent: WebCopier vx.x  
Disallow: /
User-agent: WebCopier vx.xa  
Disallow: /
User-agent: WebCorp/1.0  
Disallow: /
User-agent: WebDownloader for X x.xx  
Disallow: /
User-agent: WebFetch  
Disallow: /
User-agent: WebFilter Robot 1.0  
Disallow: /
User-agent: WebFilter Robot 1.x  
Disallow: /
User-agent: WebFindBot(http://www.web-find.com)  
Disallow: /
User-agent: WebImages 0.3 ( http://herbert.groot.jebbink.nl/?app=WebImages )  
Disallow: /
User-agent: WebLight/4.x.x (support@illumit.com; http://www.illumit.com/Products/weblight/)  
Disallow: /
User-agent: WebMiner/x.x [en] (Win98; I)  
Disallow: /
User-agent: WebPix 1.0 (www.netwu.com)  
Disallow: /
User-agent: WebQL  
Disallow: /
User-agent: WebRACE/1.1 (University of Cyprus- Distributed Crawler)  
Disallow: /
User-agent: WebRankSpider/1.37 (+http://ulm191.server4you.de/crawler/)  
Disallow: /
User-agent: WebReaper [info@webreaper.net]  
Disallow: /
User-agent: WebReaper [webreaper@webreaper.net]  
Disallow: /
User-agent: WebReaper vx.x - www.webreaper.net  
Disallow: /
User-agent: WebSearch.COM.AU/3.0.1 (The Australian Search Engine; http://WebSearch.COM.AU; Search@WebSearch.COM.AU)  
Disallow: /
User-agent: WebSearchBench WebCrawler V1.0 (Beta)- Prof. Dr.-Ing. Christoph Lindemann- Universit�t Dortmund- cl@cs.uni-dortmund.de- http://websearchbench.cs.uni-dortmund.de/  
Disallow: /
User-agent: WebSearchBench WebCrawler v0.1(Experimental)  
Disallow: /
User-agent: WebStat/1.0 (Unix; beta; 20040314)  
Disallow: /
User-agent: WebStripper/2.xx  
Disallow: /
User-agent: WebTrafficExpress/x.0  
Disallow: /
User-agent: WebTrends/3.0 (WinNT)  
Disallow: /
User-agent: WebVac (webmaster@pita.stanford.edu)  
Disallow: /
User-agent: WebVal/1.0  
Disallow: /
User-agent: WebVulnCrawl.blogspot.com/1.0 libwww-perl/5.803  
Disallow: /
User-agent: WebWatcherMonitor/2.01  
Disallow: /
User-agent: WebZIP/x.x (http://www.spidersoft.com)  
Disallow: /
User-agent: WebarooBot (Webaroo Bot; http://64.124.122.252/feedback.html)  
Disallow: /
User-agent: WebarooBot (Webaroo Bot; http://www.webaroo.com/rooSiteOwners.html)  
Disallow: /
User-agent: Webclipping.com  
Disallow: /
User-agent: Webdup/0.9  
Disallow: /
User-agent: Webglimpse 2.xx.x (http://webglimpse.net)  
Disallow: /
User-agent: Weblink's checker/  
Disallow: /
User-agent: Weblog Attitude Diffusion 1.0  
Disallow: /
User-agent: Website Explorer/0.9.x.x  
Disallow: /
User-agent: Website eXtractor  
Disallow: /
User-agent: WebsiteWorth v1.0  
Disallow: /
User-agent: Webspinne/1.0 webmaster@webspinne.de  
Disallow: /
User-agent: Websquash.com (Add url robot)  
Disallow: /
User-agent: Webster v0.3 ( http://webster.healeys.net/ )  
Disallow: /
User-agent: Webverzeichnis.de - Telefon: 01908 / 26005  
Disallow: /
User-agent: Wells Search II  
Disallow: /
User-agent: West Wind Internet Protocols 4.xx  
Disallow: /
User-agent: Wget/1.x(.x)GNU wget http://www.gnu.org/software/wget/wget.html - file downloader  
Disallow: /
User-agent: Wget/1.x+cvs-stable (Red Hat modified)  
Disallow: /
User-agent: Wget/1.x.x+cvs  
Disallow: /
User-agent: Whatsup/x.x  
Disallow: /
User-agent: WhizBang! Lab  
Disallow: /
User-agent: Wildsoft Surfer  
Disallow: /
User-agent: Willow Internet Crawler by Twotrees V2.1  
Disallow: /
User-agent: WinGet 1.1  
Disallow: /
User-agent: WinHTTP Example/1.0  
Disallow: /
User-agent: WinPodder (http://winpodder.com)  
Disallow: /
User-agent: WinWAP/3.x (3.x.x.xx; Win32) (Google WAP Proxy/1.0)  
Disallow: /
User-agent: WinampMPEG/2.00 (larbin@unspecified.mail)  
Disallow: /
User-agent: WincerSong Agent v1.0  
Disallow: /
User-agent: Windows-Media-Player/10.00.00.xxxx  
Disallow: /
User-agent: WinkBot/0.06 (Wink.com search engine web crawler; http:/
Disallow: /
User-agent: *
Disallow: /*.gif$
Disallow: /*.Gif$
Disallow: /*.GIF$
Disallow: /*.jpg$
Disallow: /*.Jpg$
Disallow: /*.JPG$
Disallow: /*.jpeg$
Disallow: /*.Jpeg$
Disallow: /*.JPEG$
Disallow: /*.pdf$
Disallow: /*.Pdf$
Disallow: /*.PDF$
Disallow: /*.zip$
Disallow: /*.Zip$
Disallow: /*.ZIP$