# $Id: robots.txt,v 1.0 19/05/2002 17:00:23 Ivan van der Tuuk $ # # This is a file retrieved by webwalkers a.k.a. spiders that # conform to a defacto standard. # See # # The webmaster for this site is # # Format is: # User-agent: # Disallow: | # Flag Date By Reason # $l1- 19950130 itk finally understood what the file was for! # $L2= 19960909 itk fixed url since make moved to Webcrawler... # $L3= 19970811 itk drop /Stretch/ # $L4= 19991102 itk fixed User-agent capitalization and contact info # $L5= 20010327 itk Updated disallow rules # ----------------------------------------------------------------------------- Sitemap: http://www.wdnederland.nl/sitemap.xml # PARTIAL access (Acoon) User-agent: Acoon Robot Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (AllThatNet) User-agent: ATN_Worldwide Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Alta Vista) User-agent: Scooter Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Anzwers) User-agent: AnzwersCrawl Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (AustLII) User-agent: Gromit Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Direct Hit Grabber) User-agent: grabber Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Entire Web) User-agent: speedy Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (EuroSeek) User-agent: Freecrawl Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Excite) User-agent: ArchitextSpider Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (FAST/AllTheWeb) User-agent: FAST-WebCrawler Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Fireball) User-agent: KIT-Fireball Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Goo) User-agent: moget Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Google) User-agent: Googlebot Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Griffon) User-agent: griffon Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Hämähäkki) User-agent: Hämähäkki Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Industry Central) User-agent: Open Text Site Crawler Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (InfoSeek.de) User-agent: marvin Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Ingrid) User-agent: INGRID/0.1 Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Inktomi) User-agent: Slurp Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Internet Cruiser) User-agent: Internet Cruiser Robot Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Kvasir) User-agent: solbot Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Legs) User-agent: legs Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Lets Find It Now!) User-agent: elfinbot Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Lisa) User-agent: Voyager/0.0 Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Lycos) User-agent: Lycos_Spider_(T-Rex) Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (MPRM Group Limited) User-agent: spider_monkey Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Mirago) User-agent: mirago Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (NetScoop) User-agent: NetScoop Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Northern Light) User-agent: Gulliver Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (ODiN) User-agent: Valkyrie libwww-perl Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Openfind) User-agent: openbot Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (PlanetSearch) User-agent: fido Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Portal Juice) User-agent: pjspider Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (SearchUK) User-agent: MegaSheep Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Suchmaschine21) User-agent: CoolBot Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (Thunderstone) User-agent: T-H-U-N-D-E-R-S-T-O-N-E Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (TopicLink) User-agent: topiclink Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (VietGATE) User-agent: jcrawler Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (WhoWhere?) User-agent: whowhere Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # NO access (e-collector, CMC/0.01, Google Image) User-agent: ecollector User-agent: CMC/0.01 User-agent: Googlebot-Image Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (iaNett.com) User-agent: ParaSite Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (kensaku.jp) User-agent: suke Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (mopilot.com) User-agent: wapspider Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (nathan) User-agent: tarantula Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (wdstatsscan) User-agent: wdstatsscan-online Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # FULL access (whatUseek) User-agent: winona Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # PARTIAL access (All Spiders) User-agent: * Disallow: /cgi-bin/ Disallow: /catalog5/ Disallow: /admin/ Disallow: /catalog3/ Disallow: /catalog5/admin/ # deny access (wb wininet) User-agent: vb wininet Disallow: / # deny access (Internet Archiver) User-agent: ia_archiver Disallow: /