# # robots.txt for http://www.fortygrand.com # # User-agent: msnbot Disallow: /*.gif$ Disallow: /*.mp3$ Disallow: /*.wmv$ Disallow: /fortygrand/*.gif$ Disallow: /fortygrand/*.mp3$ Disallow: /fortygrand/*.wmv$ User-agent: Googlebot-Image Disallow: /*.gif$ Disallow: /*.mp3$ Disallow: /*.wmv$ Disallow: /fortygrand/*.gif$ Disallow: /fortygrand/*.mp3$ Disallow: /fortygrand/*.wmv$ # Crawlers that are kind enough to obey, but which we'd rather not have # unless they're feeding search engines. User-agent: UbiCrawler Disallow: / User-agent: DOC Disallow: / User-agent: Zao Disallow: / # Some bots are known to be trouble, particularly those designed to copy # entire sites. Please obey robots.txt. User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: HTTrack Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: libwww Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / # # Sorry, wget in its recursive mode is a frequent problem. # Please read the man page and use it properly; there is a # --wait option you can use to set the delay between hits, # for instance. # User-agent: wget Disallow: / # # The 'grub' distributed client has been *very* poorly behaved. # User-agent: grub-client Disallow: / # # Doesn't follow robots.txt anyway, but... # User-agent: k2spider Disallow: / # # Hits many times per second, not acceptable # http://www.nameprotect.com/botinfo.html User-agent: NPBot Disallow: / # A capture bot, downloads gazillions of pages with no public benefit # http://www.webreaper.net/ User-agent: WebReaper Disallow: / User-agent: yahoo-mmcrawler Disallow: / User-agent: psbot Disallow: / User-agent: * Disallow: /*.gif$ Disallow: /*.mp3$ Disallow: /*.wmv$ Disallow: /fortygrand/*.gif$ Disallow: /fortygrand/*.mp3$ Disallow: /fortygrand/*.wmv$ Disallow: /fortygrand/bandblooddrive.gif Disallow: /fortygrand/persons.gif Disallow: /fortygrand/bandmarithon.gif Disallow: /fortygrand/sultans_of_swing.mp3 Disallow: /fortygrand/summertime.mp3 Disallow: /fortygrand/group-wide-2.gif Disallow: /fortygrand/band.gif Disallow: /fortygrand/group-tall-2.gif Disallow: /fortygrand/never_been_any_reason.mp3 Disallow: /fortygrand/youre_no_good.mp3 Disallow: /fortygrand/aint_seen_nothing_yet.mp3 Disallow: /fortygrand/unchained_melody.mp3 Disallow: /fortygrand/green_eyed_lady.mp3 Disallow: /fortygrand/wake_me_when_september_ends.mp3 Disallow: /fortygrand/breakup_song.mp3 Disallow: /fortygrand/lagrange.wmv Disallow: /fortygrand/wonderful_tonight.mp3 Disallow: /fortygrand/playthatfunkymusic.wmv Disallow: /fortygrand/take_care.wmv Disallow: /fortygrand/band-members.gif Disallow: /fortygrand/keepyourhandstoyourself.wmv Disallow: /fortygrand/china_grove.mp3 Disallow: /fortygrand/crowd.gif Disallow: /fortygrand/midnight_hour.mp3 Disallow: /fortygrand/proud_mary.wmv Disallow: /fortygrand/wonderful_tonight.wmv Disallow: /bandblooddrive.gif Disallow: /persons.gif Disallow: /bandmarithon.gif Disallow: /sultans_of_swing.mp3 Disallow: /summertime.mp3 Disallow: /group-wide-2.gif Disallow: /band.gif Disallow: /group-tall-2.gif Disallow: /never_been_any_reason.mp3 Disallow: /youre_no_good.mp3 Disallow: /aint_seen_nothing_yet.mp3 Disallow: /unchained_melody.mp3 Disallow: /green_eyed_lady.mp3 Disallow: /wake_me_when_september_ends.mp3 Disallow: /breakup_song.mp3 Disallow: /lagrange.wmv Disallow: /wonderful_tonight.mp3 Disallow: /playthatfunkymusic.wmv Disallow: /take_care.wmv Disallow: /band-members.gif Disallow: /keepyourhandstoyourself.wmv Disallow: /china_grove.mp3 Disallow: /crowd.gif Disallow: /midnight_hour.mp3 Disallow: /proud_mary.wmv Disallow: /wonderful_tonight.wmv