# robots.txt should never be edited directly. (ssl version) # The .htaccess file determines whether it is a production or development environment and redirects appropriately to the correct dev vs prod file. # robotsproduction.txt should also never be edited directly. # robotsproduction.template should be modified instead, so that the visihow.com value below can be filled automatically by index.php # IMPORTANT!!! READ THE ABOVE LINES TO MAKE SURE YOU ARE MODIFYING THE CORRECT FILE!!! # # robots.txt for https://visihow.com # based on wikipedia.org's robots.txt # # Crawlers that are kind enough to obey, but which we'd rather not have # unless they're feeding search engines. Sitemap: https://visihow.com/sitemap_index_ssl.xml # If your bot supports such a thing using the 'Crawl-delay' or another # instruction, please let us know. We can add it to our robots.txt. # # Friendly, low-speed bots are welcome viewing article pages, but not # dynamically-generated pages please. Article pages contain our site's # real content. User-agent: UbiCrawler Disallow: / User-agent: DOC Disallow: / User-agent: Zao Disallow: / # Some bots are known to be trouble, particularly those designed to copy # entire sites. Please obey robots.txt. User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: HTTrack Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: libwww Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / # wget in recursive mode uses too many resources for us. # Please read the man page and use it properly; there is a # --wait option you can use to set the delay between hits, # for instance. Please wait 3 seconds between each request. User-agent: wget Disallow: / # Doesn't follow robots.txt anyway, but... User-agent: k2spider Disallow: / # Requests many pages per second # http://www.nameprotect.com/botinfo.html User-agent: NPBot Disallow: / # A capture bot, downloads gazillions of pages with no public benefit # http://www.webreaper.net/ User-agent: WebReaper Disallow: / User-agent: HMSE_Robot Disallow: / User-agent: Googlebot Allow: /Special:Categorylisting Allow: /Special:Allpages Allow: /Special:Popularpages Allow: /Special:Newpages Allow: /Special:Sitemap Allow: /Special:LSearch Allow: /Special:Userlogin Allow: /Special:Emailuser Allow: /Special:ThankAuthors Allow: /Special:RCTiles Allow: /index.php?*action=credits Allow: /index.php?*action=ajax* Allow: /index.php?*printable #Disallow: /w/ #Disallow: /forum/ #Disallow: /index.php #Disallow: /*feed=rss #Disallow: /*action=delete #Disallow: /*action=edit #Disallow: /*action=history #Disallow: /*action=protect #Disallow: /*action=watch #Disallow: /Special: Disallow: /api.php Disallow: /images/samplepdfs Disallow: /images/sampledocs*pdf Disallow: /images/sampledocs*doc Disallow: /images/sampledocs*txt Noindex: /w/ Noindex: /forum/ Noindex: /index.php Noindex: /*feed=rss Noindex: /*action=history Noindex: /*action=delete Noindex: /Special: Noindex: /api.php # these rules should be identical to the User-agent: Googlebot rules User-agent: Googlebot-Mobile Allow: /Special:Categorylisting Allow: /Special:Allpages Allow: /Special:Popularpages Allow: /Special:Newpages Allow: /Special:Sitemap Allow: /Special:LSearch Allow: /Special:Userlogin Allow: /Special:Ipblocklist Allow: /Special:Blockip Allow: /Special:Emailuser Allow: /Special:ThankAuthors Allow: /Special:RCTiles Allow: /index.php?*action=credits Allow: /index.php?*action=ajax* Allow: /index.php?*printable Allow: /index.php?title=Special:Booksources Allow: /index.php?*ajaxLoadWiki* #Disallow: /w/ #Disallow: /forum/ #Disallow: /index.php #Disallow: /*feed=rss #Disallow: /*action=delete #Disallow: /*action=edit #Disallow: /*action=history #Disallow: /*action=protect #Disallow: /*action=watch #Disallow: /Special: #Disallow: /api.php Disallow: /images/samplepdfs Disallow: /images/sampledocs*pdf Disallow: /images/sampledocs*doc Disallow: /images/sampledocs*txt Noindex: /w/ Noindex: /forum/ Noindex: /index.php Noindex: /*feed=rss Noindex: /*action=history Noindex: /*action=delete Noindex: /Special: Noindex: /api.php User-agent: * Allow: /Special:Categorylisting Allow: /Special:Allpages Allow: /Special:Popularpages Allow: /Special:Newpages Allow: /Special:Sitemap Allow: /Special:LSearch #Disallow: /w/ #Disallow: /forum/ #Disallow: /index.php #Disallow: /*feed=rss #Disallow: /*action=delete #Disallow: /*action=edit #Disallow: /*action=history #Disallow: /*action=protect #Disallow: /*action=watch #Disallow: /Special: Disallow: /images/samplepdfs Disallow: /images/sampledocs*pdf Disallow: /images/sampledocs*doc Disallow: /images/sampledocs*txt