##################################### # List of WHITELIST ALLOW user-agents # CPM: Adsense banner and contexual bots User-agent: Mediapartners-Google* # support.google.com/adsense/bin/answer.py?hl=en&answer=99376 Allow: / ##################################### # List of WHITELIST ALLOW user-agents # CPM: Adsense banner and contexual bots User-agent: Mediapartners-Google # Adsense contextual targeting bot Allow: / ##################################### # List of WHITELIST ALLOW user-agents # CPM: Adsense banner and contexual bots User-agent: msnbot-media/1.0 # MSN contextual targeting bot Allow: / # PPC: Adwords & AdCenter landing page bots User-agent: Adsbot-Google # AdwordsPPC tinyurl.com/list-of-Google-Crawlers Allow: / # PPC: Adwords & AdCenter landing page bots User-agent: AdsBot-Google-Mobile # AdwordsPPC for mobile campaigns Allow: / # PPC: Adwords & AdCenter landing page bots User-agent: Adidxbot # BingPPC (aka MSNPTC) www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0 Allow: / # PPC: Adwords & AdCenter landing page bots User-agent: Yahoo!-AdCrawler # YahooPPC tinyurl.com/About-Yahoo-Adcrawler Allow: / # SEO: Organic Crawlers User-agent: Googlebot # support.google.com/webmasters/bin/answer.py?hl=en&answer=1061943 Allow: / # SEO: Organic Crawlers User-agent: Googlebot-Mobile Allow: / # SEO: Organic Crawlers User-agent: Googlebot-News Allow: / # SEO: Organic Crawlers User-agent: Googlebot-Video Allow: / # SEO: Organic Crawlers User-agent: Googlebot-Image Allow: / # SEO: Organic Crawlers User-agent: gsa-crawler # Google-Appliance tinyurl.com/google-sitesearch-appliance Allow: / # SEO: Organic Crawlers User-agent: Bingbot # aka MSNBOT www.bing.com/bingbot.htm Allow: / # SEO: Organic Crawlers User-agent: MSNBot # old version of bingbot still used for multimedia and feeds crawls Allow: / # SEO: Organic Crawlers User-agent: BingPreview # bing.com/community/site_blogs/b/webmaster/archive/2012/10/26/page-snapshots-in-bing-windows-8-app-to-bring-new-crawl-traffic-to-sites.aspx Allow: / # SEO: Organic Crawlers User-agent: Slurp # aka Yahoo! Slurp - now part of Bingbot Allow: / # SEO: Organic Crawlers User-agent: TEOMA # Ask.com Allow: / # SEO: Organic Crawlers User-agent: Yahoo Allow: / # SEO: Organic Crawlers User-agent: Archive.org Allow: / # SEO: Organic Crawlers User-agent: YandexBot Allow: / ##################################### # Other user agents you might want to include: # ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider # gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot ##################################### # Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: * User-agent: SiteCon # www.nccgroup.com/en/our-services/website-performance/website-performance-monitoring/ Disallow: / ##################################### # Other user agents you might want to include: # ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider # gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot ##################################### # Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: * User-agent: GomezAgent # gomeznetworks.com/help/gpn/MySettings/Last_Mile_PP_Test_Settings.htm Disallow: / ##################################### # Other user agents you might want to include: # ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider # gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot ##################################### # Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: * User-agent: GomezAgent 1.0 Disallow: / ##################################### # Other user agents you might want to include: # ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider # gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot ##################################### # Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: * User-agent: GomezAgent 2.0 Disallow: / ##################################### # Other user agents you might want to include: # ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider # gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot ##################################### # Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: * User-agent: GomezAgent 3.0 Disallow: / ##################################### # Other user agents you might want to include: # ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider # gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot ##################################### # Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: * User-agent: YottaaMonitor Disallow: / ##################################### # Other user agents you might want to include: # ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider # gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot ##################################### # Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: * User-agent: FunWebProducts Disallow: / ##################################### # Other user agents you might want to include: # ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider # gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot ##################################### # Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: * User-agent: FastCrawler 3.0.X (crawler@1klik.dk) - http://www.1klik.dk Disallow: / ##################################### # Other user agents you might want to include: # ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider # gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot ##################################### # Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: * User-agent: FAST+Enterprise+Crawler+6+used+by+LexisNexis+(lnbot@lexisnexis.com) Disallow: / ##################################### # Other user agents you might want to include: # ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider # gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot ##################################### # Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: * User-agent: FAST Enterprise Crawler 6 used by LexisNexis Disallow: / ##################################### # Other user agents you might want to include: # ArchitextSpider, Lycos_Spider_(T-Rex), FAST-WebCrawler, JikeSpider # gulliver, fast, scooter, vscooter, Nutch, ZyBorg, SurveyBot ##################################### # Server-Monitoring: BLACKLIST SiteConfidence & Gomez just incase they ignore user-agent: * User-agent: FAST+Enterprise+Crawler/6.7.0+(crawler@fast.no) Disallow: / User-agent: AhrefsBot Disallow: / User-agent: Baiduspider Disallow: / User-agent: SEOkicks-Robot Disallow: / User-agent: BLEXBot Disallow: / User-agent: FAST* Disallow: / # All undefined bots User-agent: * Disallow: Disallow: /wp-admin/ Disallow: /wp-snapshots/ Disallow: /cache/ Disallow: /tmp/ Disallow: /xmlrpc.php Sitemap: https://www.myideaplus.com/sitemap.xml