using System.Collections.Generic;
public static void Main()
Console.WriteLine(IsInSearchUserAgent("AdsBot-Google"));
private static bool IsInSearchUserAgent(string useAgent)
var crawlerUserAgents = GetCrawlerUserAgents();
useAgent.IndexOf(crawlerUserAgent, StringComparison.InvariantCultureIgnoreCase) >= 0));
private static IEnumerable<String> GetCrawlerUserAgents()
var crawlerUserAgents = new List<string>(new[]
"googlebot", "yahoo", "bingbot", "yandex", "baiduspider", "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot",
"embedly", "quora link preview", "showyoubot", "outbrain", "pinterest/0.",
"developers.google.com/+/web/snippet", "slackbot", "vkShare", "W3C_Validator",
"redditbot", "Applebot", "WhatsApp", "flipboard", "tumblr", "bitlybot",
"SkypeUriPreview", "nuzzel", "Discordbot", "Google Page Speed", "x-bufferbot"
var CrawlerUserAgents = "AdsBot-Google (+http://www.google.com/adsbot.html),Googlebot-Image/1.0,Googlebot/2.1 (+http://www.googlebot.com/bot.html),Mozilla/5.0+(compatible;+Googlebot/2.1;++http://www.google.com/bot.html),Googlebot/Test (+http://www.googlebot.com/bot.html),Googlebot/Test,Mediapartners-Google/2.1 (+http://www.googlebot.com/bot.html),Mediapartners-Google/2.1,Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html),Mozilla/4.0 (compatible; MSIE 5.0; Windows NT),Mozilla/4.0,Mozilla/4.05 [en],Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html),Mozilla/5.0 (compatible; Yahoo! DE Slurp; http://help.yahoo.com/help/us/ysearch/slurp),Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html),Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp),Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp),Scooter/3.3,Y!J-BSC/1.0 (http://help.yahoo.co.jp/help/jp/blog-search/),Yahoo! Mindset,Yahoo-Blogs/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html ),Yahoo-MMAudVid/1.0 (mms dash mmaudvidcrawler dash support at yahoo dash inc dot com),Yahoo-MMCrawler/3.x (mms dash mmcrawler dash support at yahoo dash inc dot com),YahooFeedSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; my.yahoo.com/s/publishers.html),YahooSeeker-Testing/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/),YahooSeeker/1.1 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/),YahooSeeker/1.2 (compatible; Mozilla 4.0; MSIE 5.5; yahooseeker at yahoo-inc dot com ; http://help.yahoo.com/help/us/shop/merchant/),YahooSeeker/CafeKelsa-dev (compatible; Konqueror/3.2; FreeBSD ;cafekelsa-dev-webmaster@yahoo-inc.com ) (KHTML, like Gecko),YahooVideoSearch www.yahoo.com/,YahooYSMcm/2.0.0,LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com),facebot|facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php),facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php),XING-contenttabreceiver/2.0|Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5,Twitterbot/1.0,Bingbot,Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm),Baiduspider,Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html),Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html),Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/61.0.3159.5 Safari/537.36 Prerender (+https://github.com/prerender/prerender),Yandex,Googlebot";
crawlerUserAgents.AddRange(CrawlerUserAgents.Trim().Split(','));
return crawlerUserAgents;