Merge branch 'refs/heads/develop'

Bots and Crawler.txt

@@ -1,76 +1,79 @@
-BLEXBot
-Barkrowler
-Livelapbot
-Franck, the Fediverse Graph Crawler
-YisouSpider
-DotBot
-VelenPublicWebCrawler
-ChatGPT-User
-GoogleOther
-GoogleOther-Image
-GoogleOther-Video
-Applebot
-Applebot-Extended
-AhrefsBot
-Baiduspider
-SemrushBot
-DotBot
-MJ12bot
-PetalBot
-SeznamBot
-Mediapartners-Google
-Bytespider
-Claude-Web
-GPTBot
-PerplexityBot
-Applebot
-Amazonbot
-Meta-ExternalAgent
-Meta-ExternalFetcher
-FacebookBot
-DuckAssistBot
-Anthropic-ai
-Bytedance
-fidget-spinner-bot
-EmailCollector
-WebEMailExtrac
-ClaudeBot
-facebookexternalhit
-facebookcatalog
-YandexBot
-Bingbot
-Googlebot
 AI2Bot
 Ai2Bot-Dolma
-Brightbot
+aiHitBot
+Amazonbot
+anthropic-ai
+Applebot
+Applebot-Extended
+Brightbot 1.0
+Bytespider
 CCBot
 ChatGPT-User
+Claude-Web
+ClaudeBot
 cohere-ai
 cohere-training-data-crawler
+Cotoyogi
 Crawlspace
 Diffbot
+DuckAssistBot
+FacebookBot
+Factset_spyderbot
+FirecrawlAgent
 FriendlyCrawler
 Google-Extended
 GoogleOther
 GoogleOther-Image
 GoogleOther-Video
-iaskspider
+GPTBot
+iaskspider/2.0
 ICC-Crawler
 ImagesiftBot
 img2dataset
+imgproxy
 ISSCyberRiskCrawler
-KangarooBot
+Kangaroo Bot
+meta-externalagent
+Meta-ExternalAgent
+meta-externalfetcher
+Meta-ExternalFetcher
+NovaAct
 OAI-SearchBot
 omgili
 omgilibot
+Operator
 PanguBot
 Perplexity-User
+PerplexityBot
 PetalBot
 Scrapy
 SemrushBot-OCOB
 SemrushBot-SWA
-Sidetradeindexerbot
+Sidetrade indexer bot
+TikTokSpider
 Timpibot
 VelenPublicWebCrawler
 Webzio-Extended
-YouBot
+YouBot
+AhrefsBot
+Baiduspider
+Barkrowler
+Bingbot
+BLEXBot
+Bytedance
+DotBot
+EmailCollector
+facebookcatalog
+facebookexternalhit
+fidget-spinner-bot
+Franck the Fediverse Graph Crawler
+Googlebot
+Livelapbot
+Mediapartners-Google
+MJ12bot
+SemrushBot
+SeznamBot
+VelenPublicWebCrawler
+WebEMailExtrac
+YandexBot
+YisouSpider