Improves detection for generic bots (matomo-org#8236)

* Add TprAdsTxtCrawler to generic bots

* Adds detection for CopyvioDetector

* Improves detection for generic bots

* Improves detection for generic bots

* Improves regex

* Improves detection for generic bots

* Improves detection for generic bots

* Move Secweb-Sectxt

Author: liviuconcioiu

Tests/fixtures/bots.yml

@@ -9059,7 +9059,8 @@
 -
   user_agent: Secweb-Sectxt/1.0
   bot:
-    name: Generic Bot
+    name: Secweb-Sectxt
+    category: Crawler
 -
   user_agent: parser
   bot:
@@ -9763,3 +9764,45 @@
     producer:
       name: DEED Baltic, UAB
       url: http://www.deed.lt/
+-
+  user_agent: TprAdsTxtCrawler/1.1
+  bot:
+    name: TprAdsTxtCrawler
+    category: Crawler
+-
+  user_agent: 'Mozilla/5.0 (compatible; CopyvioDetector/0.4.dev0; +wikipedia.earwig[at]gmail.com)'
+  bot:
+    name: CopyvioDetector
+    category: Crawler
+    url: https://copyvios.toolforge.org/
+    producer:
+      name: Ben Kurtovic
+      url: https://github.com/earwig
+-
+  user_agent: Mozilla/5.0 (compatible; Dormouse/1.0)
+  bot:
+    name: Dormouse
+    category: Crawler
+-
+  user_agent: Mozilla/5.0 (compatible; FormFinder/1.0)
+  bot:
+    name: FormFinder
+    category: Crawler
+-
+  user_agent: Mozilla/5.0 (compatible; ImportDomains/1.0)
+  bot:
+    name: ImportDomains
+    category: Crawler
+-
+  user_agent: YourUserAgentHere
+  bot:
+    name: Generic Bot
+-
+  user_agent: desktop
+  bot:
+    name: Generic Bot
+-
+  user_agent: StudyBot/1.0 (Research Purposes)
+  bot:
+    name: StudyBot
+    category: Crawler

regexes/bots.yml

@@ -2507,10 +2507,6 @@
     name: 'Spotify'
     url: 'https://www.spotify.com'
 
-- regex: 'The Knowledge AI'
-  name: 'The Knowledge AI'
-  category: 'Crawler'
-
 - regex: 'Embedly'
   name: 'Embedly'
   category: 'Crawler'
@@ -2734,10 +2730,6 @@
   category: 'Service bot'
   url: 'https://www.grammarly.com'
 
-- regex: 'Robozilla'
-  name: 'Robozilla'
-  category: 'Crawler'
-
 - regex: 'Domains Project'
   name: 'Domains Project'
   category: 'Crawler'
@@ -3221,14 +3213,6 @@
   category: 'Site Monitor'
   url: 'https://www.dotcom-monitor.com'
 
-- regex: 'ThinkChaos/'
-  name: 'ThinkChaos'
-  category: 'Crawler'
-
-- regex: 'Thinkbot/'
-  name: 'Thinkbot'
-  category: 'Crawler'
-
 - regex: 'DataForSeoBot'
   name: 'DataForSeoBot'
   category: 'Crawler'
@@ -3454,14 +3438,6 @@
   category: 'Crawler'
   url: 'https://domaincrawler.com/about-us/'
 
-- regex: 'DNSResearchBot'
-  name: 'DNSResearchBot'
-  category: 'Crawler'
-
-- regex: 'GitCrawlerBot'
-  name: 'GitCrawlerBot'
-  category: 'Crawler'
-
 - regex: 'AdAuth'
   name: 'AdAuth'
   category: 'Crawler'
@@ -4675,10 +4651,6 @@
   name: 'WebMon'
   category: 'Site Monitor'
 
-- regex: 'AdsTxtCrawlerTP'
-  name: 'AdsTxtCrawlerTP'
-  category: 'Crawler'
-
 - regex: 'fragFINN'
   name: 'fragFINN'
   category: 'Crawler'
@@ -5836,12 +5808,20 @@
     name: 'Semantic Visions, s.r.o.'
     url: 'https://www.semantic-visions.com/'
 
+- regex: 'CopyvioDetector'
+  name: 'CopyvioDetector'
+  category: 'Crawler'
+  url: 'https://copyvios.toolforge.org/'
+  producer:
+    name: 'Ben Kurtovic'
+    url: 'https://github.com/earwig'
+
 # Generic bots
-- regex: '(ABEvalBot|HanaleiBot|PrivacyPolicyBot|SeoCherryBot)'
+- regex: '(ABEvalBot|AdsTxtCrawlerTP|DNSResearchBot|Dormouse|FormFinder|GitCrawlerBot|HanaleiBot|ImportDomains|PrivacyPolicyBot|Robozilla|Secweb-Sectxt|SeoCherryBot|StudyBot|The Knowledge AI|Thinkbot|ThinkChaos|TprAdsTxtCrawler)'
   name: '$1'
   category: 'Crawler'
 
-- regex: 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\.o\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|\(compatible\)|John Recon|SPARK COMMIT|masjesu|Komaru_The_Cat|Jesus Christ of Nazareth is LORD|Kowai|Hakai|LoliSec|LMAO|^xenu|^(?:chrome|firefox|Abcd|Dark|KvshClient|node|Node.js|Report Runner|url|Zeus|ZmEu)$|OnlyScans|TheInternetSearchx|Laravel Reaver|bang2013|libredtail|Mozilliqa|Tiberius|Secweb-Sectxt|honeygain|AW-WB-Filter|SaferSoftwashLeadGen'
+- regex: 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\.o\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|\(compatible\)|John Recon|SPARK COMMIT|masjesu|Komaru_The_Cat|Jesus Christ of Nazareth is LORD|Kowai|Hakai|LoliSec|LMAO|^xenu|^(?:chrome|desktop|firefox|Abcd|Dark|KvshClient|node|Node.js|Report Runner|url|Zeus|ZmEu)$|OnlyScans|TheInternetSearchx|Laravel Reaver|bang2013|libredtail|Mozilliqa|Tiberius|honeygain|AW-WB-Filter|SaferSoftwashLeadGen|YourUserAgentHere'
   name: 'Generic Bot'
 
 # Generic detections