Improves detection for generic bots (matomo-org#8200)

* Improves detection for generic bots

* Add more generic bots

---------

Co-authored-by: Stefan Giehl <stefan@matomo.org>

Author: liviuconcioiu

Parser/Bot.php

@@ -74,12 +74,18 @@ public function parse(): ?array
             foreach ($this->getRegexes() as $regex) {
                 $matches = $this->matchUserAgent($regex['regex']);
 
-                if ($matches) {
-                    unset($regex['regex']);
-                    $result = $regex;
+                if (!$matches) {
+                    continue;
+                }
+
+                unset($regex['regex']);
+                $result = $regex;
 
-                    break;
+                if (\array_key_exists('name', $result)) {
+                    $result['name'] = $this->buildByMatch($result['name'], $matches);
                 }
+
+                break;
             }
         }
 

Tests/fixtures/bots.yml

@@ -8948,6 +8948,26 @@
     Sec-CH-UA: 'Mozilliqa"<?=print(9347655345-4954366);?>"'
     Sec-CH-UA-Platform: 'Lindows&lt;?=print(9347655345-4954366);?&gt;'
     Sec-CH-UA-Mobile: '?0&lt;?=print(9347655345-4954366);?&gt;'
+-
+  user_agent: Mozilla/5.0 (compatible; PrivacyPolicyBot/1.0)
+  bot:
+    name: PrivacyPolicyBot
+    category: Crawler
+-
+  user_agent: Mozilla/5.0 (compatible; SeoCherryBot/1.0; +https://en.wikipedia.org/wiki/Web_crawler)
+  bot:
+    name: SeoCherryBot
+    category: Crawler
+-
+  user_agent: HanaleiBot runid=beta-stage-integration-test
+  bot:
+    name: HanaleiBot
+    category: Crawler
+-
+  user_agent: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ABEvalBot/0.1) Version/11.1.2 Safari/605.1.15
+  bot:
+    name: ABEvalBot
+    category: Crawler
 -
   user_agent: Google-NotebookLM
   bot:

regexes/bots.yml

@@ -5651,6 +5651,10 @@
     url: 'https://www.semantic-visions.com/'
 
 # Generic bots
+- regex: '(ABEvalBot|HanaleiBot|PrivacyPolicyBot|SeoCherryBot)'
+  name: '$1'
+  category: 'Crawler'
+
 - regex: 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\.o\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|\(compatible\)|John Recon|SPARK COMMIT|masjesu|Komaru_The_Cat|Jesus Christ of Nazareth is LORD|Kowai|Hakai|LoliSec|LMAO|^xenu|^(?:chrome|firefox|Abcd|Dark|KvshClient|Node.js|Report Runner|url|Zeus|ZmEu)$|OnlyScans|TheInternetSearchx|Laravel Reaver|bang2013|libredtail|Mozilliqa|Tiberius'
   name: 'Generic Bot'