Add support for importing detection rules from https://github.com/aboutcode-data/detection-rules-collector

Add a test

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/improvers/__init__.py

@@ -19,10 +19,12 @@
 from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
 from vulnerabilities.pipelines import remove_duplicate_advisories
 from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees
+from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2
 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2
 from vulnerabilities.pipelines.v2_improvers import (
     computer_package_version_rank as compute_version_rank_v2,
 )
+from vulnerabilities.pipelines.v2_improvers import detection_rules
 from vulnerabilities.pipelines.v2_improvers import enhance_with_exploitdb as exploitdb_v2
 from vulnerabilities.pipelines.v2_improvers import enhance_with_kev as enhance_with_kev_v2
 from vulnerabilities.pipelines.v2_improvers import (
@@ -72,5 +74,6 @@
         collect_ssvc_trees.CollectSSVCPipeline,
         relate_severities.RelateSeveritiesPipeline,
         group_advisories_for_packages.GroupAdvisoriesForPackages,
+        detection_rules.DetectionRulesPipeline,
     ]
 )

vulnerabilities/models.py

@@ -3740,3 +3740,43 @@ class GroupedAdvisory(NamedTuple):
     weighted_severity: Optional[float]
     exploitability: Optional[float]
     risk_score: Optional[float]
+
+
+class DetectionRuleTypes(models.TextChoices):
+    """Defines the supported formats for security detection rules."""
+
+    YARA = "yara", "Yara"
+    YARA_X = "yara-x", "Yara-X"
+    SIGMA = "sigma", "Sigma"
+    CLAMAV = "clamav", "ClamAV"
+    SURICATA = "suricata", "Suricata"
+
+
+class DetectionRule(models.Model):
+    """
+    A Detection Rule is code used to identify malicious activity or security threats.
+    """
+
+    rule_type = models.CharField(
+        max_length=50,
+        choices=DetectionRuleTypes.choices,
+        help_text="The type of the detection rule content (e.g., YARA, Sigma).",
+    )
+
+    source_url = models.URLField(
+        max_length=1024, help_text="URL to the original source or reference for this rule."
+    )
+
+    rule_metadata = models.JSONField(
+        null=True,
+        blank=True,
+        help_text="Additional structured data such as tags, or author information.",
+    )
+
+    rule_text = models.TextField(help_text="The content of the detection signature.")
+
+    related_advisories = models.ManyToManyField(
+        AdvisoryV2,
+        related_name="detection_rules",
+        help_text="Advisories associated with this DetectionRule.",
+    )

vulnerabilities/pipelines/v2_improvers/detection_rules.py

@@ -0,0 +1,102 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+from pathlib import Path
+
+from fetchcode.vcs import fetch_via_vcs
+
+from vulnerabilities.models import AdvisoryAlias
+from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.models import DetectionRule
+from vulnerabilities.models import DetectionRuleTypes
+from vulnerabilities.pipelines import VulnerableCodePipeline
+
+
+class DetectionRulesPipeline(VulnerableCodePipeline):
+    """
+    Pipeline to collect vulnerability scanner rules (Sigma, YARA, Suricata, ClamAV entries)
+    """
+
+    pipeline_id = "detection_rules"
+    license_url = "https://github.com/ziadhany/detection-rules-collector/blob/master/LICENSE"
+    precedence = 200
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.clone,
+            cls.collect_detection_rules,
+            cls.clean_downloads,
+        )
+
+    def clone(self):
+        self.repo_url = "git+https://github.com/aboutcode-data/detection-rules-collector"
+        self.log(f"Cloning `{self.repo_url}`")
+        self.vcs_response = fetch_via_vcs(self.repo_url)
+
+    def advisories_count(self):
+        root = Path(self.vcs_response.dest_dir)
+        return sum(1 for _ in root.rglob("*.json"))
+
+    def collect_detection_rules(self):
+        base_path = Path(self.vcs_response.dest_dir) / "data"
+        rule_type_mapping = {
+            DetectionRuleTypes.YARA: "yara/**/*.json",
+            DetectionRuleTypes.SURICATA: "suricata/**/*.json",
+            DetectionRuleTypes.SIGMA: "sigma/**/*.json",
+            DetectionRuleTypes.CLAMAV: "clamav/**/*.json",
+        }
+
+        for rule_type, glob_pattern in rule_type_mapping.items():
+            for file_path in base_path.glob(glob_pattern):
+                with open(file_path, "r") as f:
+                    try:
+                        json_data = json.load(f)
+                    except json.JSONDecodeError:
+                        self.log(f"Failed to parse JSON in {file_path}")
+                        continue
+
+                source_url = json_data.get("source_url")
+                for rule in json_data.get("rules", []):
+                    advisories = set()
+                    for vulnerability_id in rule.get("vulnerabilities", []):
+                        try:
+                            if alias := AdvisoryAlias.objects.get(alias=vulnerability_id):
+                                for adv in alias.advisories.all():
+                                    advisories.add(adv)
+                            else:
+                                advs = AdvisoryV2.objects.filter(
+                                    advisory_id=vulnerability_id
+                                ).latest_per_avid()
+                                for adv in advs:
+                                    advisories.add(adv)
+                        except AdvisoryAlias.DoesNotExist:
+                            self.log(f"No advisory found for aliases {vulnerability_id}")
+
+                    raw_text = rule.get("rule_text")
+                    detection_rule, _ = DetectionRule.objects.get_or_create(
+                        rule_text=raw_text,
+                        rule_type=rule_type,
+                        defaults={
+                            "source_url": source_url,
+                        },
+                    )
+                    if advisories:
+                        detection_rule.related_advisories.add(*advisories)
+
+    def clean_downloads(self):
+        """Cleanup any temporary repository data."""
+        if self.vcs_response:
+            self.log(f"Removing cloned repository")
+            self.vcs_response.delete()
+
+    def on_failure(self):
+        """Ensure cleanup is always performed on failure."""
+        self.clean_downloads()

vulnerabilities/tests/pipelines/v2_improvers/test_detection_rules.py

@@ -0,0 +1,46 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import os
+from datetime import datetime
+from unittest.mock import Mock
+from unittest.mock import patch
+
+import pytest
+
+from vulnerabilities.models import AdvisoryAlias
+from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.models import DetectionRule
+from vulnerabilities.pipelines.v2_improvers.detection_rules import DetectionRulesPipeline
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+TEST_DATA = os.path.join(BASE_DIR, "../../test_data", "detection_rules")
+
+
+@pytest.mark.django_db
+@patch("vulnerabilities.pipelines.v2_improvers.detection_rules.fetch_via_vcs")
+def test_detection_rules_improver(mock_fetch_via_vcs):
+    mock_vcs_response = Mock()
+    mock_vcs_response.dest_dir = TEST_DATA
+    mock_fetch_via_vcs.return_value = mock_vcs_response
+
+    adv1 = AdvisoryV2.objects.create(
+        advisory_id="VCIO-123-2002",
+        datasource_id="ds",
+        avid="ds/VCIO-123-2002",
+        unique_content_id="i3giu",
+        url="https://test.com",
+        date_collected=datetime.now(),
+    )
+    alias = AdvisoryAlias.objects.create(alias="CVE-2007-4387")
+    adv1.aliases.add(alias)
+
+    improver = DetectionRulesPipeline()
+    improver.execute()
+    assert DetectionRule.objects.count() > 0

vulnerabilities/tests/test_data/detection_rules/data/clamav/main_hdb.json

@@ -0,0 +1,32 @@
+{
+  "source_url": "https://database.clamav.net/main.cvd?api-version=1",
+  "source_filename": "main.hdb",
+  "rules": [
+    {
+      "rule_metadata": {
+        "name": "Eicar-Test-Signature",
+        "line_num": 1
+      },
+      "rule_text": "44d88612fea8a8f36de82e1278abb02f:68:Eicar-Test-Signature",
+      "vulnerabilities": []
+    },
+    {
+      "rule_metadata": {
+        "name": "Win.Trojan.Yat-2",
+        "line_num": 3
+      },
+      "rule_text": "de3430cd6a3e24bfb9f78743a25f7c96:1098752:Win.Trojan.Yat-2",
+      "vulnerabilities": []
+    },
+    {
+        "rule_metadata": {
+            "name": "Java.Exploit.CVE_2012_5076-1",
+            "line_num": 10079
+        },
+        "rule_text": "a9b65b78619002a1b30ceee2d85fa770:205:Java.Exploit.CVE_2012_5076-1",
+        "vulnerabilities": [
+            "CVE-2012-5076"
+        ]
+    }
+  ]
+}

vulnerabilities/tests/test_data/detection_rules/data/clamav/main_ldb.json

@@ -0,0 +1,32 @@
+{
+  "source_url": "https://database.clamav.net/main.cvd?api-version=1",
+  "source_filename": "main.ldb",
+  "rules": [
+    {
+      "rule_metadata": {
+        "name": "Win.Exploit.CVE_2016_7185-1",
+        "line_num": 1
+      },
+      "rule_text": "Win.Exploit.CVE_2016_7185-1;Engine:51-255,Target:1;(0&1&2&3);44616e6765726f757347657448616e646c65;5361666548616e646c655a65726f4f724d696e75734f6e654973496e76616c6964;52656c6561736548616e646c65;5c004400650076006900630065005c0044006600730043006c00690065006e007400",
+      "vulnerabilities": [
+        "CVE-2016-7185"
+      ]
+    },
+    {
+      "rule_metadata": {
+        "name": "Doc.Trojan.Agent-1383193",
+        "line_num": 2
+      },
+      "rule_text": "Doc.Trojan.Agent-1383193;Engine:53-255,Target:2;0&1&2&3&4;57683370314d4c73576c69454b30626476376d707563704156724856585141694f30383755365a48556f;507a33593934674e796c784e724d5937706a3068;586c49766b65446349324259514d5169556b764d436165345144415452746d3842;434c70577561534d6f4c4845437a4172754d4d6466484b3334444e78;4256504271623368394c6e6c",
+      "vulnerabilities": []
+    },
+    {
+      "rule_metadata": {
+        "name": "Doc.Trojan.Agent-1383194",
+        "line_num": 3
+      },
+      "rule_text": "Doc.Trojan.Agent-1383194;Engine:53-255,Target:2;0&1>50;28373835362920417320427974652c20;3d2059656172284e6f77292027",
+      "vulnerabilities": []
+    }
+  ]
+}

vulnerabilities/tests/test_data/detection_rules/data/clamav/main_ndb.json

@@ -0,0 +1,32 @@
+{
+  "source_url": "https://database.clamav.net/main.cvd?api-version=1",
+  "source_filename": "main.ndb",
+  "rules": [
+    {
+      "rule_metadata": {
+        "name": "Legacy.Trojan.Agent-1",
+        "line_num": 1
+      },
+      "rule_text": "Legacy.Trojan.Agent-1:0:*:dd6d70241f674d8fc13e1eb3af731a7b5c43173c1cdd75722fa556c373b65c5275d513147b070077757064080386898ae75c6fb7f717b562ef636f6d6d613f2e0e202f6336c5eed52064f120228e2f6d27c101",
+      "vulnerabilities": []
+    },
+    {
+      "rule_metadata": {
+        "name": "Win.Trojan.Hotkey-1",
+        "line_num": 2
+      },
+      "rule_text": "Win.Trojan.Hotkey-1:0:*:c01640006a3cffb684000000ff159cef420089869800000089be940000008bc75f5ec20400565733ff8bf1397c240c741fff762089be8c000000ff1560ef42",
+      "vulnerabilities": []
+    },
+    {
+        "rule_metadata": {
+            "name": "Win.Exploit.CVE_2001_0500-1",
+            "line_num": 31344
+        },
+        "rule_text": "Win.Exploit.CVE_2001_0500-1:0:*:7961686f6f3a20607065726c202d6520277072696e7420225c783930227831313830302760245348454c4c434f44453d3230",
+        "vulnerabilities": [
+            "CVE-2001-0500"
+        ]
+    }
+  ]
+}

vulnerabilities/tests/test_data/detection_rules/data/sigma/web_sql_injection_in_access_logs.json

@@ -0,0 +1,16 @@
+{
+    "source_url": "https://github.com/SigmaHQ/sigma/blob/master/rules/web/webserver_generic/web_sql_injection_in_access_logs.yml",
+    "rules": [
+        {
+            "rule_metadata": {
+                "status": "test",
+                "author": "Saw Win Naung, Nasreddine Bencherchali (Nextron Systems), Thurein Oo (Yoma Bank)",
+                "date": "2020-02-22",
+                "title": "SQL Injection Strings In URI",
+                "id": "5513deaf-f49a-46c2-a6c8-3f111b5cb453"
+            },
+            "rule_text": "title: SQL Injection Strings In URI\nid: 5513deaf-f49a-46c2-a6c8-3f111b5cb453\nstatus: test\ndescription: Detects potential SQL injection attempts via GET requests in access logs.\nreferences:\n- https://www.acunetix.com/blog/articles/exploiting-sql-injection-example/\n- https://www.acunetix.com/blog/articles/using-logs-to-investigate-a-web-application-attack/\n- https://brightsec.com/blog/sql-injection-payloads/\n- https://github.com/payloadbox/sql-injection-payload-list\n- https://book.hacktricks.xyz/pentesting-web/sql-injection/mysql-injection\nauthor: Saw Win Naung, Nasreddine Bencherchali (Nextron Systems), Thurein Oo (Yoma\n  Bank)\ndate: 2020-02-22\nmodified: 2023-09-04\ntags:\n- attack.initial-access\n- attack.t1190\nlogsource:\n  category: webserver\ndetection:\n  selection:\n    cs-method: GET\n  keywords:\n  - '@@version'\n  - '%271%27%3D%271'\n  - '=select '\n  - =select(\n  - =select%20\n  - concat_ws(\n  - CONCAT(0x\n  - from mysql.innodb_table_stats\n  - from%20mysql.innodb_table_stats\n  - group_concat(\n  - information_schema.tables\n  - json_arrayagg(\n  - or 1=1#\n  - or%201=1#\n  - 'order by '\n  - order%20by%20\n  - 'select * '\n  - select database()\n  - select version()\n  - select%20*%20\n  - select%20database()\n  - select%20version()\n  - select%28sleep%2810%29\n  - SELECTCHAR(\n  - table_schema\n  - UNION ALL SELECT\n  - UNION SELECT\n  - UNION%20ALL%20SELECT\n  - UNION%20SELECT\n  - '''1''=''1'\n  filter_main_status:\n    sc-status: 404\n  condition: selection and keywords and not 1 of filter_main_*\nfalsepositives:\n- Java scripts and CSS Files\n- User searches in search boxes of the respective website\n- Internal vulnerability scanners can cause some serious FPs when used, if you experience\n  a lot of FPs due to this think of adding more filters such as \"User Agent\" strings\n  and more response codes\nlevel: high\n",
+            "vulnerabilities": []
+        }
+    ]
+}

vulnerabilities/tests/test_data/detection_rules/data/suricata/dhcp-events.json

@@ -0,0 +1,25 @@
+{
+    "source_url": "https://github.com/OISF/suricata/blob/master/rules/dhcp-events.rules",
+    "rules": [
+        {
+            "rule_metadata": {
+                "name": "SURICATA DHCP malformed options",
+                "version": 1,
+                "id": 2227000,
+                "enabled": true
+            },
+            "rule_text": "alert dhcp any any -> any any (msg:\"SURICATA DHCP malformed options\"; app-layer-event:dhcp.malformed_options; classtype:protocol-command-decode; sid:2227000; rev:1;)",
+            "vulnerabilities": []
+        },
+        {
+            "rule_metadata": {
+                "name": "SURICATA DHCP truncated options",
+                "version": 1,
+                "id": 2227001,
+                "enabled": true
+            },
+            "rule_text": "alert dhcp any any -> any any (msg:\"SURICATA DHCP truncated options\"; app-layer-event:dhcp.truncated_options; classtype:protocol-command-decode; sid:2227001; rev:1;)",
+            "vulnerabilities": []
+        }
+    ]
+}

vulnerabilities/tests/test_data/detection_rules/data/yara/Linux_Backdoor_Bash.json

@@ -0,0 +1,25 @@
+{
+    "source_url": "https://github.com/elastic/protections-artifacts/blob/master/yara/rules/Linux_Backdoor_Bash.yar",
+    "rules": [
+        {
+            "rule_metadata": {
+                "name": "Linux_Backdoor_Bash_e427876d",
+                "tags": [],
+                "author": "Elastic Security",
+                "id": "e427876d-c7c5-447a-ad6d-5cbc12d9dacf",
+                "fingerprint": "6cc13bb2591d896affc58f4a22b3463a72f6c9d896594fe1714b825e064b0956",
+                "creation_date": "2021-01-12",
+                "last_modified": "2021-09-16",
+                "threat_name": "Linux.Backdoor.Bash",
+                "reference_sample": "07db41a4ddaac802b04df5e5bbae0881fead30cb8f6fa53a8a2e1edf14f2d36b",
+                "severity": 100,
+                "arch_context": "x86",
+                "scan_context": "file, memory",
+                "license": "Elastic License v2",
+                "os": "linux"
+            },
+            "rule_text": "rule Linux_Backdoor_Bash_e427876d\n{\n\tmeta:\n\t\tauthor = \"Elastic Security\"\n\t\tid = \"e427876d-c7c5-447a-ad6d-5cbc12d9dacf\"\n\t\tfingerprint = \"6cc13bb2591d896affc58f4a22b3463a72f6c9d896594fe1714b825e064b0956\"\n\t\tcreation_date = \"2021-01-12\"\n\t\tlast_modified = \"2021-09-16\"\n\t\tthreat_name = \"Linux.Backdoor.Bash\"\n\t\treference_sample = \"07db41a4ddaac802b04df5e5bbae0881fead30cb8f6fa53a8a2e1edf14f2d36b\"\n\t\tseverity = 100\n\t\tarch_context = \"x86\"\n\t\tscan_context = \"file, memory\"\n\t\tlicense = \"Elastic License v2\"\n\t\tos = \"linux\"\n\n\tstrings:\n\t\t$a = { 67 65 44 6F 6B 4B 47 6C 6B 49 43 31 31 4B 54 6F 67 4C 32 56 }\n\n\tcondition:\n\t\tall of them\n}\n",
+            "vulnerabilities": []
+        }
+    ]
+}