Implement IEP009 - product & vuln. identification (#2574)

* Implement IEP009 - product & vuln. identification

The related IEP has already been discussed and
open for more than a year.

* Fix order

* Upgrade harmonisation

* Add tests

* Renumbering to 3.4.2 release

* Fix style issue

Author: kamil-certat

CHANGELOG.md

@@ -27,6 +27,18 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o
 
 ### Data Format
 
+- Implementing [IEP009](https://github.com/certtools/ieps/tree/main/009) introducing fields to
+  identify products and vulnerabilities: `product.full_name`, `product.name`, `product.vendor`,
+  `product.version`, `product.vulnerabilities`. To store in existing PostgreSQL instances, a following
+  schema update will be necessary:
+  ```sql
+  ALTER TABLE events ADD "product.full_name" text;
+  ALTER TABLE events ADD "product.name" text;
+  ALTER TABLE events ADD "product.vendor" text;
+  ALTER TABLE events ADD "product.version" text;
+  ALTER TABLE events ADD "product.vulnerabilities" text;
+  ```
+
 ### Bots
 #### Collectors
 - `intelmq.bots.collectors.mail.collector_mail_attach`: Decrypt GPG attachments (PR#2623 by Edvard Rejthar).

NEWS.md

@@ -19,6 +19,15 @@ Python `>=3.9` is now required, which is available on all platforms supported by
 ### Tools
 
 ### Data Format
+To save new fields from IntelMQ Data Format in existing PostgreSQL instances, the following schema
+update is necessary:
+```sql
+ALTER TABLE events ADD "product.full_name" text;
+ALTER TABLE events ADD "product.name" text;
+ALTER TABLE events ADD "product.vendor" text;
+ALTER TABLE events ADD "product.version" text;
+ALTER TABLE events ADD "product.vulnerabilities" text;
+```
 
 ### Configuration
 

intelmq/etc/harmonization.conf

@@ -209,6 +209,26 @@
             "description": "Event data converted into foreign format, intended to be exported by output plugin.",
             "type": "JSON"
         },
+        "product.full_name": {
+            "description": "A human readable product name. If a machine-readable format isn't available, this field should be used. It can directly use the version identification strings presented by the product. If not given, a good enough value can usually be constructed by concatenating product.product and product.version, or by consulting external sources such as the CPE Product Dictionary. Example: openssh_/8.9",
+            "type": "String"
+        },
+        "product.name": {
+            "description": "Product name, recommended being as the product in the CPE format. Example: openssh",
+            "type": "LowercaseString"
+        },
+        "product.vendor": {
+            "description": "Vendor name, recommended being as vendor in the CPE format. Example: openbsd",
+            "type": "LowercaseString"
+        },
+        "product.version": {
+            "description": "Product version, recommended being as version in the CPE format. Example: 8.9",
+            "type": "LowercaseString"
+        },
+        "product.vulnerabilities": {
+            "description": "List of vulnerability IDs, separated by semicolons. It's recommended to use a CVE ID where available, and other easily retrievable IDs in other cases, e.g. Github Advisory Database ID. Each vulnerability should only be listed once, and multiple values should be used if there are several different vulnerabilities. However, it's not necessary for a source to list all possible vulnerabilities for a given piece of software. Example: cve-2023-38408;cve-2023-28531;cve-2008-3844;cve-2007-2768",
+            "type": "LowercaseString"
+        },
         "protocol.application": {
             "description": "e.g. vnc, ssh, sip, irc, http or smtp.",
             "length": 100,

intelmq/lib/upgrades.py

@@ -43,6 +43,7 @@
            'v322_removed_feeds_and_bots',
            'v340_deprecations',
            'v341_blueliv_removal',
+           'v342_new_fields'
            ]
 
 
@@ -998,6 +999,33 @@ def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs):
     return message, configuration, harmonization
 
 
+def v342_new_fields(configuration, harmonization, dry_run, **kwargs):
+    """
+    Add new fields to IntelMQ Data Format
+    """
+    changed = None
+    if "event" not in harmonization:
+        return changed, configuration, harmonization
+
+    builtin_harmonisation = load_configuration(
+        resource_filename("intelmq", "etc/harmonization.conf")
+    )
+    for field in [
+        "product.full_name",
+        "product.name",
+        "product.vendor",
+        "product.version",
+        "product.vulnerabilities",
+    ]:
+        if field not in harmonization["event"]:
+            if field not in builtin_harmonisation["event"]:
+                # ensure forward-compatibility if we ever remove something from harmonisation
+                continue
+            harmonization["event"][field] = builtin_harmonisation["event"][field]
+            changed = True
+    return changed, configuration, harmonization
+
+
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax,)),
     ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -1029,6 +1057,7 @@ def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs):
     ((3, 3, 1), ()),
     ((3, 4, 0), (v340_deprecations, )),
     ((3, 4, 1), (v341_blueliv_removal, )),
+    ((3, 4, 2), (v342_new_fields, )),
 ])
 
 ALWAYS = (harmonization,)

intelmq/tests/bin/initdb.sql

@@ -47,6 +47,11 @@ CREATE TABLE events (
     "misp.attribute_uuid" varchar(36),
     "misp.event_uuid" varchar(36),
     "output" jsonb,
+    "product.full_name" text,
+    "product.name" text,
+    "product.vendor" text,
+    "product.version" text,
+    "product.vulnerabilities" text,
     "protocol.application" varchar(100),
     "protocol.transport" varchar(11),
     "raw" text,

intelmq/tests/lib/test_upgrades.py

@@ -872,6 +872,17 @@ def test_v341_blueliv_removal(self):
         self.assertIn('blueliv-parser', result[0])
         self.assertEqual(V341_BLUELIV_REMOVAL, result[1])
 
+    def test_v342_new_fields(self):
+        """ Test adding new harmonisation fields """
+        result = upgrades.v342_new_fields({}, {"event": {"old-field": "must stay"}}, False)
+        self.assertTrue(result[0])
+        self.assertIn("old-field", result[2]["event"])
+        self.assertIn("product.full_name", result[2]["event"])
+        self.assertIn("product.name", result[2]["event"])
+        self.assertIn("product.vendor", result[2]["event"])
+        self.assertIn("product.version", result[2]["event"])
+        self.assertIn("product.vulnerabilities", result[2]["event"])
+
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,
             generate_function(getattr(upgrades, name)))