feat: detect advisory conflicts at package alias intersections and compute partially curated advisories

vulnerabilities/importers/__init__.py

@@ -32,6 +32,7 @@
 from vulnerabilities.importers import ubuntu_usn
 from vulnerabilities.importers import vulnrichment
 from vulnerabilities.importers import xen
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
 from vulnerabilities.pipelines import alpine_linux_importer
 from vulnerabilities.pipelines import github_importer
 from vulnerabilities.pipelines import gitlab_importer
@@ -189,3 +190,9 @@
         collect_fix_commits_v2.CollectGitlabFixCommitsPipeline,
     ]
 )
+
+TODO_EXCLUDED_PIPELINES = [
+    key
+    for key, value in IMPORTERS_REGISTRY.items()
+    if issubclass(value, VulnerableCodeBaseImporterPipelineV2) and value.exclude_from_package_todo
+]

vulnerabilities/improvers/__init__.py

@@ -19,6 +19,7 @@
 from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
 from vulnerabilities.pipelines import remove_duplicate_advisories
 from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees
+from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2
 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2
 from vulnerabilities.pipelines.v2_improvers import (
     computer_package_version_rank as compute_version_rank_v2,
@@ -72,5 +73,6 @@
         collect_ssvc_trees.CollectSSVCPipeline,
         relate_severities.RelateSeveritiesPipeline,
         group_advisories_for_packages.GroupAdvisoriesForPackages,
+        compute_advisory_todo_v2.ComputeToDo,
     ]
 )

vulnerabilities/models.py

@@ -2951,6 +2951,12 @@ def latest_advisories_for_purl(self, purl):
         qs = self.filter(id__in=Subquery(adv_ids))
         return qs.latest_per_avid()
 
+    def todo_excluded(self):
+        """Exclude advisory ineligible for ToDo computation."""
+        from vulnerabilities.importers import TODO_EXCLUDED_PIPELINES
+
+        return self.exclude(datasource_id__in=TODO_EXCLUDED_PIPELINES)
+
 
 class AdvisorySet(models.Model):
 

vulnerabilities/pipelines/__init__.py

@@ -278,6 +278,11 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
     ignorable_versions = []
     precedence = 0
 
+    # Set this to True if computing fixed/affected package ToDo is not fruitful for this source.
+    # An example of such advisory would be pipeline dedicated to collecting issues,
+    # pull requests, commit messages, EPSS, exploits, etc.
+    exclude_from_package_todo = False
+
     # Control how often progress log is shown (range: 1–100, higher value = less frequent log)
     progress_step = 10
 

vulnerabilities/pipelines/v2_importers/__init__.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#

vulnerabilities/pipelines/v2_importers/aosp_importer.py

@@ -32,6 +32,7 @@ class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     license_url = "https://github.com/quarkslab/aosp_dataset/blob/master/LICENSE"
 
     precedence = 200
+    exclude_from_package_todo = True
 
     @classmethod
     def steps(cls):

vulnerabilities/pipelines/v2_importers/epss_importer_v2.py

@@ -30,6 +30,8 @@ class EPSSImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     spdx_license_expression = "unknown"
     importer_name = "EPSS Importer"
 
+    exclude_from_package_todo = True
+
     precedence = 200
 
     def advisories_count(self):

vulnerabilities/pipelines/v2_importers/nvd_importer.py

@@ -71,6 +71,8 @@ class NVDImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
         MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
     """
 
+    exclude_from_package_todo = True
+
     precedence = 100
 
     @classmethod

vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py

@@ -30,6 +30,8 @@ class ProjectKBMSR2019Pipeline(VulnerableCodeBaseImporterPipelineV2):
     license_url = "https://github.com/SAP/project-kb/blob/main/LICENSE.txt"
     repo_url = "git+https://github.com/SAP/project-kb"
 
+    exclude_from_package_todo = True
+
     precedence = 200
 
     @classmethod

vulnerabilities/pipelines/v2_importers/project_kb_statements_importer.py

@@ -37,6 +37,8 @@ class ProjectKBStatementsPipeline(VulnerableCodeBaseImporterPipelineV2):
     license_url = "https://github.com/SAP/project-kb/blob/main/LICENSE.txt"
     repo_url = "git+https://github.com/SAP/project-kb@vulnerability-data"
 
+    exclude_from_package_todo = True
+
     precedence = 200
 
     @classmethod

vulnerabilities/pipelines/v2_importers/suse_score_importer.py

@@ -23,6 +23,8 @@ class SUSESeverityScoreImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     pipeline_id = "suse_importer_v2"
     url = "https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml"
 
+    exclude_from_package_todo = True
+
     @classmethod
     def steps(cls):
         return (