feat: package db diff for v6

cmd/grype/cli/commands/db.go

@@ -27,6 +27,7 @@ func DB(app clio.Application) *cobra.Command {
 		DBUpdate(app),
 		DBSearch(app),
 		DBProviders(app),
+		DBDiff(app),
 	)
 
 	return db

cmd/grype/cli/commands/db_diff.go

@@ -0,0 +1,137 @@
+package commands
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/anchore/clio"
+	"github.com/anchore/grype/cmd/grype/cli/options"
+	"github.com/anchore/grype/grype/db/v6/diff"
+	"github.com/anchore/grype/internal/log"
+)
+
+type dbDiffOptions struct {
+	Output                  string `yaml:"output" json:"output" mapstructure:"output"`
+	options.DatabaseCommand `yaml:",inline" mapstructure:",squash"`
+	Old                     string
+	New                     string
+}
+
+var _ clio.FlagAdder = (*dbDiffOptions)(nil)
+
+func (d *dbDiffOptions) AddFlags(flags clio.FlagSet) {
+	flags.StringVarP(&d.Output, "output", "o", "format to display results (available=[text, json])")
+}
+
+func DBDiff(app clio.Application) *cobra.Command {
+	opts := &dbDiffOptions{
+		Output:          textOutputFormat,
+		DatabaseCommand: *options.DefaultDatabaseCommand(app.ID()),
+	}
+
+	cmd := &cobra.Command{
+		Use:   "diff [flags] old_db_url_or_path [new_db_url_or_path]",
+		Short: "Diff two databases, showing packages with added, removed, and modified vulnerability matches",
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			opts.DB.MaxUpdateCheckFrequency = 0
+			return disableUI(app)(cmd, args)
+		},
+		Args: cobra.RangeArgs(1, 2),
+		RunE: func(_ *cobra.Command, args []string) error {
+			opts.Old = args[0]
+			if len(args) > 1 {
+				opts.New = args[1]
+			}
+			return runDBDiff(*opts)
+		},
+	}
+
+	type configWrapper struct {
+		Hidden                   *dbDiffOptions `json:"-" yaml:"-" mapstructure:"-"`
+		*options.DatabaseCommand `yaml:",inline" mapstructure:",squash"`
+	}
+
+	return app.SetupCommand(cmd, &configWrapper{Hidden: opts, DatabaseCommand: &opts.DatabaseCommand})
+}
+
+func runDBDiff(opts dbDiffOptions) error {
+	startTime := time.Now()
+
+	// d, err := diff.NewProviderDiffer(oldResult.dir, newResult.dir)
+	d, err := diff.NewDBDiffer(diff.Config{
+		Config: opts.ToCuratorConfig(),
+		Debug:  opts.Developer.DB.Debug,
+		OldDB:  opts.Old,
+		NewDB:  opts.New,
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to create differ: %w", err)
+	}
+	defer log.CloseAndLogError(d, "differ")
+
+	result, err := d.Diff()
+	if err != nil {
+		return fmt.Errorf("unable to diff databases: %w", err)
+	}
+
+	log.Infof("diff complete in %s", time.Since(startTime))
+
+	totalAdded, totalRemoved, totalModified := 0, 0, 0
+	for _, pkg := range result.Packages {
+		totalAdded += len(pkg.Vulnerabilities.Added)
+		totalRemoved += len(pkg.Vulnerabilities.Removed)
+		totalModified += len(pkg.Vulnerabilities.Modified)
+	}
+
+	log.Infof("diff complete: %d added, %d removed, %d modified",
+		totalAdded, totalRemoved, totalModified)
+
+	slices.SortFunc(result.Packages, func(a, b diff.PackageDiff) int {
+		c := strings.Compare(a.Ecosystem, b.Ecosystem)
+		if c != 0 {
+			return c
+		}
+		return strings.Compare(a.Name, b.Name)
+	})
+
+	writer := os.Stdout
+	if opts.Output == "json" {
+		return outputJSON(writer, result)
+	}
+
+	return outputText(writer, result)
+}
+
+func outputText(writer io.Writer, result *diff.Result) error {
+	columns := []string{"Ecosystem", "Package"}
+
+	t := newTable(writer, columns)
+
+	for _, pkg := range result.Packages {
+		name := pkg.Name
+		if pkg.CPE != "" {
+			name = pkg.CPE
+		}
+		err := t.Append(pkg.Ecosystem, name)
+		if err != nil {
+			return err
+		}
+	}
+	defer log.CloseAndLogError(t, "tablewriter")
+
+	return t.Render()
+}
+
+func outputJSON(writer io.Writer, result *diff.Result) error {
+	enc := json.NewEncoder(writer)
+	enc.SetIndent("", "  ")
+	return enc.Encode(result)
+}

go.mod

@@ -79,6 +79,11 @@ require (
 	gorm.io/gorm v1.31.1
 )
 
+require (
+	github.com/bmatcuk/doublestar/v4 v4.10.0
+	github.com/google/martian v2.1.0+incompatible
+)
+
 require (
 	cel.dev/expr v0.24.0 // indirect
 	cloud.google.com/go v0.123.0 // indirect
@@ -136,7 +141,6 @@ require (
 	github.com/becheran/wildmatch-go v1.0.0 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb // indirect
-	github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect
 	github.com/bodgit/plumbing v1.3.0 // indirect
 	github.com/bodgit/sevenzip v1.6.1 // indirect
 	github.com/bodgit/windows v1.0.1 // indirect

grype/db/v6/diff/db_resolver.go

@@ -0,0 +1,225 @@
+package diff
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/google/martian/log"
+	"github.com/mholt/archives"
+
+	"github.com/anchore/go-homedir"
+	db "github.com/anchore/grype/grype/db/v6"
+)
+
+type ResolvedDB struct {
+	Dir     string
+	Info    DatabaseInfo
+	Cleanup func()
+}
+
+// ResolveDB takes a user-provided value (URL, file path, or directory path) and returns
+// a directory path containing a vulnerability.db file, suitable for passing to NewProviderDiffer.
+// If defaultDir is non-empty and value is empty, defaultDir is used.
+// The returned cleanup function should be called to remove any temporary directories created.
+//
+// Supported inputs:
+//   - empty string: falls back to defaultDir
+//   - URL (http/https): downloads the DB file to a temporary directory
+//   - directory path: used as-is (must contain vulnerability.db)
+//   - path to a .db file: uses the file's parent directory
+//   - path to a DB archive (.tar.zst, .tar.gz, etc.): extracts and hydrates into a sibling
+//     directory derived from the archive name (with .tar.* extension stripped)
+func ResolveDB(value, defaultDir string) (ResolvedDB, error) {
+	if value == "" {
+		if defaultDir == "" {
+			return ResolvedDB{}, fmt.Errorf("no database path or URL provided")
+		}
+		log.Infof("using default database directory: %s", defaultDir)
+		return newResolvedDB(defaultDir, nil)
+	}
+
+	if isURL(value) {
+		log.Infof("downloading database from: %s", value)
+		dir, err := downloadDB(value)
+		if err != nil {
+			return ResolvedDB{}, err
+		}
+		cleanup := func() { _ = os.RemoveAll(dir) }
+
+		return newResolvedDB(dir, cleanup)
+	}
+
+	expanded, err := homedir.Expand(value)
+	if err != nil {
+		return ResolvedDB{}, fmt.Errorf("unable to expand path %q: %w", value, err)
+	}
+
+	info, err := os.Stat(expanded)
+	if err != nil {
+		return ResolvedDB{}, fmt.Errorf("unable to stat %q: %w", expanded, err)
+	}
+
+	// case 1: it's already a directory containing a vulnerability.db
+	if info.IsDir() {
+		log.Infof("using database directory: %s", expanded)
+		return newResolvedDB(expanded, nil)
+	}
+
+	// case 2: it's a raw vulnerability.db file; use its parent directory
+	if strings.HasSuffix(expanded, ".db") {
+		dir := filepath.Dir(expanded)
+		log.Infof("using database file: %s", expanded)
+		return newResolvedDB(dir, nil)
+	}
+
+	// case 3: it's a DB archive; extract and hydrate into a sibling directory
+	return extractAndHydrateArchive(expanded)
+}
+
+func newResolvedDB(dir string, cleanup func()) (ResolvedDB, error) {
+	info, err := newDatabaseInfo(dir)
+	if err != nil {
+		return ResolvedDB{}, fmt.Errorf("failed to get old database info: %w", err)
+	}
+
+	return ResolvedDB{
+		Dir:     dir,
+		Info:    *info,
+		Cleanup: cleanup,
+	}, nil
+}
+
+var tarExtPattern = regexp.MustCompile(`\.tar(\.\w+)?$`)
+
+// extractAndHydrateArchive extracts a DB archive into a sibling directory (archive name with
+// .tar.* extension stripped), then runs hydration to create indexes. If the directory already
+// exists with a vulnerability.db inside, it is reused
+func extractAndHydrateArchive(archivePath string) (ResolvedDB, error) {
+	destDir := tarExtPattern.ReplaceAllString(archivePath, "")
+	if destDir == archivePath {
+		return ResolvedDB{}, fmt.Errorf("unrecognized file type (not a .db file or supported archive): %s", archivePath)
+	}
+
+	// check if already extracted
+	dbFilePath := filepath.Join(destDir, db.VulnerabilityDBFileName)
+	if _, err := os.Stat(dbFilePath); err == nil {
+		log.Infof("using previously extracted database: %s", destDir)
+	} else {
+		log.Infof("extracting database archive: %s", archivePath)
+
+		if err := os.MkdirAll(destDir, 0o700); err != nil {
+			return ResolvedDB{}, fmt.Errorf("failed to create extraction directory: %w", err)
+		}
+
+		if err := unarchiveDB(archivePath, destDir); err != nil {
+			_ = os.RemoveAll(destDir)
+			return ResolvedDB{}, fmt.Errorf("failed to extract archive %q: %w", archivePath, err)
+		}
+	}
+
+	log.Infof("hydrating database: %s", destDir)
+
+	hydrate := db.Hydrater()
+	if err := hydrate(destDir); err != nil {
+		_ = os.RemoveAll(destDir)
+		return ResolvedDB{}, fmt.Errorf("failed to hydrate database: %w", err)
+	}
+
+	return newResolvedDB(destDir, nil)
+}
+
+// unarchiveDB extracts a DB archive to the given destination directory.
+func unarchiveDB(source, destination string) error {
+	sourceFile, err := os.Open(source)
+	if err != nil {
+		return err
+	}
+	defer sourceFile.Close()
+
+	format, stream, err := archives.Identify(context.Background(), source, sourceFile)
+	if err != nil {
+		return fmt.Errorf("unable to identify archive format: %w", err)
+	}
+
+	extractor, ok := format.(archives.Extractor)
+	if !ok {
+		return fmt.Errorf("unable to extract DB file, format not supported: %s", source)
+	}
+
+	root, err := os.OpenRoot(destination)
+	if err != nil {
+		return err
+	}
+
+	visitor := func(_ context.Context, file archives.FileInfo) error {
+		if file.IsDir() || file.LinkTarget != "" {
+			return nil
+		}
+
+		fileReader, err := file.Open()
+		if err != nil {
+			return err
+		}
+		defer fileReader.Close()
+
+		filename := filepath.Clean(file.NameInArchive)
+
+		outputFile, err := root.Create(filename)
+		if err != nil {
+			return err
+		}
+		defer outputFile.Close()
+
+		_, err = io.Copy(outputFile, fileReader)
+		return err
+	}
+
+	return extractor.Extract(context.Background(), stream, visitor)
+}
+
+func isURL(value string) bool {
+	return strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://")
+}
+
+// downloadDB downloads a vulnerability database from a URL to a temporary directory
+// and returns the directory path. The caller is responsible for cleaning up the directory.
+func downloadDB(url string) (string, error) {
+	tmpDir, err := os.MkdirTemp("", "grype-db-diff-*")
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp directory: %w", err)
+	}
+
+	dbPath := filepath.Join(tmpDir, db.VulnerabilityDBFileName)
+
+	resp, err := http.Get(url) //nolint:gosec
+	if err != nil {
+		_ = os.RemoveAll(tmpDir)
+		return "", fmt.Errorf("failed to download from %s: %w", url, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		_ = os.RemoveAll(tmpDir)
+		return "", fmt.Errorf("failed to download from %s: bad status %s", url, resp.Status)
+	}
+
+	out, err := os.Create(dbPath)
+	if err != nil {
+		_ = os.RemoveAll(tmpDir)
+		return "", fmt.Errorf("failed to create file: %w", err)
+	}
+	defer out.Close()
+
+	if _, err := io.Copy(out, resp.Body); err != nil {
+		_ = os.RemoveAll(tmpDir)
+		return "", fmt.Errorf("failed to write database file: %w", err)
+	}
+
+	return tmpDir, nil
+}