Wire Kustomize activation through scan and CLI paths with dedupe and lint-safe scan pipeline refactors.

Author: whitemerch

cmd/scanner/scan.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/DataDog/datadog-iac-scanner/internal/console"
 	"github.com/DataDog/datadog-iac-scanner/pkg/config"
@@ -74,10 +75,33 @@ var scanAction = &cli.Command{
 			Usage:   "a list of platform types to scan",
 			Value:   GetSupportedPlatforms(),
 		},
-		// NOTE: --x-parallelparsing flag disabled due to pre-existing race conditions
-		// in concurrent query workers (SetupLogs shared state write, docker detector
-		// shallow slice copy) that cause non-deterministic violation counts.
-		// See K9VULN-13746 for the follow-up to fix and re-enable.
+		&cli.BoolFlag{
+			Name:  "helm-include-crds",
+			Usage: "when rendering Helm charts, include CRDs in the manifest output",
+			Value: true,
+		},
+		&cli.BoolFlag{
+			Name:  "kustomize-enable-helm-inflation",
+			Usage: "when building kustomizations, pre-render helmCharts entries with the Helm SDK",
+			Value: true,
+		},
+		&cli.DurationFlag{
+			Name:  "kustomize-render-timeout",
+			Usage: "maximum time allowed for a single kustomize build",
+			Value: 60 * time.Second,
+		},
+		&cli.IntFlag{
+			Name:  "kustomize-max-fetch-mib",
+			Usage: "soft cap for scratch data during kustomize resolve (staging, Helm prepass; best-effort)",
+			Value: 128,
+		},
+		&cli.BoolFlag{
+			Name: "kustomize-strict-loads",
+			Usage: "use kustomize RootOnly load restrictions (safer; may reject some valid ../ references " +
+				"like secretGenerator.envs outside the kustomization root)",
+			Value: false,
+		},
+		// NOTE: --x-parallelparsing left out: parallel query workers still race → flaky counts.
 		// &cli.BoolFlag{
 		// 	Name:   "x-parallelparsing",
 		// 	Hidden: true,
@@ -93,6 +117,48 @@ const (
 	dirPerms  = 0755
 )
 
+func scanParametersFromCLI(
+	c *cli.Command,
+	repoDir string,
+	repoInfo *model.RepositoryCommitInfo,
+	inputPaths []string,
+	outputPath, payloadPath string,
+	cfg *config.IacConfig,
+) *scan.Parameters {
+	maxFetchMib := c.Int("kustomize-max-fetch-mib")
+	if maxFetchMib <= 0 {
+		maxFetchMib = 128
+	}
+	return &scan.Parameters{
+		CloudProvider:                []string{""},
+		OutputPath:                   outputPath,
+		OutputName:                   c.String("output-name"),
+		PreviewLines:                 3,
+		RepoPath:                     repoDir,
+		Path:                         inputPaths,
+		QueriesPath:                  []string{"./assets/queries"},
+		LibrariesPath:                "./assets/libraries",
+		ReportFormats:                []string{"sarif"},
+		Platform:                     selectPlatforms(c.StringSlice("type")),
+		QueryExecTimeout:             c.Int("timeout"),
+		DisableSecrets:               true,
+		ScanID:                       "console",
+		MaxFileSizeFlag:              c.Int("max-file-size"),
+		MaxResolverDepth:             c.Int("max-resolver-depth"),
+		ExcludePlatform:              []string{""},
+		PayloadPath:                  payloadPath,
+		SCIInfo:                      model.SCIInfo{RepositoryDir: repoDir, RepositoryCommitInfo: *repoInfo},
+		FlagEvaluator:                getFeatureFlagEvaluator(c),
+		Config:                       *cfg,
+		DownloadQueriesFromDatadog:   c.Bool("x-downloadqueriesfromdatadog"),
+		HelmIncludeCRDs:              c.Bool("helm-include-crds"),
+		KustomizeEnableHelmInflation: c.Bool("kustomize-enable-helm-inflation"),
+		KustomizeRenderTimeout:       c.Duration("kustomize-render-timeout"),
+		KustomizeMaxFetchBytes:       int64(maxFetchMib) * 1024 * 1024,
+		KustomizeStrictLoad:          c.Bool("kustomize-strict-loads"),
+	}
+}
+
 func runScan(ctx context.Context, c *cli.Command) error {
 	if c.Args().Len() > 0 {
 		return fmt.Errorf("unexpected arguments: %v", c.Args().Slice())
@@ -141,29 +207,7 @@ func runScan(ctx context.Context, c *cli.Command) error {
 	}
 	cfg.OnlyPaths = onlyPaths
 	cfg.IgnoreRules = append(c.StringSlice("exclude-queries"), cfg.IgnoreRules...)
-	params := &scan.Parameters{
-		CloudProvider:              []string{""},
-		OutputPath:                 outputPath,
-		OutputName:                 c.String("output-name"),
-		PreviewLines:               3,
-		RepoPath:                   repoDir,
-		Path:                       inputPaths,
-		QueriesPath:                []string{"./assets/queries"},
-		LibrariesPath:              "./assets/libraries",
-		ReportFormats:              []string{"sarif"},
-		Platform:                   selectPlatforms(c.StringSlice("type")),
-		QueryExecTimeout:           c.Int("timeout"),
-		DisableSecrets:             true,
-		ScanID:                     "console",
-		MaxFileSizeFlag:            c.Int("max-file-size"),
-		MaxResolverDepth:           c.Int("max-resolver-depth"),
-		ExcludePlatform:            []string{""},
-		PayloadPath:                payloadPath,
-		SCIInfo:                    model.SCIInfo{RepositoryDir: repoDir, RepositoryCommitInfo: *repoInfo},
-		FlagEvaluator:              getFeatureFlagEvaluator(c),
-		Config:                     *cfg,
-		DownloadQueriesFromDatadog: c.Bool("x-downloadqueriesfromdatadog"),
-	}
+	params := scanParametersFromCLI(c, repoDir, repoInfo, inputPaths, outputPath, payloadPath, cfg)
 
 	metadata, err := console.ExecuteScan(ctx, params)
 	if err != nil {
@@ -366,8 +410,7 @@ func selectPlatforms(platforms []string) []string {
 
 func getFeatureFlagEvaluator(_ *cli.Command) featureflags.FlagEvaluator {
 	overrides := map[string]bool{}
-	// Parallel parsing disabled: triggers race conditions in concurrent
-	// query workers causing non-deterministic violation counts.
+	// Parallel parsing off until worker races are fixed (non-deterministic counts).
 	// overrides[featureflags.IaCEnableKicsParallelFileParsing] = c.Bool("x-parallelparsing")
 	return featureflags.NewLocalEvaluatorWithOverrides(overrides)
 }

pkg/analyzer/analyzer.go

@@ -19,6 +19,7 @@ import (
 	"github.com/DataDog/datadog-iac-scanner/pkg/engine/provider"
 	"github.com/DataDog/datadog-iac-scanner/pkg/logger"
 	"github.com/DataDog/datadog-iac-scanner/pkg/model"
+	"github.com/DataDog/datadog-iac-scanner/pkg/resolver/kustomize"
 	"github.com/DataDog/datadog-iac-scanner/pkg/utils"
 	"github.com/pkg/errors"
 	ignore "github.com/sabhiram/go-gitignore"
@@ -507,12 +508,26 @@ func needsOverride(check bool, returnType, key, ext string) bool {
 	return false
 }
 
+func (a *analyzerInfo) tryKustomizeEntry(results, unwanted chan<- string) bool {
+	if !kustomize.IsKustomizationEntryFile(a.filePath) {
+		return false
+	}
+	if a.isAvailableType(kubernetes) && checkKustomize(a.filePath) {
+		results <- kubernetes
+	}
+	unwanted <- a.filePath
+	return true
+}
+
 // checkContent will determine the file type by content when worker was unable to
 // determine by ext, if no type was determined checkContent adds it to unwanted channel
 func (a *analyzerInfo) checkContent(ctx context.Context, results, unwanted chan<- string, locCount chan<- int, linesCount int, ext string) {
 	contextLogger := logger.FromContext(ctx)
 	typesFlag := a.typesFlag
 	excludeTypesFlag := a.excludeTypesFlag
+	if a.tryKustomizeEntry(results, unwanted) {
+		return
+	}
 	// get file content
 	content, err := os.ReadFile(a.filePath)
 	if err != nil {
@@ -564,6 +579,9 @@ func (a *analyzerInfo) checkContent(ctx context.Context, results, unwanted chan<
 }
 
 func checkReturnType(ctx context.Context, path, returnType, ext string, content []byte) string {
+	if kustomize.IsKustomizationEntryFile(path) {
+		return ""
+	}
 	if returnType != "" {
 		switch returnType {
 		case "cdkTf":
@@ -578,6 +596,9 @@ func checkReturnType(ctx context.Context, path, returnType, ext string, content
 		if checkHelm(ctx, path) {
 			return kubernetes
 		}
+		if checkKustomize(path) {
+			return kubernetes
+		}
 		platform := checkYamlPlatform(ctx, content, path)
 		if platform != "" {
 			return platform
@@ -598,6 +619,14 @@ func checkHelm(ctx context.Context, path string) bool {
 	return true
 }
 
+func checkKustomize(path string) bool {
+	dir := filepath.Dir(path)
+	if _, ok := kustomize.Detect(dir); ok {
+		return true
+	}
+	return false
+}
+
 func checkYamlPlatform(ctx context.Context, content []byte, path string) string {
 	// Ansible 'templates/' directories contain Jinja2 files; {{ }} syntax is invalid YAML.
 	if isInsideAnsibleTemplatesDir(path) {

pkg/analyzer/analyzer_test.go

@@ -212,6 +212,36 @@ func TestAnalyzer_Analyze(t *testing.T) {
 			excludeGitIgnore:     false,
 			MaxFileSize:          -1,
 		},
+		{
+			name: "analyze_test_explicit_kustomization_file_enables_kustomize_without_raw_parse",
+			paths: []string{
+				filepath.FromSlash("../../test/fixtures/test_kustomize/canonical/simple/kustomization.yaml"),
+			},
+			wantTypes:            []string{"kubernetes"},
+			wantExclude:          []string{filepath.FromSlash("../../test/fixtures/test_kustomize/canonical/simple/kustomization.yaml")},
+			typesFromFlag:        []string{""},
+			excludeTypesFromFlag: []string{""},
+			wantLOC:              0,
+			wantErr:              false,
+			gitIgnoreFileName:    "",
+			excludeGitIgnore:     false,
+			MaxFileSize:          -1,
+		},
+		{
+			name: "analyze_test_overlay_directory_with_only_kustomization_still_enables_kubernetes",
+			paths: []string{
+				filepath.FromSlash("../../test/fixtures/test_kustomize/regressions/namespace_overlay/overlay"),
+			},
+			wantTypes:            []string{"kubernetes"},
+			wantExclude:          []string{filepath.FromSlash("../../test/fixtures/test_kustomize/regressions/namespace_overlay/overlay/kustomization.yaml")},
+			typesFromFlag:        []string{""},
+			excludeTypesFromFlag: []string{""},
+			wantLOC:              0,
+			wantErr:              false,
+			gitIgnoreFileName:    "",
+			excludeGitIgnore:     false,
+			MaxFileSize:          -1,
+		},
 		{
 			name:      "analyze_test_dir_single_path_types_value",
 			paths:     []string{filepath.FromSlash("../../test/fixtures/analyzer_test")},