Merge pull request #24 from boyter/hashignore

working on hashignore support and other exclusions

Author: boyter

go.mod

@@ -3,6 +3,7 @@ module github.com/boyter/hashit
 go 1.25.0
 
 require (
+	github.com/boyter/gocodewalker v1.5.1
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/djherbis/times v1.6.0
 	github.com/gosuri/uiprogress v0.0.1
@@ -15,6 +16,7 @@ require (
 )
 
 require (
+	github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gosuri/uilive v0.0.4 // indirect
@@ -25,6 +27,7 @@ require (
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
+	golang.org/x/sync v0.16.0 // indirect
 	golang.org/x/sys v0.36.0 // indirect
 	modernc.org/libc v1.66.10 // indirect
 	modernc.org/mathutil v1.7.1 // indirect

go.sum

@@ -1,6 +1,10 @@
+github.com/boyter/gocodewalker v1.5.1 h1:0YeK2QAkd+ymW5MsagMZapIXD3v9/vrZl0HkFSLpKsw=
+github.com/boyter/gocodewalker v1.5.1/go.mod h1:9k+yM6+fIx61F0xI9ChXEGE5DYoLhggw8AxSOtW+kKo=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
 github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=

main.go

@@ -145,6 +145,44 @@ func main() {
 		"input file of newline seperated file locations to process",
 	)
 
+	flags.BoolVar(
+		&processor.GitIgnore,
+		"gitignore",
+		false,
+		"enable .gitignore file logic",
+	)
+	flags.BoolVar(
+		&processor.GitModuleIgnore,
+		"gitmodule",
+		false,
+		"enable .gitmodules file logic",
+	)
+	flags.StringSliceVar(
+		&processor.PathDenyList,
+		"exclude-dir",
+		[]string{},
+		"directories to exclude",
+	)
+	flags.BoolVar(
+		&processor.Ignore,
+		"ignore",
+		false,
+		"enable .ignore file logic",
+	)
+	flags.BoolVar(
+		&processor.HashIgnore,
+		"hashignore",
+		false,
+		"enable .hashignore file logic",
+	)
+	flags.StringArrayVarP(
+		&processor.Exclude,
+		"not-match",
+		`M`,
+		[]string{},
+		"ignore files and directories matching regular expression",
+	)
+
 	if err := rootCmd.Execute(); err != nil {
 		os.Exit(1)
 	}

processor/file.go

@@ -4,8 +4,10 @@ package processor
 
 import (
 	"fmt"
+	"github.com/boyter/gocodewalker"
 	"os"
 	"path/filepath"
+	"regexp"
 )
 
 func walkDirectory(toWalk string, output chan string) {
@@ -28,3 +30,53 @@ func walkDirectory(toWalk string, output chan string) {
 		}
 	}
 }
+
+func walkDirectoryWithIgnore(toWalk string, output chan string) {
+	fileListQueue := make(chan *gocodewalker.File, 1000)
+	var fileWalker *gocodewalker.FileWalker
+	if NoThreads != 1 {
+		fileWalker = gocodewalker.NewParallelFileWalker([]string{toWalk}, fileListQueue)
+	} else {
+		fileWalker = gocodewalker.NewFileWalker(toWalk, fileListQueue)
+	}
+
+	// The user flags are to enable processing, while gocodewalker is to disable
+	// so we need to invert the values.
+	fileWalker.IgnoreGitIgnore = !GitIgnore
+	fileWalker.IgnoreIgnoreFile = !Ignore
+	fileWalker.IgnoreGitModules = !GitModuleIgnore
+	fileWalker.IncludeHidden = true
+	fileWalker.ExcludeDirectory = PathDenyList
+
+	if HashIgnore {
+		fileWalker.CustomIgnore = []string{".hashignore"}
+	}
+
+	// handle the errors by printing them out and then ignore
+	errorHandler := func(err error) bool {
+		printError(err.Error())
+		return true
+	}
+	fileWalker.SetErrorHandler(errorHandler)
+
+	for _, exclude := range Exclude {
+		regexpResult, err := regexp.Compile(exclude)
+		if err == nil {
+			fileWalker.ExcludeFilenameRegex = append(fileWalker.ExcludeFilenameRegex, regexpResult)
+			fileWalker.ExcludeDirectoryRegex = append(fileWalker.ExcludeDirectoryRegex, regexpResult)
+		} else {
+			printError(err.Error())
+		}
+	}
+
+	go func() {
+		err := fileWalker.Start()
+		if err != nil {
+			printError(err.Error())
+		}
+	}()
+
+	for f := range fileListQueue {
+		output <- f.Location
+	}
+}

processor/processor.go

@@ -71,6 +71,24 @@ var StreamSize int64 = 1_000_000
 // FileInput indicates we have a file passed in which consists of a
 var FileInput = ""
 
+// GitIgnore set false to enable .gitignore checks
+var GitIgnore = true
+
+// GitModuleIgnore set false to enable .gitmodules checks
+var GitModuleIgnore = true
+
+// Ignore set false to enable ignore file checks
+var Ignore = true
+
+// HashIgnore set true to enable hashignore file checks
+var HashIgnore = false
+
+// PathDenyList sets the paths that should be skipped
+var PathDenyList = []string{}
+
+// Exclude is a regular expression which is used to exclude files from being processed
+var Exclude = []string{}
+
 var NoThreads = runtime.NumCPU()
 
 // String mapping for hash names
@@ -156,13 +174,12 @@ func Process() {
 					} else {
 						if fi.IsDir() {
 							if Recursive {
-								walkDirectory(fp, fileListQueue)
+								walkDirectoryWithIgnore(fp, fileListQueue)
 							}
 						} else {
 							fileListQueue <- fp
 						}
 					}
-
 				}
 				close(fileListQueue)
 			}()