Merge pull request #141 from Keyruu/main

feat: add submodule support

Author: nlewo

cmd/run.go

@@ -109,7 +109,7 @@ var runCmd = &cobra.Command{
 		sched := scheduler.New()
 		sched.FetchRemotes(fetcher, cfg.Remotes)
 
-		builder := builder.New(store, executor, gitConfig.Path, gitConfig.Dir, cfg.SystemAttr, cfg.Hostname, 30*time.Minute, 30*time.Minute)
+		builder := builder.New(store, executor, gitConfig.Path, gitConfig.Dir, cfg.SystemAttr, cfg.Hostname, gitConfig.Submodules, 30*time.Minute, 30*time.Minute)
 		deployer := deployer.New(store, executor.Deploy, lastDeployment, cfg.PostDeploymentCommand)
 
 		mode, err := manager.ParseMode(cfg.BuildConfirmer.Mode)

docs/generated-module-options.md

@@ -801,6 +801,28 @@ one of “flake”, “nix”
 
 
 
+## services\.comin\.submodules
+
+
+
+Whether to fetch and include Git submodules when cloning the repository\.
+When enabled, this adds ?submodules=1 to the flake URL\.
+
+
+
+*Type:*
+boolean
+
+
+
+*Default:*
+
+```nix
+false
+```
+
+
+
 ## services\.comin\.systemAttr
 
 

internal/builder/builder.go

@@ -35,6 +35,7 @@ type Builder struct {
 	repositoryPath string
 	repositoryDir  string
 	systemAttr     string
+	submodules     bool
 	evalTimeout    time.Duration
 	buildTimeout   time.Duration
 
@@ -62,15 +63,16 @@ type Builder struct {
 	isSuspended bool
 }
 
-func New(store *store.Store, executor executor.Executor, repositoryPath, repositoryDir, systemAttr, hostname string, evalTimeout time.Duration, buildTimeout time.Duration) *Builder {
-	logrus.Infof("builder: initialization with repositoryPath=%s, repositoryDir=%s, systemAttr=%s, hostname=%s, evalTimeout=%fs, buildTimeout=%fs, )",
-		repositoryPath, repositoryDir, systemAttr, hostname, evalTimeout.Seconds(), buildTimeout.Seconds())
+func New(store *store.Store, executor executor.Executor, repositoryPath, repositoryDir, systemAttr, hostname string, submodules bool, evalTimeout time.Duration, buildTimeout time.Duration) *Builder {
+	logrus.Infof("builder: initialization with repositoryPath=%s, repositoryDir=%s, systemAttr=%s, hostname=%s, submodules=%v, evalTimeout=%fs, buildTimeout=%fs, )",
+		repositoryPath, repositoryDir, systemAttr, hostname, submodules, evalTimeout.Seconds(), buildTimeout.Seconds())
 	return &Builder{
 		store:          store,
 		executor:       executor,
 		repositoryPath: repositoryPath,
 		repositoryDir:  repositoryDir,
 		systemAttr:     systemAttr,
+		submodules:     submodules,
 		hostname:       hostname,
 		evalTimeout:    evalTimeout,
 		buildTimeout:   buildTimeout,
@@ -146,6 +148,7 @@ type Evaluator struct {
 	systemAttr      string
 	commitId        string
 	hostname        string
+	submodules      bool
 
 	evalFunc executor.EvalFunc
 
@@ -155,7 +158,7 @@ type Evaluator struct {
 }
 
 func (r *Evaluator) Run(ctx context.Context) (err error) {
-	r.drvPath, r.outPath, r.machineId, err = r.evalFunc(ctx, r.repositoryPath, r.repostorySubdir, r.commitId, r.systemAttr, r.hostname)
+	r.drvPath, r.outPath, r.machineId, err = r.evalFunc(ctx, r.repositoryPath, r.repostorySubdir, r.commitId, r.systemAttr, r.hostname, r.submodules)
 	return err
 }
 
@@ -194,6 +197,7 @@ func (b *Builder) Eval(ctx context.Context, rs *protobuf.RepositoryStatus) error
 		repositoryPath:  g.RepositoryPath,
 		repostorySubdir: g.RepositorySubdir,
 		systemAttr:      g.SystemAttr,
+		submodules:      b.submodules,
 
 		commitId: g.SelectedCommitId,
 		evalFunc: b.executor.Eval,

internal/builder/builder_test.go

@@ -33,7 +33,7 @@ func (n ExecutorMock) IsStorePathExist(storePath string) bool {
 func (n ExecutorMock) Deploy(ctx context.Context, outPath, operation string) (needToRestartComin bool, profilePath string, err error) {
 	return false, "", nil
 }
-func (n ExecutorMock) Eval(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string) (drvPath string, outPath string, machineId string, err error) {
+func (n ExecutorMock) Eval(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string, submodules bool) (drvPath string, outPath string, machineId string, err error) {
 	select {
 	case <-ctx.Done():
 		return "", "", "", ctx.Err()
@@ -69,7 +69,7 @@ func TestBuilderBuild(t *testing.T) {
 	s, err := store.New(bk, tmp+"/state.json", tmp+"/gcroots", 1, 1)
 	assert.Nil(t, err)
 	eMock := NewExecutorMock(false)
-	b := New(s, eMock, "", "", "", "my-machine", 2*time.Second, 2*time.Second)
+	b := New(s, eMock, "", "", "", "my-machine", false, 2*time.Second, 2*time.Second)
 	ctx := t.Context()
 
 	// Run the evaluator
@@ -133,7 +133,7 @@ func TestEval(t *testing.T) {
 	s, err := store.New(bk, tmp+"/state.json", tmp+"/gcroots", 1, 1)
 	assert.Nil(t, err)
 	eMock := NewExecutorMock(false)
-	b := New(s, eMock, "", "", "", "", 5*time.Second, 5*time.Second)
+	b := New(s, eMock, "", "", "", "", false, 5*time.Second, 5*time.Second)
 	_ = b.Eval(t.Context(), &protobuf.RepositoryStatus{})
 	assert.True(t, b.isEvaluating.Load())
 	eMock.evalDone <- struct{}{}
@@ -155,7 +155,7 @@ func TestEvalAlreadyBuilt(t *testing.T) {
 	s, err := store.New(bk, tmp+"/state.json", tmp+"/gcroots", 1, 1)
 	assert.Nil(t, err)
 	eMock := NewExecutorMock(true)
-	b := New(s, eMock, "", "", "", "", 5*time.Second, 5*time.Second)
+	b := New(s, eMock, "", "", "", "", false, 5*time.Second, 5*time.Second)
 	_ = b.Eval(t.Context(), &protobuf.RepositoryStatus{})
 	assert.True(t, b.IsEvaluating())
 
@@ -179,7 +179,7 @@ func TestBuilderPreemption(t *testing.T) {
 	s, err := store.New(bk, tmp+"/state.json", tmp+"/gcroots", 1, 1)
 	assert.Nil(t, err)
 	eMock := NewExecutorMock(false)
-	b := New(s, eMock, "", "", "", "", 5*time.Second, 5*time.Second)
+	b := New(s, eMock, "", "", "", "", false, 5*time.Second, 5*time.Second)
 	_ = b.Eval(t.Context(), &protobuf.RepositoryStatus{SelectedCommitId: "commit-1"})
 	assert.True(t, b.isEvaluating.Load())
 	assert.EventuallyWithT(t, func(c *assert.CollectT) {
@@ -202,7 +202,7 @@ func TestBuilderStop(t *testing.T) {
 	s, err := store.New(bk, tmp+"/state.json", tmp+"/gcroots", 1, 1)
 	assert.Nil(t, err)
 	eMock := NewExecutorMock(false)
-	b := New(s, eMock, "", "", "", "", 5*time.Second, 5*time.Second)
+	b := New(s, eMock, "", "", "", "", false, 5*time.Second, 5*time.Second)
 	_ = b.Eval(t.Context(), &protobuf.RepositoryStatus{})
 	assert.True(t, b.isEvaluating.Load())
 	b.Stop()
@@ -220,7 +220,7 @@ func TestBuilderTimeout(t *testing.T) {
 	s, err := store.New(bk, tmp+"/state.json", tmp+"/gcroots", 1, 1)
 	assert.Nil(t, err)
 	eMock := NewExecutorMock(false)
-	b := New(s, eMock, "", "", "", "", 1*time.Second, 5*time.Second)
+	b := New(s, eMock, "", "", "", "", false, 1*time.Second, 5*time.Second)
 	_ = b.Eval(t.Context(), &protobuf.RepositoryStatus{})
 	assert.True(t, b.isEvaluating.Load())
 	assert.EventuallyWithT(t, func(c *assert.CollectT) {
@@ -237,7 +237,7 @@ func TestBuilderSuspend(t *testing.T) {
 	s, err := store.New(bk, tmp+"/state.json", tmp+"/gcroots", 1, 1)
 	assert.Nil(t, err)
 	eMock := NewExecutorMock(false)
-	b := New(s, eMock, "", "", "", "", 1*time.Second, 5*time.Second)
+	b := New(s, eMock, "", "", "", "", false, 1*time.Second, 5*time.Second)
 	_ = b.Suspend()
 	assert.True(t, b.isSuspended)
 	_ = b.Eval(t.Context(), &protobuf.RepositoryStatus{})

internal/config/config.go

@@ -82,5 +82,6 @@ func MkGitConfig(config types.Configuration) types.GitConfig {
 		Dir:               config.RepositorySubdir,
 		Remotes:           config.Remotes,
 		GpgPublicKeyPaths: config.GpgPublicKeyPaths,
+		Submodules:        config.Submodules,
 	}
 }

internal/executor/executor.go

@@ -6,7 +6,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-type EvalFunc func(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string) (drvPath string, outPath string, machineId string, err error)
+type EvalFunc func(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string, submodules bool) (drvPath string, outPath string, machineId string, err error)
 type BuildFunc func(ctx context.Context, drvPath string) error
 
 // Executor contains the function used by comin to actually do actions
@@ -15,7 +15,7 @@ type BuildFunc func(ctx context.Context, drvPath string) error
 // Garnix implementation (such as proposed in
 // https://github.com/nlewo/comin/pull/74)
 type Executor interface {
-	Eval(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string) (drvPath string, outPath string, machineId string, err error)
+	Eval(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string, submodules bool) (drvPath string, outPath string, machineId string, err error)
 	Build(ctx context.Context, drvPath string) (err error)
 	Deploy(ctx context.Context, outPath, operation string) (needToRestartComin bool, profilePath string, err error)
 	NeedToReboot(outPath, operation string) bool

internal/executor/executor_test.go

@@ -60,7 +60,7 @@ func TestNixExecutorEval(t *testing.T) {
 			// Test that Eval doesn't panic and handles parameters correctly
 			// This will error in test environment since nix commands will fail,
 			// but we're testing the code path and parameter handling
-			_, _, _, err = executor.Eval(ctx, tt.repositoryPath, tt.repositorySubdir, tt.commitId, tt.systemAttr, tt.hostname)
+			_, _, _, err = executor.Eval(ctx, tt.repositoryPath, tt.repositorySubdir, tt.commitId, tt.systemAttr, tt.hostname, false)
 			t.Logf("Eval with %s returned error: %v (expected in test environment)", tt.systemAttr, err)
 		})
 	}

internal/executor/nix.go

@@ -30,8 +30,8 @@ func (n *NixLocal) NeedToReboot(outPath, operation string) bool {
 	return utils.NeedToRebootLinux(outPath, operation)
 }
 
-func (n *NixLocal) Eval(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string) (drvPath string, outPath string, machineId string, err error) {
-	tempDir, err := cloneRepoToTemp(repositoryPath, commitId)
+func (n *NixLocal) Eval(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string, submodules bool) (drvPath string, outPath string, machineId string, err error) {
+	tempDir, err := cloneRepoToTemp(repositoryPath, commitId, submodules)
 	defer os.RemoveAll(tempDir) // nolint: errcheck
 	if err != nil {
 		return
@@ -49,7 +49,7 @@ func (n *NixLocal) Deploy(ctx context.Context, outPath, operation string) (needT
 	return deployLinux(ctx, outPath, operation)
 }
 
-func cloneRepoToTemp(remoteDir string, commitId string) (string, error) {
+func cloneRepoToTemp(remoteDir string, commitId string, submodules bool) (string, error) {
 	dir, err := os.MkdirTemp("", "comin-git-clone-*")
 	if err != nil {
 		return "", err
@@ -68,9 +68,13 @@ func cloneRepoToTemp(remoteDir string, commitId string) (string, error) {
 		return "", fmt.Errorf("nix: failed to set reference 'archive' to '%s' in %s: %s", commitId, remoteDir, err)
 	}
 
-	r, err := git.PlainClone(dir, false, &git.CloneOptions{
+	cloneOpts := &git.CloneOptions{
 		URL: remoteDir,
-	})
+	}
+	if submodules {
+		cloneOpts.RecurseSubmodules = git.DefaultSubmoduleRecursionDepth
+	}
+	r, err := git.PlainClone(dir, false, cloneOpts)
 	if err != nil {
 		return "", fmt.Errorf("nix: failed to clone '%s': %s", dir, err)
 	}

internal/executor/nix_flake.go

@@ -45,8 +45,11 @@ func (n *NixFlakeLocal) ShowDerivation(ctx context.Context, flakeUrl, hostname s
 	return showDerivationWithFlake(ctx, flakeUrl, hostname, n.systemAttr)
 }
 
-func (n *NixFlakeLocal) Eval(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string) (drvPath string, outPath string, machineId string, err error) {
+func (n *NixFlakeLocal) Eval(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string, submodules bool) (drvPath string, outPath string, machineId string, err error) {
 	flakeUrl := fmt.Sprintf("git+file://%s?dir=%s&rev=%s", repositoryPath, repositorySubdir, commitId)
+	if submodules {
+		flakeUrl += "&submodules=1"
+	}
 	drvPath, outPath, err = showDerivationWithFlake(ctx, flakeUrl, hostname, n.systemAttr)
 	if err != nil {
 		return

internal/manager/manager_test.go

@@ -53,7 +53,7 @@ func (n ExecutorMock) IsStorePathExist(storePath string) bool {
 func (n ExecutorMock) Deploy(ctx context.Context, outPath, operation string) (needToRestartComin bool, profilePath string, err error) {
 	return false, "", nil
 }
-func (n ExecutorMock) Eval(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string) (drvPath string, outPath string, machineId string, err error) {
+func (n ExecutorMock) Eval(ctx context.Context, repositoryPath, repositorySubdir, commitId, systemAttr, hostname string, submodules bool) (drvPath string, outPath string, machineId string, err error) {
 	ok := <-n.evalOk
 	if ok {
 		return "drv-path", "out-path", n.machineId, nil
@@ -90,7 +90,7 @@ func TestBuild(t *testing.T) {
 	s, _ := store.New(bk, tmp+"/state.json", tmp+"/gcroots", 1, 1)
 	f.Start(t.Context())
 	eMock := NewExecutorMock("")
-	b := builder.New(s, eMock, "repoPath", "", "", "my-machine", 2*time.Second, 2*time.Second)
+	b := builder.New(s, eMock, "repoPath", "", "", "my-machine", false, 2*time.Second, 2*time.Second)
 	var deployFunc = func(context.Context, string, string) (bool, string, error) {
 		return false, "profile-path", nil
 	}
@@ -204,7 +204,7 @@ func TestDeploy(t *testing.T) {
 	eMock := NewExecutorMock("")
 	eMock.evalOk <- true
 	eMock.buildOk <- true
-	b := builder.New(s, eMock, "repoPath", "", "", "my-machine", 2*time.Second, 2*time.Second)
+	b := builder.New(s, eMock, "repoPath", "", "", "my-machine", false, 2*time.Second, 2*time.Second)
 	var deployFunc = func(context.Context, string, string) (bool, string, error) {
 		return false, "profile-path", nil
 	}
@@ -237,7 +237,7 @@ func TestIncorrectMachineId(t *testing.T) {
 
 	s, _ := store.New(bk, tmp+"/state.json", tmp+"/gcroots", 1, 1)
 	eMock := NewExecutorMock("invalid-machine-id")
-	b := builder.New(s, eMock, "repoPath", "", "", "my-machine", 2*time.Second, 2*time.Second)
+	b := builder.New(s, eMock, "repoPath", "", "", "my-machine", false, 2*time.Second, 2*time.Second)
 	d := mkDeployerMock(t)
 	e, _ := executor.NewNixOSFlake()
 	bc := NewConfirmer(bk, Without, 0, "")
@@ -269,7 +269,7 @@ func TestCorrectMachineId(t *testing.T) {
 	s, _ := store.New(bk, tmp+"/state.json", tmp+"/gcroots", 1, 1)
 	eMock := NewExecutorMock("the-test-machine-id")
 	eMock.evalOk <- true
-	b := builder.New(s, eMock, "repoPath", "", "", "my-machine", 2*time.Second, 2*time.Second)
+	b := builder.New(s, eMock, "repoPath", "", "", "my-machine", false, 2*time.Second, 2*time.Second)
 	d := mkDeployerMock(t)
 	e, _ := executor.NewNixOSFlake()
 	bc := NewConfirmer(bk, Without, 0, "")
@@ -299,7 +299,7 @@ func TestManagerWithDarwinConfiguration(t *testing.T) {
 	bk.Start()
 
 	s, _ := store.New(bk, tmp+"/state.json", tmp+"/gcroots", 1, 1)
-	b := builder.New(s, eMock, "repoPath", "", "", "my-machine", 2*time.Second, 2*time.Second)
+	b := builder.New(s, eMock, "repoPath", "", "", "my-machine", false, 2*time.Second, 2*time.Second)
 	d := mkDeployerMock(t)
 
 	// Test with Darwin configuration