Fix nixos-rebuild PATH and install context cancellation

Set NixosSystemPath in main so all exec.Command calls resolve bare names
from a systemd service (which strips PATH). Pass the same PATH via
sudo env so nixos-rebuild can find nix internally (was failing with
exit 127, nix: command not found).

Use context.WithoutCancel in the operation queue so a client disconnect
cannot cancel nixos-rebuild mid-flight. Previously, Go 1.21+ would
return context.Canceled immediately from exec.CommandContext if the
context was already done, leaving apps stuck in "installing" state
in the database permanently.

Author: d-buckner

services/host-agent/cmd/host-agent/main.go

@@ -12,6 +12,7 @@ import (
 	"codeberg.org/d-buckner/bloud-v3/services/host-agent/internal/appconfig"
 	"codeberg.org/d-buckner/bloud-v3/services/host-agent/internal/config"
 	"codeberg.org/d-buckner/bloud-v3/services/host-agent/internal/db"
+	"codeberg.org/d-buckner/bloud-v3/services/host-agent/internal/nixgen"
 	"codeberg.org/d-buckner/bloud-v3/services/host-agent/internal/system"
 	"codeberg.org/d-buckner/bloud-v3/services/host-agent/pkg/configurator"
 )
@@ -32,6 +33,12 @@ func main() {
 }
 
 func runServer() {
+	// Ensure system binaries are resolvable by bare name. systemd strips PATH
+	// to a minimal set that excludes /run/current-system/sw/bin and
+	// /run/wrappers/bin. Setting it here covers all exec.Command calls
+	// throughout the process (systemctl, journalctl, podman, nix, sudo, etc.).
+	_ = os.Setenv("PATH", nixgen.NixosSystemPath)
+
 	// Setup structured logging
 	logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
 		Level: slog.LevelInfo,

services/host-agent/internal/nixgen/rebuild.go

@@ -12,14 +12,16 @@ import (
 	"time"
 )
 
-// Stable NixOS binary paths. These are fixed by the system profile and the
-// sudo wrapper dir — safe to hardcode rather than relying on PATH, which
-// systemd strips for services running as root.
-const (
-	binSudo         = "/run/wrappers/bin/sudo"
-	binNixosRebuild = "/run/current-system/sw/bin/nixos-rebuild"
-	binSystemctl    = "/run/current-system/sw/bin/systemctl"
-)
+// NixosSystemPath is the PATH required for NixOS system commands.
+// Validated against the ISO: all binaries (nix, nixos-rebuild, systemctl,
+// journalctl, podman, machinectl, sudo) live in these two directories.
+// /usr/bin is empty on NixOS; /nix/var/nix/profiles/default/bin does not
+// exist on this system. /bin contains only sh.
+//
+// Set via os.Setenv in main so all exec.Command calls resolve bare names,
+// and passed explicitly via "sudo env" so child processes inherit it too
+// (sudo resets the environment by default).
+const NixosSystemPath = "/run/current-system/sw/bin:/run/wrappers/bin:/bin"
 
 // Rebuilder handles nixos-rebuild operations
 type Rebuilder struct {
@@ -64,12 +66,14 @@ type RebuildResult struct {
 func (r *Rebuilder) nixosRebuildCmd(ctx context.Context, args []string) *exec.Cmd {
 	if r.useSudo {
 		sudoArgs := append([]string{
-			"env", "_NIXOS_REBUILD_REEXEC=1",
-			binNixosRebuild,
+			"env",
+			"_NIXOS_REBUILD_REEXEC=1",
+			"PATH=" + NixosSystemPath,
+			"nixos-rebuild",
 		}, args...)
-		return exec.CommandContext(ctx, binSudo, sudoArgs...)
+		return exec.CommandContext(ctx, "sudo", sudoArgs...)
 	}
-	cmd := exec.CommandContext(ctx, binNixosRebuild, args...)
+	cmd := exec.CommandContext(ctx, "nixos-rebuild", args...)
 	cmd.Env = append(os.Environ(), "_NIXOS_REBUILD_REEXEC=1")
 	return cmd
 }
@@ -81,11 +85,11 @@ func (r *Rebuilder) userSystemctlCmd(ctx context.Context, args []string) *exec.C
 	if r.useSudo {
 		machinectlArgs := append([]string{
 			"shell", "bloud@",
-			binSystemctl, "--user",
+			"systemctl", "--user",
 		}, args...)
-		return exec.CommandContext(ctx, binSudo, append([]string{"machinectl"}, machinectlArgs...)...)
+		return exec.CommandContext(ctx, "sudo", append([]string{"machinectl"}, machinectlArgs...)...)
 	}
-	return exec.CommandContext(ctx, binSystemctl, append([]string{"--user"}, args...)...)
+	return exec.CommandContext(ctx, "systemctl", append([]string{"--user"}, args...)...)
 }
 
 // Switch performs a nixos-rebuild switch

services/host-agent/internal/orchestrator/queue.go

@@ -356,17 +356,25 @@ func (q *OperationQueue) executeBatch(batch []QueuedOperation) {
 }
 
 // executeInstall runs a single install operation.
+// context.WithoutCancel detaches execution from the HTTP request context so
+// that a client disconnect (browser navigation, timeout) cannot cancel
+// nixos-rebuild mid-flight. Without this, Go 1.21+ exec.CommandContext returns
+// context.Canceled immediately if the context is already done, leaving the app
+// stuck in "installing" status in the database permanently.
 func (q *OperationQueue) executeInstall(op QueuedOperation) {
-	result, err := q.orchestrator.Install(op.Ctx, *op.Install)
+	ctx := context.WithoutCancel(op.Ctx)
+	result, err := q.orchestrator.Install(ctx, *op.Install)
 	op.ResultCh <- OperationResult{
 		InstallResult: result,
 		Err:           err,
 	}
 }
 
 // executeUninstall runs a single uninstall operation.
+// Uses a detached context for the same reason as executeInstall.
 func (q *OperationQueue) executeUninstall(op QueuedOperation) {
-	result, err := q.orchestrator.Uninstall(op.Ctx, *op.Uninstall)
+	ctx := context.WithoutCancel(op.Ctx)
+	result, err := q.orchestrator.Uninstall(ctx, *op.Uninstall)
 	op.ResultCh <- OperationResult{
 		UninstallResult: result,
 		Err:             err,