Ensure only RAW images in cache(imgconv/raw) while keeping the original image

pkg/cacheutil/cacheutil.go

@@ -43,7 +43,7 @@ func EnsureNerdctlArchiveCache(ctx context.Context, y *limatype.LimaYAML, create
 				return path, nil
 			}
 		}
-		path, err := fileutils.DownloadFile(ctx, "", f, false, "the nerdctl archive", *y.Arch)
+		path, err := fileutils.DownloadFile(ctx, "", f, false, "the nerdctl archive", *y.Arch, nil)
 		if err != nil {
 			errs[i] = err
 			continue

pkg/downloader/downloader.go

@@ -15,16 +15,20 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"slices"
 	"strings"
 	"sync/atomic"
 	"time"
 
 	"github.com/cheggaaa/pb/v3"
 	"github.com/containerd/continuity/fs"
+	"github.com/lima-vm/go-qcow2reader/image/raw"
 	"github.com/opencontainers/go-digest"
 	"github.com/sirupsen/logrus"
 
 	"github.com/lima-vm/lima/v2/pkg/httpclientutil"
+	"github.com/lima-vm/lima/v2/pkg/imgutil/nativeimgutil"
+	"github.com/lima-vm/lima/v2/pkg/imgutil/proxyimgutil"
 	"github.com/lima-vm/lima/v2/pkg/localpathutil"
 	"github.com/lima-vm/lima/v2/pkg/lockutil"
 	"github.com/lima-vm/lima/v2/pkg/progressbar"
@@ -56,10 +60,11 @@ type Result struct {
 }
 
 type options struct {
-	cacheDir       string // default: empty (disables caching)
-	decompress     bool   // default: false (keep compression)
-	description    string // default: url
-	expectedDigest digest.Digest
+	cacheDir              string // default: empty (disables caching)
+	decompress            bool   // default: false (keep compression)
+	description           string // default: url
+	expectedDigest        digest.Digest
+	supportedImageFormats []string
 }
 
 func (o *options) apply(opts []Opt) error {
@@ -110,6 +115,13 @@ func WithDecompress(decompress bool) Opt {
 	}
 }
 
+func WithImageFormats(supportedImageFormats []string) Opt {
+	return func(o *options) error {
+		o.supportedImageFormats = supportedImageFormats
+		return nil
+	}
+}
+
 // WithExpectedDigest is used to validate the downloaded file against the expected digest.
 //
 // The digest is not verified in the following cases:
@@ -266,6 +278,62 @@ func getCached(ctx context.Context, localPath, remote string, o options) (*Resul
 	if _, err := os.Stat(shadData); err != nil {
 		return nil, nil
 	}
+
+	if isNonISOImage(o.description, remote, o.supportedImageFormats) {
+		imageFormat, err := nativeimgutil.DetectFormat(shadData)
+		if err != nil {
+			return nil, err
+		}
+		if !slices.Contains(o.supportedImageFormats, imageFormat) {
+			rawImgConvPath := filepath.Join(shad, "imgconv", "raw")
+			rawImgConvDigestPath := filepath.Join(shad, "imgconv", "raw.digest")
+
+			needConvert := false
+			if rawStat, err := os.Stat(rawImgConvPath); err != nil {
+				if errors.Is(err, os.ErrNotExist) {
+					needConvert = true
+				} else {
+					return nil, err
+				}
+			} else {
+				origStat, _ := os.Stat(shadData)
+				if origStat.ModTime().After(rawStat.ModTime()) {
+					needConvert = true
+				}
+			}
+
+			if o.expectedDigest != "" && shadDigest != "" {
+				if err := validateCachedDigest(shadDigest, o.expectedDigest); err != nil {
+					return nil, nil
+				}
+			}
+
+			if needConvert {
+				logrus.Infof("Converted raw image is missing or stale; (re)converting now.")
+				converted, rawDigest, err := ensureRawInCache(ctx, shadData, imageFormat, o.expectedDigest)
+				if err != nil {
+					return nil, err
+				}
+				shadData = converted
+				o.expectedDigest = rawDigest
+				shadDigest = rawImgConvDigestPath
+			} else {
+				shadData = rawImgConvPath
+				if currentDigestData, err := os.ReadFile(rawImgConvDigestPath); err == nil {
+					currentDigest := strings.TrimSpace(string(currentDigestData))
+					if d, err := digest.Parse(currentDigest); err == nil {
+						o.expectedDigest = d
+						shadDigest = rawImgConvDigestPath
+					} else {
+						return nil, fmt.Errorf("invalid digest in raw digest file %q: %w", rawImgConvDigestPath, err)
+					}
+				} else {
+					return nil, fmt.Errorf("failed to read raw digest file %q: %w", rawImgConvDigestPath, err)
+				}
+			}
+		}
+	}
+
 	ext := path.Ext(remote)
 	logrus.Debugf("file %q is cached as %q", localPath, shadData)
 	if _, err := os.Stat(shadDigest); err == nil {
@@ -299,6 +367,10 @@ func getCached(ctx context.Context, localPath, remote string, o options) (*Resul
 	return res, nil
 }
 
+func isNonISOImage(description, remote string, supportedImageFormats []string) bool {
+	return strings.Contains(description, "image") && supportedImageFormats != nil && path.Ext(remote) != ".iso"
+}
+
 // fetch downloads remote to the cache and copy the cached file to local path.
 func fetch(ctx context.Context, localPath, remote string, o options) (*Result, error) {
 	shad := cacheDirectoryPath(o.cacheDir, remote)
@@ -322,6 +394,24 @@ func fetch(ctx context.Context, localPath, remote string, o options) (*Result, e
 			return nil, err
 		}
 	}
+
+	// Try to convert to raw if it's an non-ISO image and not supported by the driver
+	if isNonISOImage(o.description, remote, o.supportedImageFormats) {
+		format, err := nativeimgutil.DetectFormat(shadData)
+		if err != nil {
+			return nil, err
+		}
+		if !slices.Contains(o.supportedImageFormats, format) {
+			converted, rawDigest, err := ensureRawInCache(ctx, shadData, format, o.expectedDigest)
+			if err != nil {
+				return nil, fmt.Errorf("failed to convert image to raw: %w", err)
+			} else if converted != "" {
+				shadData = converted
+				o.expectedDigest = rawDigest
+			}
+		}
+	}
+
 	// no need to pass the digest to copyLocal(), as we already verified the digest
 	if err := copyLocal(ctx, localPath, shadData, ext, o.decompress, "", ""); err != nil {
 		return nil, err
@@ -336,6 +426,71 @@ func fetch(ctx context.Context, localPath, remote string, o options) (*Result, e
 	return res, nil
 }
 
+// ensureRawInCache converts any image to raw and places it in the cache(imgconv/raw). It also creates a
+// digest file for the raw image(imgconv/raw.digest). Returns the converted image path, the raw digest, and any error.
+func ensureRawInCache(ctx context.Context, imagePath, format string, originalDigest digest.Digest) (string, digest.Digest, error) {
+	imgConvPath := filepath.Join(filepath.Dir(imagePath), "imgconv")
+	if err := os.MkdirAll(imgConvPath, 0o700); err != nil {
+		return "", "", err
+	}
+	rawImgConvPath := filepath.Join(imgConvPath, "raw")
+
+	logrus.Infof("Converting %s image to raw sparse format in cache: %q", format, rawImgConvPath)
+	rawPathTmp := filepath.Join(imgConvPath, "raw.tmp")
+	defer os.Remove(rawPathTmp)
+	diskUtil := proxyimgutil.NewDiskUtil(ctx)
+	if err := diskUtil.Convert(ctx, raw.Type, imagePath, rawPathTmp, nil, false); err != nil {
+		return "", "", fmt.Errorf("failed to convert %q to raw: %w", imagePath, err)
+	}
+
+	// Ensure the image is sparse to save cache space.
+	rawTmpF, err := os.OpenFile(rawPathTmp, os.O_RDWR, 0o644)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to open raw tmp file %q: %w", rawPathTmp, err)
+	}
+	fi, err := rawTmpF.Stat()
+	if err != nil {
+		_ = rawTmpF.Close()
+		return "", "", fmt.Errorf("failed to stat raw tmp file %q: %w", rawPathTmp, err)
+	}
+	if err := diskUtil.MakeSparse(ctx, rawTmpF, fi.Size()); err != nil {
+		logrus.WithError(err).Warnf("Failed to make %q sparse (non-fatal)", rawPathTmp)
+	}
+	if err := rawTmpF.Close(); err != nil {
+		return "", "", fmt.Errorf("failed to close raw tmp file %q: %w", rawPathTmp, err)
+	}
+
+	if err := os.Rename(rawPathTmp, rawImgConvPath); err != nil {
+		return "", "", fmt.Errorf("failed to replace original with raw image: %w", err)
+	}
+
+	algo := digest.Canonical
+	if originalDigest != "" {
+		algo = originalDigest.Algorithm()
+	}
+	rawDigest, err := calculateFileDigest(rawImgConvPath, algo)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to calculate digest of raw image: %w", err)
+	}
+
+	rawDigestPath := filepath.Join(imgConvPath, "raw.digest")
+	if err := os.WriteFile(rawDigestPath, []byte(rawDigest.String()), 0o644); err != nil {
+		return "", "", fmt.Errorf("failed to write raw digest file: %w", err)
+	}
+
+	return rawImgConvPath, rawDigest, nil
+}
+
+func calculateFileDigest(path string, algo digest.Algorithm) (digest.Digest, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	return algo.FromReader(f)
+}
+
 // Cached checks if the remote resource is in the cache.
 //
 // Download caches the remote resource if WithCache or WithCacheDir option is specified.

pkg/downloader/downloader_test.go

@@ -379,3 +379,129 @@ func TestDownloadCompressed(t *testing.T) {
 		assert.Equal(t, string(got), string(testDownloadCompressedContents))
 	})
 }
+
+// This test simulates the end-to-end flow of downloading a remote image and converting it.
+func TestDownloadImageConversion(t *testing.T) {
+	_, err := exec.LookPath("qemu-img")
+	if err != nil {
+		t.Skipf("qemu-img does not seem installed: %v", err)
+	}
+
+	remoteDir := t.TempDir()
+	ts := httptest.NewServer(http.FileServer(http.Dir(remoteDir)))
+	t.Cleanup(ts.Close)
+
+	qcow2Path := filepath.Join(remoteDir, "test.qcow2")
+	assert.NilError(t, exec.CommandContext(t.Context(), "qemu-img", "create", "-f", "qcow2", qcow2Path, "64K").Run())
+
+	t.Run("without-digest", func(t *testing.T) {
+		ctx := t.Context()
+		cacheDir := t.TempDir()
+		localPath := filepath.Join(t.TempDir(), "local")
+		remoteURL := ts.URL + "/test.qcow2"
+
+		// 1. First download, should convert to raw
+		r, err := Download(ctx, localPath, remoteURL,
+			WithCacheDir(cacheDir),
+			WithDescription("image"),
+			WithImageFormats([]string{"raw"}),
+		)
+		assert.NilError(t, err)
+		assert.Equal(t, StatusDownloaded, r.Status)
+
+		shad := cacheDirectoryPath(cacheDir, remoteURL)
+		rawPath := filepath.Join(shad, "imgconv", "raw")
+		_, err = os.Stat(rawPath)
+		assert.NilError(t, err)
+
+		// 2. Second download, should use cached raw
+		localPath2 := filepath.Join(t.TempDir(), "local2")
+		r, err = Download(ctx, localPath2, remoteURL,
+			WithCacheDir(cacheDir),
+			WithDescription("image"),
+			WithImageFormats([]string{"raw"}),
+		)
+		assert.NilError(t, err)
+		assert.Equal(t, StatusUsedCache, r.Status)
+		assert.Equal(t, rawPath, r.CachePath)
+	})
+
+	t.Run("with-digest", func(t *testing.T) {
+		ctx := t.Context()
+		cacheDir := t.TempDir()
+		localPath := filepath.Join(t.TempDir(), "local")
+		remoteURL := ts.URL + "/test.qcow2"
+
+		content, err := os.ReadFile(qcow2Path)
+		assert.NilError(t, err)
+		originalDigest := digest.SHA256.FromBytes(content)
+
+		// 1. First download, should convert to raw
+		r, err := Download(ctx, localPath, remoteURL,
+			WithCacheDir(cacheDir),
+			WithDescription("image"),
+			WithImageFormats([]string{"raw"}),
+			WithExpectedDigest(originalDigest),
+		)
+		assert.NilError(t, err)
+		assert.Equal(t, StatusDownloaded, r.Status)
+
+		shad := cacheDirectoryPath(cacheDir, remoteURL)
+		rawPath := filepath.Join(shad, "imgconv", "raw")
+		_, err = os.Stat(rawPath)
+		assert.NilError(t, err)
+
+		// 2. Second download, should use cached raw
+		localPath2 := filepath.Join(t.TempDir(), "local2")
+		r, err = Download(ctx, localPath2, remoteURL,
+			WithCacheDir(cacheDir),
+			WithDescription("image"),
+			WithImageFormats([]string{"raw"}),
+			WithExpectedDigest(originalDigest),
+		)
+		assert.NilError(t, err)
+		assert.Equal(t, StatusUsedCache, r.Status)
+		assert.Equal(t, rawPath, r.CachePath)
+	})
+}
+
+// This test focuses specifically on the scenario where the source image
+// is already in the Lima cache but the converted version does not exist yet.
+func TestDownloadImageConversionCached(t *testing.T) {
+	_, err := exec.LookPath("qemu-img")
+	if err != nil {
+		t.Skipf("qemu-img does not seem installed: %v", err)
+	}
+
+	ctx := t.Context()
+	tmpDir := t.TempDir()
+	cacheDir := filepath.Join(tmpDir, "cache")
+	remoteURL := "https://example.com/test.qcow2"
+
+	// Pre-populate cache with a qcow2 file
+	shad := cacheDirectoryPath(cacheDir, remoteURL)
+	assert.NilError(t, os.MkdirAll(shad, 0o700))
+	shadData := filepath.Join(shad, "data")
+	assert.NilError(t, exec.CommandContext(t.Context(), "qemu-img", "create", "-f", "qcow2", shadData, "64K").Run())
+
+	// We need to provide either a digest or a valid shadTime file to avoid re-download
+	shadTime := filepath.Join(shad, "time")
+	assert.NilError(t, os.WriteFile(shadTime, []byte(time.Now().Format(http.TimeFormat)), 0o644))
+
+	localPath := filepath.Join(tmpDir, "local")
+
+	// Call Download, it should find qcow2 in cache, see it's not supported, and convert it
+	r, err := Download(ctx, localPath, remoteURL,
+		WithCacheDir(cacheDir),
+		WithDescription("image"),
+		WithImageFormats([]string{"raw"}),
+	)
+	assert.NilError(t, err)
+	assert.Equal(t, StatusUsedCache, r.Status)
+
+	// Verify conversion happened
+	rawPath := filepath.Join(shad, "imgconv", "raw")
+	_, err = os.Stat(rawPath)
+	assert.NilError(t, err)
+	assert.Equal(t, rawPath, r.CachePath)
+}

pkg/driver/driver.go

@@ -116,11 +116,12 @@ type Info struct {
 }
 
 type DriverFeatures struct {
-	CanRunGUI            bool `json:"canRunGui,omitempty"`
-	DynamicSSHAddress    bool `json:"dynamicSSHAddress"`
-	StaticSSHPort        bool `json:"staticSSHPort"`
-	SkipSocketForwarding bool `json:"skipSocketForwarding"`
-	NoCloudInit          bool `json:"noCloudInit"`
-	RosettaEnabled       bool `json:"rosettaEnabled"`
-	RosettaBinFmt        bool `json:"rosettaBinFmt"`
+	CanRunGUI             bool     `json:"canRunGui,omitempty"`
+	DynamicSSHAddress     bool     `json:"dynamicSSHAddress"`
+	StaticSSHPort         bool     `json:"staticSSHPort"`
+	SkipSocketForwarding  bool     `json:"skipSocketForwarding"`
+	NoCloudInit           bool     `json:"noCloudInit"`
+	RosettaEnabled        bool     `json:"rosettaEnabled"`
+	RosettaBinFmt         bool     `json:"rosettaBinFmt"`
+	SupportedImageFormats []string `json:"supportedImageFormats,omitempty"`
 }

pkg/driver/krunkit/krunkit_driver_darwin_arm64.go

@@ -265,9 +265,10 @@ func (l *LimaKrunkitDriver) Info() driver.Info {
 	}
 
 	info.Features = driver.DriverFeatures{
-		DynamicSSHAddress:    false,
-		SkipSocketForwarding: false,
-		CanRunGUI:            false,
+		DynamicSSHAddress:     false,
+		SkipSocketForwarding:  false,
+		CanRunGUI:             false,
+		SupportedImageFormats: []string{string(raw.Type)},
 	}
 	return info
 }