kata.runtime: 3.26.0 -> 3.27.0

Author: charludo

nodeinstaller/internal/kataconfig/config.go

@@ -54,6 +54,10 @@ func KataRuntimeConfig(
 			return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err)
 		}
 		config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "tdx", "share", "OVMF.fd")
+		// With these params set, containers do not boot, even when kernel_verity_params
+		// is added to the enabled annotations below.
+		// TODO(charludo): can we implement this? https://github.com/kata-containers/kata-containers/pull/12396
+		config.Hypervisor["qemu"]["kernel_verity_params"] = ""
 	case platforms.IsSNP(platform):
 		if err := toml.Unmarshal([]byte(kataBareMetalQEMUSNPBaseConfig), &config); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err)

nodeinstaller/internal/kataconfig/configuration-qemu-snp.toml

@@ -14,7 +14,7 @@
 
 [hypervisor.qemu]
 path = "/opt/kata/bin/qemu-system-x86_64"
-kernel = "/opt/kata/share/kata-containers/vmlinuz-confidential.container"
+kernel = "/opt/kata/share/kata-containers/vmlinuz.container"
 initrd = "/opt/kata/share/kata-containers/kata-containers-initrd-confidential.img"
 machine_type = "q35"
 
@@ -33,7 +33,7 @@ rootfs_type = "ext4"
 #
 # Known limitations:
 # * Does not work by design:
-#   - CPU Hotplug 
+#   - CPU Hotplug
 #   - Memory Hotplug
 #   - NVDIMM devices
 #
@@ -51,7 +51,7 @@ rootless = false
 # List of valid annotation names for the hypervisor
 # Each member of the list is a regular expression, which is the base name
 # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
-enable_annotations = ["enable_iommu", "virtio_fs_extra_args", "kernel_params", "default_vcpus", "default_memory", "cc_init_data"]
+enable_annotations = ["enable_iommu", "virtio_fs_extra_args", "kernel_params", "kernel_verity_params", "default_vcpus", "default_memory", "cc_init_data"]
 
 # List of valid annotations values for the hypervisor
 # Each member of the list is a path pattern as described by glob(3).
@@ -74,7 +74,7 @@ snp_id_auth = ""
 
 # SNP Guest Policy, the ‘POLICY’ parameter to the SNP_LAUNCH_START command.
 # If unset, the QEMU default policy (0x30000) will be used.
-# Notice that the guest policy is enforced at VM launch, and your pod VMs 
+# Notice that the guest policy is enforced at VM launch, and your pod VMs
 # won't start at all if the policy denys it. This will be indicated by a
 # 'SNP_LAUNCH_START' error.
 snp_guest_policy = 196608
@@ -339,6 +339,21 @@ enable_iommu = false
 # Enabling this will result in the VM device having iommu_platform=on set
 enable_iommu_platform = false
 
+# Enable NUMA topology, default false
+# When enable_numa is enabled, the hypervisor will expose host NUMA topology
+# as is: map VM NUMA nodes to host 1:1 and bind vCPUs to related CPUs.
+# Note: To take proper advantage of NUMA, static_sandbox_resource_mgmt should
+# also be enabled for memory pre-allocation.
+enable_numa = false
+
+# NUMA node mapping allows customizing how VM NUMA nodes map to host NUMA nodes.
+# Each entry defines a VM NUMA node and the host NUMA node(s) it maps to.
+# Format: ["<host_nodes>", "<host_nodes>", ...]
+# Example: ["0", "1"] creates 2 VM NUMA nodes, mapping to host nodes 0 and 1
+# Example: ["0-1", "2-3"] creates 2 VM NUMA nodes, first maps to host 0-1, second to 2-3
+# If empty and enable_numa is true, VM NUMA nodes map 1:1 to host NUMA nodes.
+numa_mapping = []
+
 # List of valid annotations values for the vhost user store path
 # The default if not set is empty (all annotations rejected.)
 # Your distribution recommends: ["/var/run/kata-containers/vhost-user"]
@@ -384,7 +399,7 @@ msize_9p = 8192
 # Otherwise virtio-block device is used.
 #
 # nvdimm is not supported when `confidential_guest = true`.
-disable_image_nvdimm = false
+disable_image_nvdimm = true
 
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
@@ -702,9 +717,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to 60 second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to 60 second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = 60
 

nodeinstaller/internal/kataconfig/configuration-qemu-tdx.toml

@@ -12,8 +12,8 @@
 # XXX:   Type: kata
 
 [hypervisor.qemu]
-path = "PLACEHOLDER_FOR_DISTRO_QEMU_WITH_TDX_SUPPORT"
-kernel = "/opt/kata/share/kata-containers/vmlinuz-confidential.container"
+path = "/opt/kata/bin/qemu-system-x86_64"
+kernel = "/opt/kata/share/kata-containers/vmlinuz.container"
 image = "/opt/kata/share/kata-containers/kata-containers-confidential.img"
 machine_type = "q35"
 tdx_quote_generation_service_socket_port = 4050
@@ -33,7 +33,7 @@ rootfs_type = "ext4"
 #
 # Known limitations:
 # * Does not work by design:
-#   - CPU Hotplug 
+#   - CPU Hotplug
 #   - Memory Hotplug
 #   - NVDIMM devices
 #
@@ -48,13 +48,13 @@ rootless = false
 # List of valid annotation names for the hypervisor
 # Each member of the list is a regular expression, which is the base name
 # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
-enable_annotations = ["enable_iommu", "virtio_fs_extra_args", "kernel_params", "default_vcpus", "default_memory", "cc_init_data"]
+enable_annotations = ["enable_iommu", "virtio_fs_extra_args", "kernel_params", "kernel_verity_params", "default_vcpus", "default_memory", "cc_init_data"]
 
 # List of valid annotations values for the hypervisor
 # Each member of the list is a path pattern as described by glob(3).
 # The default if not set is empty (all annotations rejected.)
 # Your distribution recommends: ["/opt/kata/bin/qemu-system-x86_64"]
-valid_hypervisor_paths = ["PLACEHOLDER_FOR_DISTRO_QEMU_WITH_TDX_SUPPORT"]
+valid_hypervisor_paths = ["/opt/kata/bin/qemu-system-x86_64"]
 
 # Optional space-separated list of options to pass to the guest kernel.
 # For example, use `kernel_params = "vsyscall=emulate"` if you are having
@@ -66,7 +66,12 @@ valid_hypervisor_paths = ["PLACEHOLDER_FOR_DISTRO_QEMU_WITH_TDX_SUPPORT"]
 # may stop the virtual machine from booting.
 # To see the list of default parameters, enable hypervisor debug, create a
 # container and look for 'default-kernel-parameters' log entries.
-kernel_params = "cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1 rootfs_verity.scheme=dm-verity rootfs_verity.hash=d6a09623515a4f169f354ffcde82d79e96cf84b9ce5a6fc2eb18bda7fc864685"
+kernel_params = "cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1"
+
+# Optional dm-verity parameters (comma-separated key=value list):
+# root_hash=...,salt=...,data_blocks=...,data_block_size=...,hash_block_size=...
+# These are used by the runtime to assemble dm-verity kernel params.
+kernel_verity_params = "root_hash=1a58cb83f8e721a7a5174b9817ae502975e0e55ae81b92ff34e61223786c2300,salt=8c15fdda00c8290e45c21aef74f1143a9c7bdfb1b98fdd3900f7f700cef3f0ee,data_blocks=64000,data_block_size=4096,hash_block_size=4096"
 
 # Path to the firmware.
 # If you want that qemu uses the default firmware leave this option empty
@@ -316,6 +321,21 @@ enable_iommu = false
 # Enabling this will result in the VM device having iommu_platform=on set
 enable_iommu_platform = false
 
+# Enable NUMA topology, default false
+# When enable_numa is enabled, the hypervisor will expose host NUMA topology
+# as is: map VM NUMA nodes to host 1:1 and bind vCPUs to related CPUs.
+# Note: To take proper advantage of NUMA, static_sandbox_resource_mgmt should
+# also be enabled for memory pre-allocation.
+enable_numa = false
+
+# NUMA node mapping allows customizing how VM NUMA nodes map to host NUMA nodes.
+# Each entry defines a VM NUMA node and the host NUMA node(s) it maps to.
+# Format: ["<host_nodes>", "<host_nodes>", ...]
+# Example: ["0", "1"] creates 2 VM NUMA nodes, mapping to host nodes 0 and 1
+# Example: ["0-1", "2-3"] creates 2 VM NUMA nodes, first maps to host 0-1, second to 2-3
+# If empty and enable_numa is true, VM NUMA nodes map 1:1 to host NUMA nodes.
+numa_mapping = []
+
 # List of valid annotations values for the vhost user store path
 # The default if not set is empty (all annotations rejected.)
 # Your distribution recommends: ["/var/run/kata-containers/vhost-user"]
@@ -361,7 +381,7 @@ msize_9p = 8192
 # Otherwise virtio-block device is used.
 #
 # nvdimm is not supported when `confidential_guest = true`.
-disable_image_nvdimm = false
+disable_image_nvdimm = true
 
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
@@ -679,9 +699,9 @@ enable_pprof = false
 
 # Indicates the CreateContainer request timeout needed for the workload(s)
 # It using guest_pull this includes the time to pull the image inside the guest
-# Defaults to 60 second(s)  
-# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config 
-# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout. 
+# Defaults to 60 second(s)
+# Note: The effective timeout is determined by the lesser of two values: runtime-request-timeout from kubelet config
+# (https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#:~:text=runtime%2Drequest%2Dtimeout) and create_container_timeout.
 # In essence, the timeout used for guest pull=runtime-request-timeout<create_container_timeout?runtime-request-timeout:create_container_timeout.
 create_container_timeout = 60
 

nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp-gpu.toml

@@ -28,6 +28,7 @@ enable_iommu = false
 enable_iommu_platform = false
 enable_iothreads = false
 enable_mem_prealloc = false
+enable_numa = false
 enable_vhost_user_store = false
 enable_virtio_mem = false
 entropy_source = '/dev/urandom'
@@ -47,6 +48,7 @@ machine_type = 'q35'
 memory_offset = 0
 memory_slots = 10
 msize_9p = 8192
+numa_mapping = []
 path = '/bin/qemu-system-x86_64'
 pcie_root_port = 0
 pflashes = []

nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-snp.toml

@@ -27,6 +27,7 @@ enable_iommu = false
 enable_iommu_platform = false
 enable_iothreads = false
 enable_mem_prealloc = false
+enable_numa = false
 enable_vhost_user_store = false
 enable_virtio_mem = false
 entropy_source = '/dev/urandom'
@@ -46,6 +47,7 @@ machine_type = 'q35'
 memory_offset = 0
 memory_slots = 10
 msize_9p = 8192
+numa_mapping = []
 path = '/bin/qemu-system-x86_64'
 pcie_root_port = 0
 pflashes = []

nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-tdx-gpu.toml

@@ -28,6 +28,7 @@ enable_iommu = false
 enable_iommu_platform = false
 enable_iothreads = false
 enable_mem_prealloc = false
+enable_numa = false
 enable_vhost_user_store = false
 enable_virtio_mem = false
 entropy_source = '/dev/urandom'
@@ -42,11 +43,13 @@ indep_iothreads = 0
 initrd = '/share/kata-initrd.zst'
 kernel = '/share/kata-kernel'
 kernel_params = ''
+kernel_verity_params = ''
 machine_accelerators = ''
 machine_type = 'q35'
 memory_offset = 0
 memory_slots = 10
 msize_9p = 8192
+numa_mapping = []
 path = '/bin/qemu-system-x86_64'
 pcie_root_port = 0
 pflashes = []

nodeinstaller/internal/kataconfig/testdata/expected-configuration-qemu-tdx.toml

@@ -27,6 +27,7 @@ enable_iommu = false
 enable_iommu_platform = false
 enable_iothreads = false
 enable_mem_prealloc = false
+enable_numa = false
 enable_vhost_user_store = false
 enable_virtio_mem = false
 entropy_source = '/dev/urandom'
@@ -41,11 +42,13 @@ indep_iothreads = 0
 initrd = '/share/kata-initrd.zst'
 kernel = '/share/kata-kernel'
 kernel_params = ''
+kernel_verity_params = ''
 machine_accelerators = ''
 machine_type = 'q35'
 memory_offset = 0
 memory_slots = 10
 msize_9p = 8192
+numa_mapping = []
 path = '/bin/qemu-system-x86_64'
 pcie_root_port = 0
 pflashes = []

packages/by-name/kata/genpolicy/genpolicy_settings_dev.patch

@@ -1,8 +1,19 @@
 diff --git a/genpolicy-settings.json b/genpolicy-settings.json
-index 88171e1c40f271be9468c6c8b9b87c871a55c1f9..ac98ea9d3194bbca9cdc8c6fb4bf92a8f8472d97 100644
+index e7800bd10ba1eae91bdf61b4c1bab2af46bc4a05..185315d87306416b017cc35644c6b2b967604912 100644
 --- a/genpolicy-settings.json
 +++ b/genpolicy-settings.json
-@@ -296,7 +296,7 @@
+@@ -193,7 +193,9 @@
+                 "gpu_anno_value_regex": "^nvidia\\.com/gpu=[0-9]+$",
+                 "gpu_gk_device_type": "vfio-pci-gk",
+                 "pgpu_resource_keys": [
+-                    "nvidia.com/pgpu"
++                    "nvidia.com/pgpu",
++                    "nvidia.com/GB100_B200",
++                    "nvidia.com/GH100_H100_PCIE"
+                 ]
+             }
+         }
+@@ -301,7 +303,7 @@
          "enable_configmap_secret_storages": false
      },
      "cluster_config": {
@@ -11,7 +22,7 @@ index 88171e1c40f271be9468c6c8b9b87c871a55c1f9..ac98ea9d3194bbca9cdc8c6fb4bf92a8
          "guest_pull": true,
          "pause_container_id_policy": "v1"
      },
-@@ -317,7 +317,8 @@
+@@ -322,7 +324,8 @@
                  "^AZURE_TENANT_ID=[A-Fa-f0-9-]*$",
                  "^AZURE_FEDERATED_TOKEN_FILE=/var/run/secrets/azure/tokens/azure-identity-token$",
                  "^AZURE_AUTHORITY_HOST=https://login\\.microsoftonline\\.com/$",
@@ -21,7 +32,7 @@ index 88171e1c40f271be9468c6c8b9b87c871a55c1f9..ac98ea9d3194bbca9cdc8c6fb4bf92a8
              ]
          },
          "UpdateInterfaceRequest": {
-@@ -334,7 +335,9 @@
+@@ -339,7 +342,9 @@
          ],
          "ExecProcessRequest": {
              "allowed_commands": [],
@@ -32,7 +43,7 @@ index 88171e1c40f271be9468c6c8b9b87c871a55c1f9..ac98ea9d3194bbca9cdc8c6fb4bf92a8
          },
          "UpdateRoutesRequest": {
              "forbidden_device_names": [
-@@ -354,8 +357,8 @@
+@@ -359,8 +364,8 @@
              ]
          },
          "CloseStdinRequest": false,

packages/by-name/kata/genpolicy/genpolicy_settings_prod.patch

@@ -1,8 +1,19 @@
 diff --git a/genpolicy-settings.json b/genpolicy-settings.json
-index 88171e1c40f271be9468c6c8b9b87c871a55c1f9..3a4617ee6acad46e156cb664d710f42f1c488a32 100644
+index e7800bd10ba1eae91bdf61b4c1bab2af46bc4a05..1469b7238f4483325e63f32ea37b296b50e55e26 100644
 --- a/genpolicy-settings.json
 +++ b/genpolicy-settings.json
-@@ -296,7 +296,7 @@
+@@ -193,7 +193,9 @@
+                 "gpu_anno_value_regex": "^nvidia\\.com/gpu=[0-9]+$",
+                 "gpu_gk_device_type": "vfio-pci-gk",
+                 "pgpu_resource_keys": [
+-                    "nvidia.com/pgpu"
++                    "nvidia.com/pgpu",
++                    "nvidia.com/GB100_B200",
++                    "nvidia.com/GH100_H100_PCIE"
+                 ]
+             }
+         }
+@@ -301,7 +303,7 @@
          "enable_configmap_secret_storages": false
      },
      "cluster_config": {
@@ -11,7 +22,7 @@ index 88171e1c40f271be9468c6c8b9b87c871a55c1f9..3a4617ee6acad46e156cb664d710f42f
          "guest_pull": true,
          "pause_container_id_policy": "v1"
      },
-@@ -354,7 +354,7 @@
+@@ -359,7 +361,7 @@
              ]
          },
          "CloseStdinRequest": false,

packages/by-name/kata/kernel-uvm/package.nix

@@ -30,13 +30,13 @@ let
   };
 in
 linuxManualConfig rec {
-  version = "6.18.5";
-  modDirVersion = "${version}" + lib.optionalString withGPU "-nvidia-gpu-confidential";
+  version = "6.18.12";
+  modDirVersion = "${version}" + lib.optionalString withGPU "-nvidia-gpu";
 
   # See https://github.com/kata-containers/kata-containers/blob/5f11c0f144037d8d8f546c89a0392dcd84fa99e2/versions.yaml#L198-L201
   src = fetchurl {
     url = "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${version}.tar.xz";
-    hash = "sha256-GJ0fQJzvjQ0jQhDgRZUXLfOS+MspfhS0R+2Vcg4v2UA=";
+    hash = "sha256-4AMpStTCwqxbt3+7gllRETT1HZh7MhJRaDLcSwyD8eo=";
   };
 
   kernelPatches = [