kola: Support default confext for /etc

When we switch from a custom overlay mount for /etc to using
systemd-confext there are a few minor differences with unmounting /etc
from a new namespace for inspection and when checking extension paths
that are now resolved when being symlinks.
Also add a new test for making sure that the confext/sysext skipping
works.

Signed-off-by: Kai Lueke <kailuke@microsoft.com>

Author: pothos

kola/tests/misc/state.go

@@ -69,7 +69,7 @@ func OverlayCleanup(c cluster.TestCluster) {
 	// systemd-tmpfiles to cause any recreation at boot: This was observed with C, L, and d entries
 	// (file or tree copy, symlink setup, directory creation) and thus they are dropped during image
 	// build as workaround.
-	overlayCheck := `sudo unshare -m bash -c 'umount /etc || { echo "Could not unmount /etc"; exit 1; }; if test -e "/etc/hosts" || test -e "/etc/security" || test -e "/etc/profile.d" || test -e "/etc/shells" || test -e "/etc/os-release" ; then echo "Unexpected overlay copy in /etc %s: $_" ; exit 1; fi'`
+	overlayCheck := `sudo unshare -m bash -c 'umount -l /etc || { echo "Could not unmount /etc"; exit 1; }; if test -e "/etc/hosts" || test -e "/etc/security" || test -e "/etc/profile.d" || test -e "/etc/shells" || test -e "/etc/os-release" ; then echo "Unexpected overlay copy in /etc %s: $_" ; exit 1; fi'`
 	_ = c.MustSSH(m, fmt.Sprintf(overlayCheck, "on initial boot"))
 
 	// Do some local modifications that are expected to be kept:
@@ -92,14 +92,14 @@ func OverlayCleanup(c cluster.TestCluster) {
 	// The migration path for old machines with a full /etc and the cleanup of unwanted duplicates/
 	// upcopies can be tested the same way by copying duplicates to /etc and then rebooting to
 	// check that they get cleaned up.
-	_ = c.MustSSH(m, `sudo unshare -m bash -c 'umount /etc && cp -a /usr/share/flatcar/etc/{hosts,shells,os-release} /etc/ && mkdir /etc/security /etc/profile.d'`)
+	_ = c.MustSSH(m, `sudo unshare -m bash -c 'umount -l /etc && cp -a /usr/share/flatcar/etc/{hosts,shells,os-release} /etc/ && mkdir /etc/security /etc/profile.d'`)
 	if err := m.Reboot(); err != nil {
 		c.Fatalf("could not reboot: %v", err)
 	}
 
 	_ = c.MustSSH(m, fmt.Sprintf(overlayCheck, "after reboot"))
 	_ = c.MustSSH(m, `if sudo test -e /etc/sssd/sssd.conf || test -e /etc/kexec.conf || test -e /etc/wireguard || test ! -e /etc/bash/hello || test ! -e /etc/bash/bashrc ; then echo "Deletion or modification lost: $_" ; exit 1; fi`)
-	_ = c.MustSSH(m, `if test ! -e /etc/resolv.conf ; then echo "Files with tmpfile rule not recreated: $_" ; exit 1; fi && if ! sudo unshare -m bash -c 'umount /etc && test ! -e /etc/resolv.conf'; then echo "File with tmpfile rule exists as upcopy: $_"; exit 1; fi`)
+	_ = c.MustSSH(m, `if test ! -e /etc/resolv.conf ; then echo "Files with tmpfile rule not recreated: $_" ; exit 1; fi && if ! sudo unshare -m bash -c 'umount -l /etc && test ! -e /etc/resolv.conf'; then echo "File with tmpfile rule exists as upcopy: $_"; exit 1; fi`)
 }
 
 // Check the OS reset logic with flatcar-reset to be able to

kola/tests/sysext/sysext.go

@@ -125,8 +125,9 @@ if [[ "${oem_test_type}" != 'raw' ]]; then
 fi
 
 oem_test_path=$(jq --raw-output '.path' <<<"${list_oem_test}")
-if [[ "${oem_test_path}" != '/etc/extensions/oem-test.raw' ]]; then
-        echo "oem test image path should be '/etc/extensions/oem-test.raw', is '${oem_test_path}'"
+oem_test_real_path=$(readlink -f /etc/extensions/oem-test.raw)
+if [[ "${oem_test_path}" != "${oem_test_real_path}" ]] && [[ "${oem_test_path}" != '/etc/extensions/oem-test.raw' ]]; then
+        echo "oem test image path should be '${oem_test_real_path}' or '/etc/extensions/oem-test.raw', is '${oem_test_path}'"
         exit 1
 fi
 
@@ -239,6 +240,14 @@ func init() {
         inline: |
           sysext works`),
 	})
+	register.Register(&register.Test{
+		Name:        "confext.skiprefresh",
+		Run:         checkConfextSkipRefresh,
+		ClusterSize: 1,
+		Distros:     []string{"cl"},
+		// This test is normally not related to the cloud environment
+		Platforms:  []string{"qemu", "qemu-unpriv"},
+		MinVersion: semver.Version{Major: 4548}})
 	register.Register(&register.Test{
 		Name:        "sysext.custom-docker.torcx",
 		Run:         checkSysextCustomDocker,
@@ -324,6 +333,12 @@ func checkSysextSimple(c cluster.TestCluster, oemMountpoint string) {
 	checkHelper(c, oemMountpoint)
 }
 
+func checkConfextSkipRefresh(c cluster.TestCluster) {
+	// This test uses no extra extension images
+	_ = c.MustSSH(c.Machines()[0], `if sudo journalctl -u systemd-confext -b0 | grep "Merged extensions into"; then echo "Unexpected confext merge, expected skip" ; exit 1 ; fi`)
+	_ = c.MustSSH(c.Machines()[0], `if ! findmnt -O rw --target /etc >/dev/null; then echo "Missing or wrong /etc mount, expected rw"; exit 1; fi`)
+}
+
 func checkSysextCustomDocker(c cluster.TestCluster) {
 	arch := strings.SplitN(kola.QEMUOptions.Board, "-", 2)[0]
 	if arch == "arm64" {
@@ -341,13 +356,14 @@ func checkSysextCustomDocker(c cluster.TestCluster) {
 	// Flatcar has no mksquashfs and btrfs is missing a bugfix but at least ext4 works
 	// The first test is for a fixed Docker version, which with the time will get old and older but is still expected to work because users may also "freeze" their Docker version this way
 	_ = c.MustSSH(c.Machines()[0], fmt.Sprintf(`ARCH=%[1]s ONLY_DOCKER=1 FORMAT=ext4 sysext-bakery/create_docker_sysext.sh 20.10.21 docker && ARCH=%[1]s ONLY_CONTAINERD=1 FORMAT=ext4 sysext-bakery/create_docker_sysext.sh 20.10.21 containerd && sudo mv docker.raw containerd.raw /etc/extensions/`, arch))
-	_ = c.MustSSH(c.Machines()[0], `sudo systemctl restart systemd-sysext`)
+	// Explicit daemon-reload and docker start because the old bake.sh does not set EXTENSION_RELOAD_MANAGER=1
+	_ = c.MustSSH(c.Machines()[0], `sudo systemctl restart systemd-sysext && sudo systemctl daemon-reload`)
 	// We should now be able to use Docker
 	_ = c.MustSSH(c.Machines()[0], cmdWorking)
 	// The next test is with a recent Docker version, here the one from the Flatcar image to couple it to something that doesn't change under our feet
 	version := string(c.MustSSH(c.Machines()[0], `bzcat /usr/share/licenses/licenses.json.bz2 | grep -m 1 -o 'app-\(containers\|emulation\)/docker-[0-9][^:]*' | cut -d - -f 3`))
 	_ = c.MustSSH(c.Machines()[0], fmt.Sprintf(`ONLY_DOCKER=1 FORMAT=ext4 ARCH=%[2]s sysext-bakery/create_docker_sysext.sh %[1]s docker && ONLY_CONTAINERD=1 FORMAT=ext4 ARCH=%[2]s sysext-bakery/create_docker_sysext.sh %[1]s containerd && sudo mv docker.raw containerd.raw /etc/extensions/`, version, arch))
-	_ = c.MustSSH(c.Machines()[0], `sudo systemctl restart systemd-sysext && sudo systemctl restart docker containerd`)
+	_ = c.MustSSH(c.Machines()[0], `sudo systemctl restart systemd-sysext && sudo systemctl daemon-reload && sudo systemctl restart docker containerd`)
 	// We should now still be able to use Docker
 	_ = c.MustSSH(c.Machines()[0], cmdWorking)
 }

kola/tests/update/update.go

@@ -548,7 +548,8 @@ func sysextBootLogic(c cluster.TestCluster, oemMountpoint string) {
 	// a) stored on the rootfs and will stay there and the new one is is moved to the OEM partition
 	// b) stored on the OEM partition and gets moved to the rootfs and the new one is moved to the OEM partition
 	_ = c.MustSSH(noIgnition, fmt.Sprintf(`set -euxo pipefail
-sudo systemctl mask --now systemd-sysext ensure-sysext
+sudo systemctl mask --now ensure-sysext || true # Old releases
+sudo systemctl mask --now systemd-sysext
 sudo mkdir -p /etc/flatcar/sysext /etc/flatcar/oem-sysext %[2]s/sysext /etc/extensions
 echo ID=test | sudo tee %[2]s/oem-release
 echo myext | sudo tee /etc/flatcar/enabled-sysext.conf