CI: Test & fix Linux ZFS built-in build

ZFS can be built directly into the Linux kernel.  Add a test build
of this to the CI to verify it works.  The test build is only enabled
on Fedora runners (since they run the newest kernels) and is done in
parallel with ZTS.  The test build is done on vm2, since it typically
finishes ~15min before vm1 and thus has time to spare.

In addition:

- Update 'copy-builtin' to check that $1 is a directory
- Fix some VERIFYs that were causing the built-in build to fail

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Closes #18234

Author: tonyhutter

.github/workflows/scripts/qemu-6-tests.sh

@@ -4,9 +4,10 @@
 # 6) load openzfs module and run the tests
 #
 # called on runner:  qemu-6-tests.sh
-# called on qemu-vm: qemu-6-tests.sh $OS $2 $3 [--lustre] [quick|default]
+# called on qemu-vm: qemu-6-tests.sh $OS $2 $3 [--lustre|--builtin] [quick|default]
 #
 # --lustre: Test build lustre in addition to the normal tests
+# --builtin: Test build ZFS as a kernel built-in in addition to the normal tests
 ######################################################################
 
 set -eu
@@ -50,6 +51,44 @@ function do_lustre_build() {
 }
 export -f do_lustre_build
 
+# Test build ZFS into the kernel directly
+function do_builtin_build() {
+  local rc=0
+  # Get currently full kernel version (like '6.18.8')
+  fullver=$(uname -r | grep -Eo '^[0-9]+\.[0-9]+\.[0-9]+')
+
+  # Get just the major ('6')
+  major=$(echo $fullver | grep -Eo '^[0-9]+')
+  (
+  set -e
+
+  wget https://cdn.kernel.org/pub/linux/kernel/v${major}.x/linux-$fullver.tar.xz
+  tar -xf $HOME/linux-$fullver.tar.xz
+  cd $HOME/linux-$fullver
+  make tinyconfig
+  ./scripts/config --enable EFI_PARTITON
+  ./scripts/config --enable BLOCK
+  # BTRFS_FS is easiest config option to enable CONFIG_ZLIB_INFLATE|DEFLATE
+  ./scripts/config --enable BTRFS_FS
+  yes "" | make oldconfig
+  make prepare
+
+  cd $HOME/zfs
+  ./configure --with-linux=$HOME/linux-$fullver --enable-linux-builtin --enable-debug
+  ./copy-builtin $HOME/linux-$fullver
+
+  cd $HOME/linux-$fullver
+  ./scripts/config --enable ZFS
+  yes "" | make oldconfig
+  make -j `nproc`
+  ) &> /var/tmp/builtin.txt || rc=$?
+  echo "$rc" > /var/tmp/builtin-exitcode.txt
+  if [ "$rc" != "0" ] ; then
+      echo "$rc" > /var/tmp/tests-exitcode.txt
+  fi
+}
+export -f do_builtin_build
+
 # called directly on the runner
 if [ -z ${1:-} ]; then
   cd "/var/tmp"
@@ -66,9 +105,15 @@ if [ -z ${1:-} ]; then
     # on almalinux*.  At the time of writing, the vm2 tests were
     # completing roughly 15min before the vm1 tests, so it makes sense
     # to have vm2 do the build.
+    #
+    # In addition, we do an additional test build of ZFS as a Linux
+    # kernel built-in on Fedora.  Again, we do it on vm2 to exploit vm2's
+    # early finish time.
     extra=""
     if [[ "$OS" == almalinux* ]] && [[ "$i" == "2" ]] ; then
         extra="--lustre"
+    elif [[ "$OS" == fedora* ]] && [[ "$i" == "2" ]] ; then
+        extra="--builtin"
     fi
 
     daemonize -c /var/tmp -p vm${i}.pid -o vm${i}log.txt -- \
@@ -106,9 +151,13 @@ DEN="$1"
 shift
 
 BUILD_LUSTRE=0
+BUILD_BUILTIN=0
 if [ "$1" == "--lustre" ] ; then
   BUILD_LUSTRE=1
   shift
+elif [ "$1" == "--builtin" ] ; then
+  BUILD_BUILTIN=1
+  shift
 fi
 
 if [ "$1" == "quick" ] ; then
@@ -162,6 +211,9 @@ esac
 # The Lustre build on its own takes ~15min.
 if [ "$BUILD_LUSTRE" == "1" ] ; then
   do_lustre_build &
+elif [ "$BUILD_BUILTIN" == "1" ] ; then
+  # Try building ZFS directly into the Linux kernel (not as a module)
+  do_builtin_build &
 fi
 
 # run functional testings and save exitcode

.github/workflows/scripts/qemu-8-summary.sh

@@ -33,7 +33,7 @@ EOF
 
 function showfile_tail() {
   echo "##[group]$2 (final lines)"
-  tail -n 40 $1
+  tail -n 80 $1
   echo "##[endgroup]"
 }
 
@@ -66,6 +66,18 @@ for ((i=1; i<=VMs; i++)); do
     test -s "$file" && showfile_tail "$file" "$vm: Lustre build"
   fi
 
+  if [ -f vm$i/builtin-exitcode.txt ] ; then
+    rv=$(< vm$i/builtin-exitcode.txt)
+    if [ $rv = 0 ]; then
+      vm="vm$i"
+    else
+      vm="vm$i"
+      touch /tmp/have_failed_tests
+    fi
+    file="vm$i/builtin.txt"
+    test -s "$file" && showfile_tail "$file" "$vm: Linux built-in build"
+  fi
+
   rv=$(cat vm$i/tests-exitcode.txt)
 
   if [ $rv = 0 ]; then

copy-builtin

@@ -8,7 +8,9 @@ usage()
 	exit 1
 }
 
-[ "$#" -eq 1 ] || usage
+if ! [ -d "$1" ] ; then
+	usage
+fi
 KERNEL_DIR="$1"
 
 if ! [ -e 'zfs_config.h' ]
@@ -31,6 +33,7 @@ cat > "$KERNEL_DIR/fs/zfs/Kconfig" <<EOF
 config ZFS
 	tristate "ZFS filesystem support"
 	depends on EFI_PARTITION
+	depends on BLOCK
 	select ZLIB_INFLATE
 	select ZLIB_DEFLATE
 	help

module/zfs/dataset_kstats.c

@@ -212,7 +212,7 @@ dataset_kstats_rename(dataset_kstats_t *dk, const char *name)
 	char *ds_name;
 
 	ds_name = KSTAT_NAMED_STR_PTR(&dkv->dkv_ds_name);
-	ASSERT3S(ds_name, !=, NULL);
+	ASSERT3P(ds_name, !=, NULL);
 	(void) strlcpy(ds_name, name,
 	    KSTAT_NAMED_STR_BUFLEN(&dkv->dkv_ds_name));
 }

module/zfs/ddt.c

@@ -1508,7 +1508,7 @@ ddt_configure(ddt_t *ddt, boolean_t new)
 		    DMU_POOL_DIRECTORY_OBJECT, name, sizeof (uint64_t), 1,
 		    &ddt->ddt_dir_object);
 		if (error == 0) {
-			ASSERT3U(spa->spa_meta_objset, ==, ddt->ddt_os);
+			ASSERT3P(spa->spa_meta_objset, ==, ddt->ddt_os);
 
 			error = zap_lookup(ddt->ddt_os, ddt->ddt_dir_object,
 			    DDT_DIR_VERSION, sizeof (uint64_t), 1,

module/zfs/ddt_log.c

@@ -262,7 +262,7 @@ ddt_log_update_entry(ddt_t *ddt, ddt_log_t *ddl, ddt_lightweight_entry_t *ddlwe)
 void
 ddt_log_entry(ddt_t *ddt, ddt_lightweight_entry_t *ddlwe, ddt_log_update_t *dlu)
 {
-	ASSERT3U(dlu->dlu_dbp, !=, NULL);
+	ASSERT3P(dlu->dlu_dbp, !=, NULL);
 
 	ddt_log_update_entry(ddt, ddt->ddt_log_active, ddlwe);
 	ddt_histogram_add_entry(ddt, &ddt->ddt_log_histogram, ddlwe);
@@ -312,7 +312,7 @@ ddt_log_entry(ddt_t *ddt, ddt_lightweight_entry_t *ddlwe, ddt_log_update_t *dlu)
 void
 ddt_log_commit(ddt_t *ddt, ddt_log_update_t *dlu)
 {
-	ASSERT3U(dlu->dlu_dbp, !=, NULL);
+	ASSERT3P(dlu->dlu_dbp, !=, NULL);
 	ASSERT3U(dlu->dlu_block+1, ==, dlu->dlu_ndbp);
 	ASSERT3U(dlu->dlu_offset, >, 0);
 

module/zfs/spa_misc.c

@@ -1245,7 +1245,7 @@ spa_vdev_enter(spa_t *spa)
 	mutex_enter(&spa->spa_vdev_top_lock);
 	mutex_enter(&spa_namespace_lock);
 
-	ASSERT0(spa->spa_export_thread);
+	ASSERT0P(spa->spa_export_thread);
 
 	vdev_autotrim_stop_all(spa);
 
@@ -1264,7 +1264,7 @@ spa_vdev_detach_enter(spa_t *spa, uint64_t guid)
 	mutex_enter(&spa->spa_vdev_top_lock);
 	mutex_enter(&spa_namespace_lock);
 
-	ASSERT0(spa->spa_export_thread);
+	ASSERT0P(spa->spa_export_thread);
 
 	vdev_autotrim_stop_all(spa);
 

module/zfs/zio_compress.c

@@ -128,7 +128,7 @@ zio_compress_data(enum zio_compress c, abd_t *src, abd_t **dst, size_t s_len,
 	uint8_t complevel;
 	zio_compress_info_t *ci = &zio_compress_table[c];
 
-	ASSERT3U(ci->ci_compress, !=, NULL);
+	ASSERT3P(ci->ci_compress, !=, NULL);
 	ASSERT3U(s_len, >, 0);
 
 	complevel = ci->ci_level;