enhance PythonPackage and PythonBundle to verify Python package names and versions with pip list with sanity_check_pip_list parameter

easybuild/easyblocks/generic/pythonbundle.py

@@ -26,13 +26,14 @@
 EasyBuild support for installing a bundle of Python packages, implemented as a generic easyblock
 
 @author: Kenneth Hoste (Ghent University)
+@author: Samuel Moors (Vrije Universiteit Brussel)
 """
 import os
 
 from easybuild.easyblocks.generic.bundle import Bundle
 from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_DUMMY_PACKAGES, EXTS_FILTER_PYTHON_PACKAGES
 from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, find_python_cmd_from_ec
-from easybuild.easyblocks.generic.pythonpackage import run_pip_check, set_py_env_vars
+from easybuild.easyblocks.generic.pythonpackage import run_pip_check, run_pip_list, set_py_env_vars
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES
 from easybuild.tools.modules import get_software_root
@@ -210,30 +211,43 @@ def _sanity_check_step_extensions(self):
         """Run the pip check for extensions if enabled"""
         super()._sanity_check_step_extensions()
 
-        sanity_pip_check = self.cfg['sanity_pip_check']
+        params = {
+            'sanity_pip_check': self.cfg['sanity_pip_check'],
+            'sanity_check_pip_list': self.cfg['sanity_check_pip_list'],
+        }
         unversioned_packages = set(self.cfg['unversioned_packages'])
 
         # The options should be set in the main EC and cannot be different between extensions.
         # For backwards compatibility and to avoid surprises enable the pip-check if it is enabled
         # in the main EC or any extension and build the union of all unversioned_packages.
-        has_sanity_pip_check_mismatch = False
         all_unversioned_packages = unversioned_packages.copy()
-        for ext in self.ext_instances:
-            if isinstance(ext, PythonPackage):
-                if ext.cfg['sanity_pip_check'] != sanity_pip_check:
-                    has_sanity_pip_check_mismatch = True
-                all_unversioned_packages.update(ext.cfg['unversioned_packages'])
-
-        if has_sanity_pip_check_mismatch:
-            self.log.deprecated("For bundles of PythonPackage extensions the sanity_pip_check parameter "
-                                "must be set at the top level, outside of exts_list", '6.0')
-            sanity_pip_check = True  # Either the main set it or any extension enabled it
-        if all_unversioned_packages != unversioned_packages:
-            self.log.deprecated("For bundles of PythonPackage extensions the unversioned_packages parameter "
-                                "must be set at the top level, outside of exts_list", '6.0')
+        py_exts = [x for x in self.ext_instances if isinstance(x, PythonPackage)]
+
+        mismatched_params = set()
+
+        for ext in py_exts:
+            for param, value in params.items():
+                if ext.cfg[param] != value:
+                    mismatched_params.add(param)
+            all_unversioned_packages.update(ext.cfg['unversioned_packages'])
 
-        if sanity_pip_check:
-            run_pip_check(python_cmd=self.python_cmd, unversioned_packages=all_unversioned_packages)
+        for param in params:
+            if param in mismatched_params:
+                params[param] = True  # Either the main set it or any extension enabled it
+
+        if all_unversioned_packages != unversioned_packages:
+            mismatched_params.add('unversioned_packages')
+
+        for mismatch in mismatched_params:
+            msg = (f"For bundles of PythonPackage extensions the {mismatch} parameter "
+                   "must be set at the top level, outside of exts_list")
+            self.log.deprecated(msg, '6.0')
+
+        if params['sanity_pip_check']:
+            run_pip_check(python_cmd=self.python_cmd)
+            pkgs = [(x.name, x.version) for x in py_exts]
+            run_pip_list(pkgs, python_cmd=self.python_cmd, unversioned_packages=all_unversioned_packages,
+                         check_names_versions=params['sanity_check_pip_list'])
 
     def make_module_footer(self):
         """

easybuild/easyblocks/generic/pythonpackage.py

@@ -31,6 +31,7 @@
 @author: Pieter De Baets (Ghent University)
 @author: Jens Timmerman (Ghent University)
 @author: Alexander Grund (TU Dresden)
+@author: Samuel Moors (Vrije Universiteit Brussel)
 """
 import os
 import re
@@ -42,7 +43,7 @@
 import easybuild.tools.environment as env
 from easybuild.base import fancylogger
 from easybuild.easyblocks.python import EXTS_FILTER_DUMMY_PACKAGES, EXTS_FILTER_PYTHON_PACKAGES, set_py_env_vars
-from easybuild.easyblocks.python import det_installed_python_packages, det_pip_version, run_pip_check
+from easybuild.easyblocks.python import det_installed_python_packages, det_pip_version, run_pip_check, run_pip_list
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.easyconfig.default import DEFAULT_CONFIG
 from easybuild.framework.easyconfig.templates import PYPI_SOURCE
@@ -508,6 +509,8 @@ def extra_options(extra_vars=None):
             'max_py_minver': [None, "Maximum minor Python version (only relevant when using system Python)", CUSTOM],
             'sanity_pip_check': [True, "Run 'python -m pip check' to ensure all required Python packages are "
                                        "installed and check for any package with an invalid (0.0.0) version.", CUSTOM],
+            'sanity_check_pip_list': [None, "Run 'python -m pip list' to ensure specified package names and versions "
+                                            "are correct.", CUSTOM],
             'runtest': [True, "Run unit tests.", CUSTOM],  # overrides default
             'testinstall': [False, "Install into temporary directory prior to running the tests.", CUSTOM],
             'unpack_sources': [None, "Unpack sources prior to build/install. Defaults to 'True' except for whl files",
@@ -521,7 +524,7 @@ def extra_options(extra_vars=None):
             # see https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras
             'use_pip_extras': [None, "String with comma-separated list of 'extras' to install via pip", CUSTOM],
             'use_pip_for_deps': [False, "Install dependencies using '%s'" % PIP_INSTALL_CMD, CUSTOM],
-            'use_pip_requirement': [False, "Install using 'python -m pip install --requirement'. The sources is " +
+            'use_pip_requirement': [False, "Install using 'python -m pip install --requirement'. The sources is "
                                            "expected to be the requirements file.", CUSTOM],
             'zipped_egg': [False, "Install as a zipped eggs", CUSTOM],
         })
@@ -1048,8 +1051,8 @@ def test_step(self, return_output_ec=False):
                 # Requires having the installation in place to work correctly, since no path configuration files
                 # will be found otherwise
                 if self.should_use_ebpythonprefixes():
-                    extrapath += "export EBPYTHONPREFIXES=%s && " % os.pathsep.join([self.pypkg_test_installdir] +
-                                                                                    ['$EBPYTHONPREFIXES'])
+                    extrapath += "export EBPYTHONPREFIXES=%s && " % os.pathsep.join([self.pypkg_test_installdir]
+                                                                                    + ['$EBPYTHONPREFIXES'])
             if self.testcmd:
                 testcmd = self.testcmd % {'python': self.python_cmd}
                 cmd = ' '.join([
@@ -1242,34 +1245,37 @@ def sanity_check_step(self, *args, **kwargs):
 
         if self.multi_python:
             # when installing for multiple Python versions, we must use 'python', not a full-path 'python' command!
-            pip_check_python_cmd = 'python'
+            python_cmd = 'python'
             if 'exts_filter' not in kwargs:
                 kwargs.update({'exts_filter': exts_sanity_filter})
         else:
             # 'python' is replaced by full path to active 'python' command
             # (which is required especially when installing with system Python)
             if self.python_cmd is None:
                 self.prepare_python()
-            pip_check_python_cmd = self.python_cmd
+            python_cmd = self.python_cmd
             if 'exts_filter' not in kwargs:
                 exts_filter = (exts_sanity_filter[0].replace('python', self.python_cmd), exts_sanity_filter[1])
                 kwargs.update({'exts_filter': exts_filter})
 
         # inject extra '%(python)s' template value for use by sanity check commands
-        self.cfg.template_values['python'] = pip_check_python_cmd
+        self.cfg.template_values['python'] = python_cmd
+
+        params = {
+            'sanity_pip_check': self.cfg.get('sanity_pip_check', True),
+            'sanity_check_pip_list': self.cfg.get('sanity_check_pip_list'),
+        }
 
-        sanity_pip_check = self.cfg.get('sanity_pip_check', True)
         if self.is_extension:
-            sanity_pip_check_main = self.master.cfg.get('sanity_pip_check')
-            if sanity_pip_check_main is not None:
-                # If the main easyblock (e.g. PythonBundle) defines the variable
-                # we trust it does the pip check if requested and checks for mismatches
-                sanity_pip_check = False
-                self.log.info(f"Sanity 'pip check' disabled for {self.name} extension, "
-                              f"assuming that parent will take care of it"
-                              )
-
-        if sanity_pip_check:
+            for key in params:
+                if self.master.cfg.get(key) is not None:
+                    # If the main easyblock (e.g. PythonBundle) defines the variable
+                    # we trust it does the 'pip check' or 'pip list' if requested and checks for mismatches
+                    params[key] = False
+                    self.log.info(f"'{key}' disabled for {self.name} extension, "
+                                  f"assuming that parent will take care of it")
+
+        if params['sanity_pip_check']:
             if not self.is_extension:
                 # for stand-alone Python package installations (not part of a bundle of extensions),
                 # the (fake or real) module file must be loaded at this point,
@@ -1280,9 +1286,12 @@ def sanity_check_step(self, *args, **kwargs):
                     self.log.debug("Currently loaded modules: %s", loaded_modules)
                     raise EasyBuildError("%s module is not loaded, this should never happen...",
                                          self.short_mod_name)
+            run_pip_check(python_cmd=python_cmd)
 
             unversioned_packages = self.cfg.get('unversioned_packages', [])
-            run_pip_check(python_cmd=pip_check_python_cmd, unversioned_packages=unversioned_packages)
+            pkgs = [(self.name, self.version)]
+            run_pip_list(pkgs, python_cmd=python_cmd, unversioned_packages=unversioned_packages,
+                         check_names_versions=params['sanity_check_pip_list'])
 
         # ExtensionEasyBlock handles loading modules correctly for multi_deps, so we clean up fake_mod_data
         # and let ExtensionEasyBlock do its job