diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 1abb2ada591..f108ce8b6e5 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -26,13 +26,14 @@ EasyBuild support for installing a bundle of Python packages, implemented as a generic easyblock @author: Kenneth Hoste (Ghent University) +@author: Samuel Moors (Vrije Universiteit Brussel) """ import os from easybuild.easyblocks.generic.bundle import Bundle from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_DUMMY_PACKAGES, EXTS_FILTER_PYTHON_PACKAGES from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, find_python_cmd_from_ec -from easybuild.easyblocks.generic.pythonpackage import run_pip_check, set_py_env_vars +from easybuild.easyblocks.generic.pythonpackage import run_pip_check, run_pip_list, set_py_env_vars from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES from easybuild.tools.modules import get_software_root @@ -210,30 +211,43 @@ def _sanity_check_step_extensions(self): """Run the pip check for extensions if enabled""" super()._sanity_check_step_extensions() - sanity_pip_check = self.cfg['sanity_pip_check'] + params = { + 'sanity_pip_check': self.cfg['sanity_pip_check'], + 'sanity_check_pip_list': self.cfg['sanity_check_pip_list'], + } unversioned_packages = set(self.cfg['unversioned_packages']) # The options should be set in the main EC and cannot be different between extensions. # For backwards compatibility and to avoid surprises enable the pip-check if it is enabled # in the main EC or any extension and build the union of all unversioned_packages. - has_sanity_pip_check_mismatch = False all_unversioned_packages = unversioned_packages.copy() - for ext in self.ext_instances: - if isinstance(ext, PythonPackage): - if ext.cfg['sanity_pip_check'] != sanity_pip_check: - has_sanity_pip_check_mismatch = True - all_unversioned_packages.update(ext.cfg['unversioned_packages']) - - if has_sanity_pip_check_mismatch: - self.log.deprecated("For bundles of PythonPackage extensions the sanity_pip_check parameter " - "must be set at the top level, outside of exts_list", '6.0') - sanity_pip_check = True # Either the main set it or any extension enabled it - if all_unversioned_packages != unversioned_packages: - self.log.deprecated("For bundles of PythonPackage extensions the unversioned_packages parameter " - "must be set at the top level, outside of exts_list", '6.0') + py_exts = [x for x in self.ext_instances if isinstance(x, PythonPackage)] + + mismatched_params = set() + + for ext in py_exts: + for param, value in params.items(): + if ext.cfg[param] != value: + mismatched_params.add(param) + all_unversioned_packages.update(ext.cfg['unversioned_packages']) - if sanity_pip_check: - run_pip_check(python_cmd=self.python_cmd, unversioned_packages=all_unversioned_packages) + for param in params: + if param in mismatched_params: + params[param] = True # Either the main set it or any extension enabled it + + if all_unversioned_packages != unversioned_packages: + mismatched_params.add('unversioned_packages') + + for mismatch in mismatched_params: + msg = (f"For bundles of PythonPackage extensions the {mismatch} parameter " + "must be set at the top level, outside of exts_list") + self.log.deprecated(msg, '6.0') + + if params['sanity_pip_check']: + run_pip_check(python_cmd=self.python_cmd) + pkgs = [(x.name, x.version) for x in py_exts] + run_pip_list(pkgs, python_cmd=self.python_cmd, unversioned_packages=all_unversioned_packages, + check_names_versions=params['sanity_check_pip_list']) def make_module_footer(self): """ diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 847a59f9a9a..33cd95cce2a 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -31,6 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Alexander Grund (TU Dresden) +@author: Samuel Moors (Vrije Universiteit Brussel) """ import os import re @@ -42,7 +43,7 @@ import easybuild.tools.environment as env from easybuild.base import fancylogger from easybuild.easyblocks.python import EXTS_FILTER_DUMMY_PACKAGES, EXTS_FILTER_PYTHON_PACKAGES, set_py_env_vars -from easybuild.easyblocks.python import det_installed_python_packages, det_pip_version, run_pip_check +from easybuild.easyblocks.python import det_installed_python_packages, det_pip_version, run_pip_check, run_pip_list from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.default import DEFAULT_CONFIG from easybuild.framework.easyconfig.templates import PYPI_SOURCE @@ -508,6 +509,8 @@ def extra_options(extra_vars=None): 'max_py_minver': [None, "Maximum minor Python version (only relevant when using system Python)", CUSTOM], 'sanity_pip_check': [True, "Run 'python -m pip check' to ensure all required Python packages are " "installed and check for any package with an invalid (0.0.0) version.", CUSTOM], + 'sanity_check_pip_list': [None, "Run 'python -m pip list' to ensure specified package names and versions " + "are correct.", CUSTOM], 'runtest': [True, "Run unit tests.", CUSTOM], # overrides default 'testinstall': [False, "Install into temporary directory prior to running the tests.", CUSTOM], 'unpack_sources': [None, "Unpack sources prior to build/install. Defaults to 'True' except for whl files", @@ -521,7 +524,7 @@ def extra_options(extra_vars=None): # see https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras 'use_pip_extras': [None, "String with comma-separated list of 'extras' to install via pip", CUSTOM], 'use_pip_for_deps': [False, "Install dependencies using '%s'" % PIP_INSTALL_CMD, CUSTOM], - 'use_pip_requirement': [False, "Install using 'python -m pip install --requirement'. The sources is " + + 'use_pip_requirement': [False, "Install using 'python -m pip install --requirement'. The sources is " "expected to be the requirements file.", CUSTOM], 'zipped_egg': [False, "Install as a zipped eggs", CUSTOM], }) @@ -1048,8 +1051,8 @@ def test_step(self, return_output_ec=False): # Requires having the installation in place to work correctly, since no path configuration files # will be found otherwise if self.should_use_ebpythonprefixes(): - extrapath += "export EBPYTHONPREFIXES=%s && " % os.pathsep.join([self.pypkg_test_installdir] + - ['$EBPYTHONPREFIXES']) + extrapath += "export EBPYTHONPREFIXES=%s && " % os.pathsep.join([self.pypkg_test_installdir] + + ['$EBPYTHONPREFIXES']) if self.testcmd: testcmd = self.testcmd % {'python': self.python_cmd} cmd = ' '.join([ @@ -1242,7 +1245,7 @@ def sanity_check_step(self, *args, **kwargs): if self.multi_python: # when installing for multiple Python versions, we must use 'python', not a full-path 'python' command! - pip_check_python_cmd = 'python' + python_cmd = 'python' if 'exts_filter' not in kwargs: kwargs.update({'exts_filter': exts_sanity_filter}) else: @@ -1250,26 +1253,29 @@ def sanity_check_step(self, *args, **kwargs): # (which is required especially when installing with system Python) if self.python_cmd is None: self.prepare_python() - pip_check_python_cmd = self.python_cmd + python_cmd = self.python_cmd if 'exts_filter' not in kwargs: exts_filter = (exts_sanity_filter[0].replace('python', self.python_cmd), exts_sanity_filter[1]) kwargs.update({'exts_filter': exts_filter}) # inject extra '%(python)s' template value for use by sanity check commands - self.cfg.template_values['python'] = pip_check_python_cmd + self.cfg.template_values['python'] = python_cmd + + params = { + 'sanity_pip_check': self.cfg.get('sanity_pip_check', True), + 'sanity_check_pip_list': self.cfg.get('sanity_check_pip_list'), + } - sanity_pip_check = self.cfg.get('sanity_pip_check', True) if self.is_extension: - sanity_pip_check_main = self.master.cfg.get('sanity_pip_check') - if sanity_pip_check_main is not None: - # If the main easyblock (e.g. PythonBundle) defines the variable - # we trust it does the pip check if requested and checks for mismatches - sanity_pip_check = False - self.log.info(f"Sanity 'pip check' disabled for {self.name} extension, " - f"assuming that parent will take care of it" - ) - - if sanity_pip_check: + for key in params: + if self.master.cfg.get(key) is not None: + # If the main easyblock (e.g. PythonBundle) defines the variable + # we trust it does the 'pip check' or 'pip list' if requested and checks for mismatches + params[key] = False + self.log.info(f"'{key}' disabled for {self.name} extension, " + f"assuming that parent will take care of it") + + if params['sanity_pip_check']: if not self.is_extension: # for stand-alone Python package installations (not part of a bundle of extensions), # the (fake or real) module file must be loaded at this point, @@ -1280,9 +1286,12 @@ def sanity_check_step(self, *args, **kwargs): self.log.debug("Currently loaded modules: %s", loaded_modules) raise EasyBuildError("%s module is not loaded, this should never happen...", self.short_mod_name) + run_pip_check(python_cmd=python_cmd) unversioned_packages = self.cfg.get('unversioned_packages', []) - run_pip_check(python_cmd=pip_check_python_cmd, unversioned_packages=unversioned_packages) + pkgs = [(self.name, self.version)] + run_pip_list(pkgs, python_cmd=python_cmd, unversioned_packages=unversioned_packages, + check_names_versions=params['sanity_check_pip_list']) # ExtensionEasyBlock handles loading modules correctly for multi_deps, so we clean up fake_mod_data # and let ExtensionEasyBlock do its job diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 1883d90ceb3..172cde0dffe 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -31,7 +31,9 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +@author: Samuel Moors (Vrije Universiteit Brussel) """ +import difflib import glob import json import os @@ -74,6 +76,8 @@ 'PIP_DISABLE_PIP_VERSION_CHECK': 'true', } +REGEX_PIP_NORMALIZE = re.compile(r"[-_.]+") + # We want the following import order: # 1. Packages installed into VirtualEnv # 2. Packages installed into $EBPYTHONPREFIXES (e.g. our modules) @@ -176,28 +180,26 @@ def det_installed_python_packages(names_only=True, python_cmd=None): return [pkg['name'] for pkg in pkgs] if names_only else pkgs -def run_pip_check(python_cmd=None, unversioned_packages=None): +def run_pip_check(python_cmd=None, **kwargs): """ Check installed Python packages using 'pip check' - :param unversioned_packages: set of Python packages to exclude in the version existence check :param python_cmd: Python command to use (if None, 'python' is used) """ - log = fancylogger.getLogger('det_installed_python_packages', fname=False) + log = fancylogger.getLogger('run_pip_check', fname=False) - if python_cmd is None: - python_cmd = 'python' + kwargs_keys = kwargs.keys() + if 'unversioned_packages' in kwargs_keys: + msg = ("Parameter unversioned_packages is no longer supported in run_pip_check, " + "it has been moved to run_pip_list.") + log.deprecated(msg, '6.0') + kwargs_keys -= {'unversioned_packages'} - if unversioned_packages is None: - unversioned_packages = set() - elif isinstance(unversioned_packages, (list, tuple)): - unversioned_packages = set(unversioned_packages) - elif not isinstance(unversioned_packages, set): - raise EasyBuildError("Incorrect value type for 'unversioned_packages' in run_pip_check: %s", - type(unversioned_packages)) + if kwargs_keys: + raise EasyBuildError("Parameter(s) used which are not supported in run_pip_check: " + ', '.join(kwargs_keys)) - if build_option('ignore_pip_unversioned_pkgs'): - unversioned_packages.update(build_option('ignore_pip_unversioned_pkgs')) + if python_cmd is None: + python_cmd = 'python' pip_check_cmd = f"{python_cmd} -m pip check" @@ -219,43 +221,143 @@ def run_pip_check(python_cmd=None, unversioned_packages=None): trace_msg(msg + 'OK') log.info(f"`{pip_check_cmd}` passed successfully") - # Also check for a common issue where the package version shows up as 0.0.0 often caused - # by using setup.py as the installation method for a package which is released as a generic wheel - # named name-version-py2.py3-none-any.whl. `tox` creates those from version controlled source code - # so it will contain a version, but the raw tar.gz does not. - pkgs = det_installed_python_packages(names_only=False, python_cmd=python_cmd) - faulty_version = '0.0.0' - faulty_pkg_names = sorted([pkg['name'] for pkg in pkgs if pkg['version'] == faulty_version]) + if pip_check_errors: + raise EasyBuildError('\n'.join(pip_check_errors)) + + +def normalize_pip(name): + """ + Normalize pip package name according to + https://packaging.python.org/en/latest/specifications/name-normalization/ + """ + return REGEX_PIP_NORMALIZE.sub('-', name).lower() + - for unversioned_package in sorted(unversioned_packages): +def run_pip_list(pkgs, python_cmd=None, unversioned_packages=None, check_names_versions=None): + """ + Run pip list to verify normalized names and versions of installed Python packages + + :param pkgs: list of package tuples (name, version) as specified in the easyconfig + :param python_cmd: Python command to use (if None, 'python' is used) + :param unversioned_packages: set of Python packages to exclude in the version existence check + :param check_names_versions: boolean to indicate whether name and versions of Python packages should be checked + """ + + log = fancylogger.getLogger('run_pip_list', fname=False) + + if unversioned_packages is None: + unversioned_packages = set() + elif isinstance(unversioned_packages, (list, tuple)): + unversioned_packages = set(unversioned_packages) + elif not isinstance(unversioned_packages, set): + raise EasyBuildError("Incorrect value type for 'unversioned_packages' in run_pip_list: %s", + type(unversioned_packages)) + + if build_option('ignore_pip_unversioned_pkgs'): + unversioned_packages.update(build_option('ignore_pip_unversioned_pkgs')) + + if check_names_versions is None: + # by default only check names and versions if --upload-test-report is used, + # so we can enforce that extension names/versions are correct for contributions + if build_option('upload_test_report'): + check_names_versions = True + else: + check_names_versions = False + + pip_list_errors = [] + + msg = "Check on installed Python package names and versions with 'pip list': " + try: + pip_pkgs_dict = det_installed_python_packages(names_only=False, python_cmd=python_cmd) + trace_msg(msg + 'OK') + log.info("pip list cmd passed successfully") + except EasyBuildError as err: + trace_msg(msg + 'FAIL') + raise EasyBuildError(f"pip list cmd failed:\n{err}") + + if unversioned_packages: + normalized_unversioned = {normalize_pip(x) for x in unversioned_packages} + else: + normalized_unversioned = set() + + # Create normalized name: version mapping from pip list output + normalized_pip_pkgs = {normalize_pip(x['name']): x['version'] for x in pip_pkgs_dict} + + # Check for packages that likely were not installed correctly (version '0.0.0'), excluding packages that are listed + # as "unversioned". This is a common issue caused by using setup.py as the installation method for a package which + # is released as a generic wheel named name-version-py2.py3-none-any.whl. `tox` creates those from version + # controlled source code so it will contain a version, but the raw tar.gz does not. + zero_version = '0.0.0' + zero_pkg_names = sorted([name for (name, version) in normalized_pip_pkgs.items() if version == zero_version]) + + for unversioned_package in sorted(normalized_unversioned): try: - faulty_pkg_names.remove(unversioned_package) + zero_pkg_names.remove(unversioned_package) log.debug(f"Excluding unversioned package '{unversioned_package}' from check") except ValueError: try: - version = next(pkg['version'] for pkg in pkgs if pkg['name'] == unversioned_package) - except StopIteration: + version = normalized_pip_pkgs[unversioned_package] + except KeyError: msg = f"Package '{unversioned_package}' in unversioned_packages was not found in " msg += "the installed packages. Check that the name from `python -m pip list` is used " msg += "which may be different than the module name." else: msg = f"Package '{unversioned_package}' in unversioned_packages has a version of {version} " msg += "which is valid. Please remove it from unversioned_packages." - pip_check_errors.append(msg) + pip_list_errors.append(msg) - log.info("Found %s invalid packages out of %s packages", len(faulty_pkg_names), len(pkgs)) - if faulty_pkg_names: - faulty_pkg_names_str = '\n'.join(faulty_pkg_names) + log.info("Found %s invalid packages out of %s packages", len(zero_pkg_names), len(normalized_pip_pkgs)) + if zero_pkg_names: + zero_pkg_names_str = '\n'.join(zero_pkg_names) msg = "The following Python packages were likely not installed correctly because they show a " - msg += f"version of '{faulty_version}':\n{faulty_pkg_names_str}\n" + msg += f"version of '{zero_version}':\n{zero_pkg_names_str}\n" msg += "This may be solved by using a *-none-any.whl file as the source instead. " msg += "See e.g. the SOURCE*_WHL templates.\n" msg += "Otherwise you could check if the package provides a version at all or if e.g. poetry is " msg += "required (check the source for a pyproject.toml and see PEP517 for details on that)." - pip_check_errors.append(msg) + pip_list_errors.append(msg) - if pip_check_errors: - raise EasyBuildError('\n'.join(pip_check_errors)) + if check_names_versions: + normalized_pkgs = [(normalize_pip(name), version) for name, version in pkgs] + + missing_names = [] + missing_versions = [] + + for name, version in normalized_pkgs: + # Skip packages in the unversioned list: they have already been checked + if name in normalized_unversioned: + continue + + # Skip packages in the zero_pkg_names list: they have already been added to pip_list_errors + if name in zero_pkg_names: + continue + + # Check for missing (likely wrong) packages names and propose close matches + if name not in normalized_pip_pkgs: + close_matches = difflib.get_close_matches(name, normalized_pip_pkgs.keys()) + missing_names.append(f"{name} (close matches in 'pip list' output: " + ', '.join(close_matches)) + + # Check for missing (likely wrong) package versions + elif version != normalized_pip_pkgs[name]: + missing_versions.append(f"{name} {version} (version in 'pip list' output: {normalized_pip_pkgs[name]})") + + log.info(f"Found {len(missing_names)} missing names and {len(missing_versions)} missing versions " + f"out of {len(pkgs)} packages") + + if missing_names: + missing_names_str = '\n'.join(missing_names) + msg = "The following Python packages were likely specified with a wrong name because they are missing " + msg += f"in the 'pip list' output:\n{missing_names_str}" + pip_list_errors.append(msg) + + if missing_versions: + missing_versions_str = '\n'.join(missing_versions) + msg = "The following Python packages were likely specified with a wrong version because they have " + msg += f"another version in the 'pip list' output:\n{missing_versions_str}" + pip_list_errors.append(msg) + + if pip_list_errors: + raise EasyBuildError('\n' + '\n'.join(pip_list_errors)) def set_py_env_vars(log, verbose=False): @@ -330,10 +432,15 @@ def __init__(self, *args, **kwargs): 'pip_ignore_installed': False, # disable per-extension 'pip check', since it's a global check done in sanity check step of Python easyblock 'sanity_pip_check': False, + # disable per-extension 'pip list', since it's a global check done in sanity check step of Python easyblock + 'sanity_check_pip_list': False, # EasyBuild 5 'use_pip': True, } + # build and install additional packages with PythonPackage easyblock + self.cfg['exts_defaultclass'] = "PythonPackage" + exts_default_options = self.cfg.get_ref('exts_default_options') for key, default_value in ext_defaults.items(): if key not in exts_default_options: @@ -492,8 +599,6 @@ def prepare_for_extensions(self): """ Set default class and filter for Python packages """ - # build and install additional packages with PythonPackage easyblock - self.cfg['exts_defaultclass'] = "PythonPackage" self.cfg['exts_filter'] = EXTS_FILTER_PYTHON_PACKAGES # don't pass down any build/install options that may have been specified @@ -839,6 +944,14 @@ def sanity_check_step(self): # global 'pip check' to verify that version requirements are met for Python packages installed as extensions run_pip_check(python_cmd='python') + exts_list = self.cfg.get_ref('exts_list') + if exts_list and not self.ext_instances: + # populate self.ext_instances if not done yet (e.g. with --sanity-check-only or --rebuild --module-only) + self.init_ext_instances() + + pkgs = [(x.name, x.version) for x in self.ext_instances] + run_pip_list(pkgs, python_cmd='python') + abiflags = '' if LooseVersion(self.version) >= LooseVersion("3"): run_shell_cmd("command -v python", hidden=True) diff --git a/test/easyblocks/easyblock_specific.py b/test/easyblocks/easyblock_specific.py index 80d9059cd7a..fa00a0c33a6 100644 --- a/test/easyblocks/easyblock_specific.py +++ b/test/easyblocks/easyblock_specific.py @@ -496,13 +496,11 @@ def test_handle_local_py_install_scheme(self): self.assertTrue(os.path.exists(local_test_py)) def test_run_pip_check(self): - """Test run_pip_check function provided by PythonPackage easyblock.""" + """Test run_pip_check function provided by EB_Python easyblock.""" def mocked_run_shell_cmd_pip(cmd, **kwargs): if "pip check" in cmd: output = "No broken requirements found." - elif "pip list" in cmd: - output = '[{"name": "example", "version": "1.2.3"}]' elif "pip --version" in cmd: output = "pip 20.0" else: @@ -516,14 +514,45 @@ def mocked_run_shell_cmd_pip(cmd, **kwargs): with self.mocked_stdout_stderr(): python.run_pip_check(python_cmd=sys.executable) - # test ignored of unversioned Python packages + # inject all possible errors def mocked_run_shell_cmd_pip(cmd, **kwargs): if "pip check" in cmd: - output = "No broken requirements found." - elif "pip list" in cmd: - output = '[{"name": "zero", "version": "0.0.0"}]' + output = "foo-1.2.3 requires bar-4.5.6, which is not installed." + exit_code = 1 elif "pip --version" in cmd: output = "pip 20.0" + exit_code = 0 + else: + # unexpected command + return None + + return RunShellCmdResult(cmd=cmd, exit_code=exit_code, output=output, stderr=None, work_dir=None, + out_file=None, err_file=None, cmd_sh=None, thread_id=None, task_id=None) + + python.run_shell_cmd = mocked_run_shell_cmd_pip + error_pattern = '\n'.join([ + "pip check.*failed.*", + "foo.*requires.*bar.*not installed.*", + ]) + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, error_pattern, python.run_pip_check, + python_cmd=sys.executable) + + # invalid pip version + def mocked_run_shell_cmd_pip(cmd, **kwargs): + return RunShellCmdResult(cmd=cmd, exit_code=0, output="1.2.3", stderr=None, work_dir=None, + out_file=None, err_file=None, cmd_sh=None, thread_id=None, task_id=None) + + python.run_shell_cmd = mocked_run_shell_cmd_pip + error_pattern = "Failed to determine pip version!" + self.assertErrorRegex(EasyBuildError, error_pattern, python.run_pip_check, python_cmd=sys.executable) + + def test_run_pip_list(self): + """Test run_pip_list function provided by EB_Python easyblock.""" + + def mocked_run_shell_cmd_pip(cmd, **kwargs): + if "pip list" in cmd: + output = '[{"name": "example", "version": "1.2.3"}]' else: # unexpected command return None @@ -533,22 +562,31 @@ def mocked_run_shell_cmd_pip(cmd, **kwargs): python.run_shell_cmd = mocked_run_shell_cmd_pip with self.mocked_stdout_stderr(): - python.run_pip_check(python_cmd=sys.executable, unversioned_packages=('zero', )) + python.run_pip_list([], python_cmd=sys.executable) + # test ignored unversioned Python packages + def mocked_run_shell_cmd_pip(cmd, **kwargs): + if "pip list" in cmd: + output = '[{"name": "zero", "version": "0.0.0"}, {"name": "example-pkg", "version": "1.2.3"}]' + else: + # unexpected command + return None + + return RunShellCmdResult(cmd=cmd, exit_code=0, output=output, stderr=None, work_dir=None, + out_file=None, err_file=None, cmd_sh=None, thread_id=None, task_id=None) + + python.run_shell_cmd = mocked_run_shell_cmd_pip with self.mocked_stdout_stderr(): - python.run_pip_check(python_cmd=sys.executable, unversioned_packages={'zero'}) + python.run_pip_list([('example_pkg', '1.2.3')], python_cmd=sys.executable, unversioned_packages=('zero', )) - # inject all possible errors + with self.mocked_stdout_stderr(): + python.run_pip_list([('example.pkg', '1.2.3')], python_cmd=sys.executable, unversioned_packages={'zero'}) + + # inject all possible errors with unversioned packages def mocked_run_shell_cmd_pip(cmd, **kwargs): - if "pip check" in cmd: - output = "foo-1.2.3 requires bar-4.5.6, which is not installed." - exit_code = 1 - elif "pip list" in cmd: + if "pip list" in cmd: output = '[{"name": "example", "version": "1.2.3"}, {"name": "wrong", "version": "0.0.0"}]' exit_code = 0 - elif "pip --version" in cmd: - output = "pip 20.0" - exit_code = 0 else: # unexpected command return None @@ -558,25 +596,38 @@ def mocked_run_shell_cmd_pip(cmd, **kwargs): python.run_shell_cmd = mocked_run_shell_cmd_pip error_pattern = '\n'.join([ - "pip check.*failed.*", - "foo.*requires.*bar.*not installed.*", r"Package 'example'.*version of 1\.2\.3 which is valid.*", "Package 'nosuchpkg' in unversioned_packages was not found in the installed packages.*", r".*not installed correctly.*version of '0\.0\.0':", "wrong", ]) with self.mocked_stdout_stderr(): - self.assertErrorRegex(EasyBuildError, error_pattern, python.run_pip_check, + self.assertErrorRegex(EasyBuildError, error_pattern, python.run_pip_list, [], python_cmd=sys.executable, unversioned_packages=['example', 'nosuchpkg']) - # invalid pip version + # inject errors with mismatched packages name or version def mocked_run_shell_cmd_pip(cmd, **kwargs): - return RunShellCmdResult(cmd=cmd, exit_code=0, output="1.2.3", stderr=None, work_dir=None, + if "pip list" in cmd: + output = '[{"name": "example", "version": "1.2.3"}, {"name": "wrong-version", "version": "1.1.1"}]' + exit_code = 0 + else: + # unexpected command + return None + + return RunShellCmdResult(cmd=cmd, exit_code=exit_code, output=output, stderr=None, work_dir=None, out_file=None, err_file=None, cmd_sh=None, thread_id=None, task_id=None) python.run_shell_cmd = mocked_run_shell_cmd_pip - error_pattern = "Failed to determine pip version!" - self.assertErrorRegex(EasyBuildError, error_pattern, python.run_pip_check, python_cmd=sys.executable) + error_pattern = '\n'.join([ + r"The following Python packages were likely specified with a wrong name because they are missing.*", + r"wrong-name.*", + r"The following Python packages were likely specified with a wrong version.*", + r"wrong-version 5.6.7.*", + ]) + with self.mocked_stdout_stderr(): + self.assertErrorRegex(EasyBuildError, error_pattern, python.run_pip_list, + [('wrong_name', '1.2.3'), ('wrong_version', '5.6.7')], + python_cmd=sys.executable, check_names_versions=True) def test_symlink_dist_site_packages(self): """Test symlink_dist_site_packages provided by PythonPackage easyblock."""