Split Pex .whl into two .whls. (#3057)

We now publish two wheels:
1. `pex-<rev>.py27.py35.py36.py37.py38.py39.py310.py311-none-any.whl`:
   Targets exactly the Pythons in the python tag.
2. `pex-<rev>.py3.py312-none-any.whl`: Targets Python>=3.12.

The 1st wheel carries the same contents as the existing
`pex-<rev>.py2.py3-none-any.whl` wheel and the 2nd wheel
(`pex-<rev>.py3.py312-none-any.whl`) ships without vendored `pip`,
`setuptools`, `toml` or `tomli` since these are either unusable or
un-needed under Python>=3.12.

Fixes #1526
Fixes #1527
Fixes #1528
Fixes #1877
Fixes #2731
Fixes #2785

Author: jsirois

.github/workflows/ci.yml

@@ -72,7 +72,8 @@ jobs:
         run: |
           uv run dev-cmd package -- \
             --additional-format sdist \
-            --additional-format wheel \
+            --additional-format whl \
+            --additional-format whl-3.12-plus \
             --embed-docs \
             --clean-docs \
             --scies \

.github/workflows/release.yml

@@ -64,8 +64,15 @@ jobs:
           python-version: "3.11"
       - name: Install uv
         uses: astral-sh/setup-uv@v5
-      - name: Build sdist and wheel
-        run: uv run dev-cmd package -- --no-pex --additional-format sdist --additional-format wheel --embed-docs --clean-docs
+      - name: Build sdist and wheels
+        run: |
+          uv run dev-cmd package -- \
+            --no-pex \
+            --additional-format sdist \
+            --additional-format whl \
+            --additional-format whl-3.12-plus \
+            --embed-docs \
+            --clean-docs
       - name: Publish Pex ${{ needs.determine-tag.outputs.release-tag }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

CHANGES.md

@@ -1,5 +1,17 @@
 # Release Notes
 
+## 2.77.0
+
+This release has no fixes or new features per-se, but just changes the set of distributions that
+Pex releases to PyPI. Previously Pex released an sdist and a universal (`py2.py3-none-any`) `.whl`.
+Pex now releases two wheels in addition to the sdist. The `py3.py312-none-any.whl` targets
+Python>=3.12 and has un-needed vendored libraries elided making it bith a smaller `.whl` and less
+prone to false-positive security scan issues since unused vendored code is now omitted. The other
+wheel carries the same contents as prior and supports creating PEXes for Python 2.7 and
+Python>=3.5,<3.12.
+
+* Split Pex `.whl` into two `.whl`s. (#3057)
+
 ## 2.76.1
 
 This release fixes bootstrapping of Pips specified via `--pip-version` to respect Pex Pip

build-backend/pex_build/__init__.py

@@ -14,11 +14,25 @@
 from pex.common import safe_mkdir, safe_mkdtemp
 from pex.third_party.packaging.markers import Marker
 from pex.typing import TYPE_CHECKING
+from pex.vendor import iter_vendor_specs
 
 if TYPE_CHECKING:
     from typing import Callable, Iterator, Optional
 
-INCLUDE_DOCS = os.environ.get("__PEX_BUILD_INCLUDE_DOCS__", "False").lower() in ("1", "true")
+
+def read_bool_env(
+    name,  # type: str
+    default,  # type: bool
+):
+    # type: (...) -> bool
+    value = os.environ.get(name)
+    if value is None:
+        return default
+    return value.lower() in ("1", "true")
+
+
+INCLUDE_DOCS = read_bool_env("__PEX_BUILD_INCLUDE_DOCS__", default=False)
+WHEEL_3_12_PLUS = read_bool_env("__PEX_BUILD_WHL_3_12_PLUS__", default=False)
 
 
 @contextmanager
@@ -128,3 +142,8 @@ def modify_wheel(
                 out_dir,
             ]
         )
+    if WHEEL_3_12_PLUS:
+        for vendor_spec in iter_vendor_specs():
+            if vendor_spec.key in ("pip", "setuptools", "toml", "tomli"):
+                shutil.rmtree(os.path.join(wheel_dir, vendor_spec.relpath))
+                print("Removed un-needed vendored library", vendor_spec.relpath, file=sys.stderr)

pex/environment.py

@@ -59,7 +59,7 @@ def _import_pkg_resources():
     except ImportError:
         from pex import third_party
 
-        third_party.install(expose=["setuptools"])
+        third_party.install(expose_if_available=["setuptools"])
         try:
             import pkg_resources  # vendor:skip
 

pex/pep_425.py

@@ -49,7 +49,9 @@ class RankedTag(object):
     rank = attr.ib()  # type: TagRank
 
     def select_higher_rank(self, other):
-        # type: (RankedTag) -> RankedTag
+        # type: (Optional[RankedTag]) -> RankedTag
+        if other is None:
+            return self
         return Rank.select_highest_rank(
             self, other, extract_rank=lambda ranked_tag: ranked_tag.rank
         )

pex/pep_427.py

@@ -76,8 +76,8 @@ class InstallableType(Enum["InstallableType.Value"]):
     class Value(Enum.Value):
         pass
 
-    INSTALLED_WHEEL_CHROOT = Value("installed wheel chroot")
-    WHEEL_FILE = Value(".whl file")
+    INSTALLED_WHEEL_CHROOT = Value("installed-wheel-chroot")
+    WHEEL_FILE = Value(".whl-file")
 
 
 InstallableType.seal()
@@ -974,8 +974,9 @@ def is_entry_point_script(script_path):
         if not compile and installed_file.path.endswith(".pyc"):
             continue
 
-        src_file = os.path.realpath(os.path.join(wheel.location, installed_file.path))
-        if not os.path.exists(src_file):
+        src_file = os.path.normpath(os.path.join(wheel.location, installed_file.path))
+        src_file_realpath = os.path.realpath(src_file)
+        if not os.path.exists(src_file_realpath):
             if not warned_bad_record:
                 pex_warnings.warn(
                     "The wheel {whl} has a bad RECORD. Skipping install of non-existent file "
@@ -989,17 +990,24 @@ def is_entry_point_script(script_path):
         dst_components = None  # type: Optional[Tuple[Text, Text, bool]]
         for path_name, installed_path in wheel.iter_install_paths_by_name():
             installed_path = os.path.realpath(installed_path)
-            if installed_path == commonpath((installed_path, src_file)):
+
+            src_path = None  # type: Optional[Text]
+            if installed_path == commonpath((installed_path, src_file_realpath)):
+                src_path = src_file_realpath
+            elif installed_path == commonpath((installed_path, src_file)):
+                src_path = src_file
+
+            if src_path:
                 rewrite_script = False
                 if "scripts" == path_name:
-                    if is_entry_point_script(src_file):
+                    if is_entry_point_script(src_path):
                         # This entry point script will be installed afresh below as needed.
                         break
                     rewrite_script = interpreter is not None and is_python_script(
-                        src_file, check_executable=False
+                        src_path, check_executable=False
                     )
 
-                dst_rel_path = os.path.relpath(src_file, installed_path)
+                dst_rel_path = os.path.relpath(src_path, installed_path)
                 dst_components = path_name, dst_rel_path, rewrite_script
                 break
         else:

pex/pex_builder.py

@@ -543,7 +543,11 @@ def _prepare_bootstrap(self):
         # NB: We use pip here in the builder, but that's only at build time, and
         # although we don't use pyparsing directly, packaging.markers, which we
         # do use at runtime, does.
-        root_module_names = ["appdirs", "attr", "colors", "packaging", "pkg_resources", "pyparsing"]
+        root_module_names = ["appdirs", "attr", "colors", "packaging", "pyparsing"]
+        for vendor_spec in vendor.iter_vendor_specs():
+            if vendor_spec.key == "setuptools":
+                root_module_names.append("pkg_resources")
+
         prepared_sources = vendor.vendor_runtime(
             chroot=self._chroot,
             dest_basedir=self._pex_info.bootstrap,

pex/pip/installation.py

@@ -278,14 +278,7 @@ def bootstrap_pip():
         try:
             bootstrap_pip_version = PipVersion.for_value(pip_version.raw)
         except ValueError:
-            # Backstop with the version of Pip CPython 3.12.0 shipped with. This should be the
-            # oldest Pip we need in the Pip bootstrap process (which is only required for
-            # Python >= 3.12 which have distutils removed rendering vendored Pip unusable).
-            bootstrap_pip_version = PipVersion.v23_2
-            for rev in sorted(PipVersion.values(), reverse=True):
-                if pip_version > rev.version:
-                    bootstrap_pip_version = rev
-                    break
+            bootstrap_pip_version = PipVersionValue(pip_version.raw)
 
         pip = BootstrapPip(
             pip_venv=PipVenv(
@@ -388,7 +381,7 @@ def _resolved_installation(
             warn=False,
         )
     )
-    if bootstrap_pip_version is PipVersion.VENDORED:
+    if bootstrap_pip_version is PipVersion.VENDORED and PipVersion.VENDORED.available:
 
         def resolve_distribution_locations():
             for resolved_distribution in resolver.resolve_requirements(
@@ -555,7 +548,7 @@ def get_pip(
     pip = _PIP.get(installation)
     if pip is None:
         installation.check_python_applies()
-        if installation.version is PipVersion.VENDORED:
+        if installation.version is PipVersion.VENDORED and PipVersion.VENDORED.available:
             pip = _vendored_installation(
                 interpreter=interpreter,
                 resolver=resolver,

pex/pip/tool.py

@@ -365,7 +365,7 @@ def _calculate_resolver_version_args(
         if (
             resolver_version == ResolverVersion.PIP_2020
             and interpreter.version[0] == 2
-            and self.version.version < PipVersion.v22_3.version
+            and PipVersion.VENDORED <= self.version < PipVersion.v22_3
         ):
             yield "--use-feature"
             yield "2020-resolver"
@@ -398,7 +398,7 @@ def _spawn_pip_isolated(
             # We are not interactive.
             "--no-input",
         ]
-        if self.version < PipVersion.v25_0:
+        if PipVersion.VENDORED <= self.version < PipVersion.v25_0:
             # If we want to warn about a version of python we support, we should do it, not pip.
             # That said, the option does nothing in Pip 25.0 and is deprecated and slated for
             # removal.