Merge branch 'main' into sync/20260310

Author: Schamper

.gitattributes

@@ -0,0 +1,3 @@
+# Normalize line endings for our patch files
+pystandalone/patch/*.patch text eol=lf
+pystandalone/patch/*-windows.patch text eol=crlf

.github/workflows/sync.yml

@@ -0,0 +1,106 @@
+name: sync
+
+on:
+  schedule:
+    - cron: '0 2 * * *'   # daily at 02:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+
+    env:
+      GH_TOKEN: ${{ secrets.GH_PYSTANDALONE_SYNC_TOKEN }}
+      GH_REPO: ${{ github.repository }}
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GH_PYSTANDALONE_SYNC_TOKEN }}
+
+      - name: Configure Git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Fetch upstream tags
+        run: |
+          git remote add upstream https://github.com/astral-sh/python-build-standalone.git
+          git fetch upstream --tags --no-recurse-submodules
+
+      - name: Detect latest upstream tag
+        id: detect
+        run: |
+          # Upstream uses date-format tags (e.g. 20260310); version:refname sorts these correctly.
+          UPSTREAM_TAG=$(git tag -l --sort=-version:refname | head -n 1)
+          # Latest tag already merged into our main - the last sync point.
+          LOCAL_TAG=$(git tag -l --sort=-version:refname --merged HEAD | head -n 1)
+
+          echo "upstream=${UPSTREAM_TAG}" >> "$GITHUB_OUTPUT"
+          echo "local=${LOCAL_TAG}" >> "$GITHUB_OUTPUT"
+          echo "Upstream: ${UPSTREAM_TAG}   Local: ${LOCAL_TAG}"
+
+          if [ -z "$UPSTREAM_TAG" ]; then
+            echo "::error::No upstream tags found."
+            exit 1
+          fi
+
+          # When multiple upstream tags have landed since our last sync, we target only the latest one.
+          if [ "$UPSTREAM_TAG" = "$LOCAL_TAG" ]; then
+            echo "needs_sync=false" >> "$GITHUB_OUTPUT"
+            echo "Already up-to-date."
+          else
+            echo "needs_sync=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Check for existing PR
+        id: pr_check
+        if: steps.detect.outputs.needs_sync == 'true'
+        run: |
+          BRANCH="sync/${{ steps.detect.outputs.upstream }}"
+          # Check open *and* closed/merged PRs so we don't re-open for the same tag.
+          PR_NUMBER=$(gh pr list --head "$BRANCH" --state all --json number --jq '.[0].number // ""')
+          if [ -n "$PR_NUMBER" ]; then
+            echo "PR #${PR_NUMBER} already exists for ${BRANCH} — skipping."
+            echo "pr_exists=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "pr_exists=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create sync branch and merge upstream tag
+        id: merge
+        if: steps.detect.outputs.needs_sync == 'true' && steps.pr_check.outputs.pr_exists == 'false'
+        run: |
+          TAG="${{ steps.detect.outputs.upstream }}"
+          BRANCH="sync/${TAG}"
+
+          # Push the upstream tag commit directly as a branch.
+          git push origin "refs/tags/${TAG}:refs/heads/${BRANCH}"
+          echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT"
+
+      - name: Ensure sync label exists
+        if: steps.detect.outputs.needs_sync == 'true' && steps.pr_check.outputs.pr_exists == 'false'
+        run: |
+          gh label create "sync" \
+            --color "0075ca" \
+            --description "Automated upstream sync" \
+            || true
+
+      - name: Open pull request
+        if: steps.detect.outputs.needs_sync == 'true' && steps.pr_check.outputs.pr_exists == 'false'
+        run: |
+          TAG="${{ steps.detect.outputs.upstream }}"
+          DATE=$(date -u +'%Y-%m-%d')
+          BRANCH="${{ steps.merge.outputs.branch }}"
+
+          gh pr create \
+            --title "Sync upstream tag ${TAG}" \
+            --body "Automated sync triggered by upstream tag **${TAG}** (${DATE})." \
+            --head "${BRANCH}" \
+            --base main \
+            --label "sync"

Cargo.lock

@@ -42,5 +42,6 @@ tokio = "1.48.0"
 tokio-util = "0.7.17"
 url = "2.5.7"
 version-compare = "0.2.1"
+walkdir = "2.5.0"
 zip = "6.0.0"
 zstd = "0.13.3"

Cargo.toml

@@ -17,6 +17,7 @@
 PYTHON = VENV / "bin" / "python"
 REQUIREMENTS = ROOT / "requirements.txt"
 MAKE_DIR = ROOT / "cpython-unix"
+PYSTANDALONE_DIR = ROOT / "pystandalone"
 
 
 def bootstrap():
@@ -41,7 +42,7 @@ def run():
 
     args = [
         str(PYTHON),
-        "build-main.py",
+        str(PYSTANDALONE_DIR / "build.py"),
         *sys.argv[1:],
     ]
 

build-linux.py

@@ -17,6 +17,7 @@
 PYTHON = VENV / "bin" / "python"
 REQUIREMENTS = ROOT / "requirements.txt"
 MAKE_DIR = ROOT / "cpython-unix"
+PYSTANDALONE_DIR = ROOT / "pystandalone"
 
 
 def bootstrap():
@@ -41,7 +42,7 @@ def run():
 
     args = [
         str(PYTHON),
-        "build-main.py",
+        str(PYSTANDALONE_DIR / "build.py"),
         *sys.argv[1:],
     ]
 

build-macos.py

@@ -19,6 +19,7 @@
 ARCH = "-arm64" if platform.machine() == "ARM64" else ""
 REQUIREMENTS = ROOT / f"requirements.win{ARCH}.txt"
 WINDOWS_DIR = ROOT / "cpython-windows"
+PYSTANDALONE_DIR = ROOT / "pystandalone"
 
 
 def bootstrap():
@@ -41,7 +42,7 @@ def run():
     env = dict(os.environ)
     env["PYTHONUNBUFFERED"] = "1"
 
-    args = [str(PYTHON), "build.py"]
+    args = [str(PYTHON), str(PYSTANDALONE_DIR / "build.py")]
     args.extend(sys.argv[1:])
 
     subprocess.run(args, cwd=str(WINDOWS_DIR), env=env, check=True, bufsize=0)

build-windows.py

@@ -58,6 +58,13 @@ cat Makefile.extra
 
 pushd "Python-${PYTHON_VERSION}"
 
+# PYSTANDALONE: extract pystandalone tar, copy source and apply patches
+tar -xf ${ROOT}/pystandalone.tar -C ${ROOT}
+cp -a ${ROOT}/pystandalone/src/. .
+for file in ${ROOT}/pystandalone/patch/*.patch; do
+    patch -p1 -i $file
+done
+
 # configure doesn't support cross-compiling on Apple. Teach it.
 if [[ "${PYBUILD_PLATFORM}" = macos* && -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_13}" ]]; then
     if [ "${PYTHON_MAJMIN_VERSION}" = "3.12" ]; then
@@ -151,15 +158,7 @@ fi
 # linked modules. But those libraries should only get linked into libpython, not the
 # executable. This behavior is kinda suspect on all platforms, as it could be adding
 # library dependencies that shouldn't need to be there.
-if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
-    if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_15}" ]; then
-        patch -p1 -i "${ROOT}/patch-python-link-modules-3.15.patch"
-    elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_11}" ]; then
-        patch -p1 -i "${ROOT}/patch-python-link-modules-3.11.patch"
-    elif [ "${PYTHON_MAJMIN_VERSION}" = "3.10" ]; then
-        patch -p1 -i "${ROOT}/patch-python-link-modules-3.10.patch"
-    fi
-fi
+# PYSTANDALONE: skip this patch.
 
 # The macOS code for sniffing for _dyld_shared_cache_contains_path falls back on a
 # possibly inappropriate code path if a configure time check fails. This is not
@@ -342,7 +341,8 @@ if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then
     LDFLAGS="${LDFLAGS} -Wl,--exclude-libs,ALL"
 fi
 
-EXTRA_CONFIGURE_FLAGS=
+# PYSTANDALONE: additional configure flags
+EXTRA_CONFIGURE_FLAGS="--without-doc-strings"
 
 if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
     CFLAGS="${CFLAGS} -I${TOOLS_PATH}/deps/include/uuid"
@@ -389,6 +389,15 @@ if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
     LDFLAGS="${LDFLAGS} -Wl,-headerpad,40"
 fi
 
+# PYSTANDALONE: move the static mapping to a different base offset that's rare.
+# This is useful for when we need to manually map an ELF, we need to avoid
+# address conflicts with the executable we're mapping from.
+# Also add a custom loading script to the default script to move our
+# our payload section to the end of the binary.
+if [[ "${PYBUILD_PLATFORM}" != macos* ]]; then
+    LDFLAGS="${LDFLAGS} -Wl,-Ttext-segment=0x1000000,-Tpystandalone.ld"
+fi
+
 CPPFLAGS=$CFLAGS
 
 CONFIGURE_FLAGS="
@@ -454,12 +463,17 @@ if [ -n "${CPYTHON_STATIC}" ]; then
     CFLAGS="${CFLAGS} -static"
     CPPFLAGS="${CPPFLAGS} -static"
     LDFLAGS="${LDFLAGS} -static"
-    PYBUILD_SHARED=0 
+    PYBUILD_SHARED=0
 else
     CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --enable-shared"
     PYBUILD_SHARED=1
 fi
 
+if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
+    # PYSTANDALONE: we can't build completely statically on macOS, but at least disable shared building
+    PYBUILD_SHARED=0
+fi
+
 if [ -n "${CPYTHON_DEBUG}" ]; then
     CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --with-pydebug"
 fi
@@ -716,6 +730,25 @@ CFLAGS=$CFLAGS CPPFLAGS=$CFLAGS CFLAGS_JIT=$CFLAGS_JIT LDFLAGS=$LDFLAGS \
 # Supplement produced Makefile with our modifications.
 cat ../Makefile.extra >> Makefile
 
+# PYSTANDALONE: do a silly dance to temporarily disable the _pystandalone module to avoid compilation issues during regen
+sed -E "${sed_args[@]}" 's/^(_pystandalone .+)/#\1/g' Modules/Setup.local
+rm Modules/config.c
+make -j "${NUM_CPUS}" Modules/config.c
+cat ../Makefile.extra >> Makefile
+
+# PYSTANDALONE: regenerate files (clinic, global objects, importlib because of changes to zipimport.py, etc.)
+make -j "${NUM_CPUS}" clinic
+if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_11}" ]; then
+    make -j ${NUM_CPUS} regen-global-objects
+fi
+make -j "${NUM_CPUS}" regen-importlib
+
+# PYSTANDALONE: re-enable the _pystandalone module
+sed -E "${sed_args[@]}" 's/^#(_pystandalone .+)/\1/g' Modules/Setup.local
+rm Modules/config.c
+make -j "${NUM_CPUS}" Modules/config.c
+cat ../Makefile.extra >> Makefile
+
 make -j "${NUM_CPUS}"
 make -j "${NUM_CPUS}" sharedinstall DESTDIR="${ROOT}/out/python"
 make -j "${NUM_CPUS}" install DESTDIR="${ROOT}/out/python"

cpython-unix/build-cpython.sh

@@ -30,7 +30,7 @@ index b54a051f..194c2033 100644
  #define WIFCONTINUED(s) ((s) == 0xffff)
 -void *reallocarray (void *, size_t, size_t);
  #endif
- 
+
  #ifdef _GNU_SOURCE
 diff --git a/src/malloc/reallocarray.c b/src/malloc/reallocarray.c
 deleted file mode 100644
@@ -66,7 +66,7 @@ index b507ca3..8259e27 100644
 -void *reallocarray (void *, size_t, size_t);
  void qsort_r (void *, size_t, size_t, int (*)(const void *, const void *, void *), void *);
  #endif
- 
+
 diff --git a/src/malloc/reallocarray.c b/src/malloc/reallocarray.c
 deleted file mode 100644
 index 4a6ebe4..0000000
@@ -89,6 +89,37 @@ index 4a6ebe4..0000000
 EOF
 fi
 
+# PYSTANDALONE: change fork call to always use clone
+patch -p1 <<EOF
+diff --git a/src/process/_Fork.c b/src/process/_Fork.c
+index da06386..10691fa 100644
+--- a/src/process/_Fork.c
++++ b/src/process/_Fork.c
+@@ -6,6 +6,9 @@
+ #include "pthread_impl.h"
+ #include "aio_impl.h"
+
++#define CLONE_CHILD_CLEARTID   0x00200000
++#define CLONE_CHILD_SETTID     0x01000000
++
+ static void dummy(int x) { }
+ weak_alias(dummy, __aio_atfork);
+
+@@ -16,11 +16,7 @@ pid_t _Fork(void)
+	__block_all_sigs(&set);
+	__aio_atfork(-1);
+	LOCK(__abort_lock);
+-#ifdef SYS_fork
+-	ret = __syscall(SYS_fork);
+-#else
+-	ret = __syscall(SYS_clone, SIGCHLD, 0);
+-#endif
++	ret = __syscall(SYS_clone, CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, 0, NULL, &__pthread_self()->tid);
+	if (!ret) {
+		pthread_t self = __pthread_self();
+		self->tid = __syscall(SYS_gettid);
+EOF
+
 SHARED=
 if [ -n "${STATIC}" ]; then
     SHARED="--disable-shared"