don't embed Python code into libwallycore.so

Embedding SWIG Python code into libwallycore.so ties the library to a
specific version of Python, thereby precluding concurrent installation
of the wallycore Python module for multiple implementations of Python
on the same system if the Python modules are linked with a system-wide
libwallycore.so. On Gentoo at least, linking libwallycore.so with
libpython3.14.so and then attempting to use the wallycore Python module
from Python 3.11, 3.12, or 3.13 causes an immediate segfault.

Rather than injecting Python-specific glue into libwallycore.so, we can
and should keep it contained within the Python native extension library
that we build for each Python implementation. The Python wheel actually
already compiles swig_python_wrap.c and links it into the native
extension library, so linking it into libwallycore.so as well is
redundant and harmful.

* Remove libswig_python.la from libwallycore_la_LIBADD, and expunge its
  existence from Makefile.am entirely since it's now unused.

* The Python wheel build for each Python implementation compiles
  swig_python_wrap.c against the headers for that particular Python
  implementation, ensuring that the resulting native extension library
  matches the ABI of the Python implementation for which it is
  installed. This may be a different Python implementation than the one
  found by Autoconf, the only relevance of which now is in finding the
  interpreter with which to run the tests.

* Since we no longer link libwallycore.so with libpython*.so, there
  should no longer be any manylinux compatibility issues, so drop the
  --enable-python-manylinux Autoconf option and the PYTHON_MANYLINUX
  Automake conditional. This also means that the test programs no longer
  need to link with $(PYTHON_LIBS) since libwallycore.la now never has
  any dependence on Python (implicit or explicit).

* Note that the Python native extension libraries do not explicitly link
  with libpython*.so either. This is correct, as the dynamic linker
  resolves their undefined Py* symbols when it loads them into the
  Python interpreter.

* Remove check-libwallycore and check-swig-java prerequisites from the
  check-swig-python target, as these are three orthogonal test suites
  with no interdependencies.

After applying these changes, the Gentoo ebuild for libwally-core 1.5.4
now successfully runs the SWIG Python tests on all currently supported
versions of Python (3.12 through 3.14), all using the same
libwallycore.so.6.

Author: whitslack

README.md

@@ -73,9 +73,6 @@ $ brew install swig
 - `--enable-swig-python`. Enable the [SWIG](http://www.swig.org/) Python
    interface. The resulting shared library can be imported from Python using
    the generated interface file `src/swig_python/wallycore/__init__.py`. (default: no).
-- `--enable-python-manylinux`. Enable [manylinux](https://github.com/pypa/manylinux)
-   support for building [PyPI](https://pypi.org/) compatible python wheels. Using
-   the resulting library in non-python programs requires linking with `libpython.so`.
 - `--enable-swig-java`. Enable the [SWIG](http://www.swig.org/) Java (JNI)
    interface. After building, see `src/swig_java/src/com/blockstream/libwally/Wally.java`
    for the Java interface definition (default: no).

configure.ac

@@ -338,11 +338,10 @@ AC_SUBST([libsecp256k1_LIBS])
 #
 # Python facilities
 #
-AC_ARG_ENABLE(python-manylinux,
-    AS_HELP_STRING([--enable-python-manylinux],[enable manylinux Python compatibility (default: no)]),
-    [python_manylinux=$enableval], [python_manylinux=no])
-AM_CONDITIONAL([PYTHON_MANYLINUX], [test "x$python_manylinux" = "xyes"])
-
+dnl We set the 'optional' flag here because manylinux cibuildwheel containers
+dnl don't include libpython*.so, and thus AX_PYTHON_DEVEL fails its final sanity
+dnl check. We actually don't link against libpython*.so anyway, so all we care
+dnl about is that we get a good value of $PYTHON_CPPFLAGS to pass to SWIG.
 AX_PYTHON_DEVEL([>= '3.9.0'], [true])
 AM_CONDITIONAL([HAVE_PYTHON], [test "x$ax_python_devel_found" = "xyes"])
 
@@ -365,10 +364,10 @@ AC_ARG_ENABLE(swig-python,
     [swig_python=$enableval], [swig_python=no])
 AM_CONDITIONAL([USE_SWIG_PYTHON], [test "x$swig_python" = "xyes"])
 
-AM_CONDITIONAL([RUN_PYTHON_TESTS], [test "$PYTHON" != "" -a "x$pythonexists" = "xyes"])
+AM_CONDITIONAL([RUN_PYTHON_TESTS], [test -n "$PYTHON"])
 
 if test "x$swig_python" = "xyes"; then
-    if test "x$pythonexists" != "xyes"; then
+    if test -z "$PYTHON_CPPFLAGS"; then
         AC_MSG_FAILURE([ERROR: No usable Python was found for swig-python])
     fi
     if test "x$elements_abi" = "xno"; then

setup.py

@@ -62,7 +62,7 @@ def _call(args, cwd=ABS_PATH):
     _call(['git', 'submodule', 'update', '--init', '--recursive'])
 
 CONFIGURE_ARGS = [
-    '--with-pic', '--enable-swig-python', '--enable-python-manylinux',
+    '--with-pic', '--enable-swig-python',
     '--disable-swig-java', '--disable-tests', '--disable-dependency-tracking'
 ]
 

src/Makefile.am

@@ -38,15 +38,7 @@ TOOLS_EXTRA_ARGS :=
 endif
 
 if USE_SWIG_PYTHON
-noinst_LTLIBRARIES += libswig_python.la
-libswig_python_la_SOURCES = swig_python/swig_python_wrap.c
-
-libswig_python_la_CFLAGS = -I$(top_srcdir) $(libsecp256k1_CFLAGS) $(AM_CFLAGS) $(SWIG_PYTHON_CPPFLAGS) $(SWIG_WARN_CFLAGS) $(NOALIAS_CFLAGS)
-if PYTHON_MANYLINUX
-else
-libswig_python_la_LIBADD = $(PYTHON_LIBS)
-endif # PYTHON_MANYLINUX
-
+BUILT_SOURCES = swig_python/swig_python_wrap.c
 # Append our extra wrapper code to the package
 swig_python/swig_python_wrap.c : swig_python/swig.i swig_python/python_extra.py_in
 	$(AM_V_at)$(SWIG) $(SWIG_PYTHON_OPT) $(SWIG_GEN_FLAGS) -outdir swig_python -o $@ $< && \
@@ -60,20 +52,6 @@ clean-local: clean-swig-python
 endif # USE_SWIG_PYTHON
 
 if RUN_PYTHON_TESTS
-# Python requires the shared library to be named _wallycore.so
-# for 'import' to work.
-if IS_OSX
-platform_dso_ext = dylib
-else
-if IS_MINGW
-platform_dso_ext = dll
-else
-platform_dso_ext = so
-endif # IS_MINGW
-endif # IS_OSX
-.libs/_wallycore.so: .libs/libwallycore.$(platform_dso_ext)
-	$(AM_V_at)ln -sfn libwallycore.$(platform_dso_ext) $@
-PYTHON_TEST_DEPS = .libs/_wallycore.so
 PYTHON_TEST = PYTHONDONTWRITEBYTECODE=1 $(PYTHON)
 endif
 
@@ -225,82 +203,66 @@ endif
 
 TESTS =
 noinst_PROGRAMS =
+
 if RUN_TESTS
 TESTS += test_bech32
 noinst_PROGRAMS += test_bech32
 test_bech32_SOURCES = ctest/test_bech32.c
 test_bech32_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS)
 test_bech32_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@
-if PYTHON_MANYLINUX
-test_bech32_LDADD += $(PYTHON_LIBS)
-endif
+
 TESTS += test_psbt
 noinst_PROGRAMS += test_psbt
 test_psbt_SOURCES = ctest/test_psbt.c ccan/ccan/str/hex/hex.c
 test_psbt_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS) -I$(srcdir)/ccan
 test_psbt_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@
-if PYTHON_MANYLINUX
-test_psbt_LDADD += $(PYTHON_LIBS)
-endif
+
 TESTS += test_psbt_limits
 noinst_PROGRAMS += test_psbt_limits
 test_psbt_limits_SOURCES = ctest/test_psbt_limits.c ccan/ccan/str/hex/hex.c
 test_psbt_limits_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS) -I$(srcdir)/ccan
 test_psbt_limits_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@
-if PYTHON_MANYLINUX
-test_psbt_limits_LDADD += $(PYTHON_LIBS)
-endif
+
 if USE_PTHREAD
 TESTS += test_clear
 noinst_PROGRAMS += test_clear
 test_clear_SOURCES = ctest/test_clear.c
 test_clear_CFLAGS = -I$(top_srcdir)/include $(PTHREAD_CFLAGS) $(AM_CFLAGS) $(NOOPT_CFLAGS) $(NOBUILTIN_CFLAGS)
 test_clear_LIBS = $(PTHREAD_LIBS)
 test_clear_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@
-if PYTHON_MANYLINUX
-test_clear_LDADD += $(PYTHON_LIBS)
-endif
 endif
+
 TESTS += test_coinselection
 noinst_PROGRAMS += test_coinselection
 test_coinselection_SOURCES = ctest/test_coinselection.c
 test_coinselection_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS)
 test_coinselection_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@
-if PYTHON_MANYLINUX
-test_coinselection_LDADD += $(PYTHON_LIBS)
-endif
+
 TESTS += test_tx
 noinst_PROGRAMS += test_tx
 test_tx_SOURCES = ctest/test_tx.c
 test_tx_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS)
 test_tx_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@
-if PYTHON_MANYLINUX
-test_tx_LDADD += $(PYTHON_LIBS)
-endif
+
 TESTS += test_descriptor
 noinst_PROGRAMS += test_descriptor
 test_descriptor_SOURCES = ctest/test_descriptor.c
 test_descriptor_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS)
 test_descriptor_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@
-if PYTHON_MANYLINUX
-test_descriptor_LDADD += $(PYTHON_LIBS)
-endif
+
 if BUILD_ELEMENTS
 TESTS += test_elements_tx
 noinst_PROGRAMS += test_elements_tx
 test_elements_tx_SOURCES = ctest/test_elements_tx.c
 test_elements_tx_CFLAGS = -I$(top_srcdir)/include $(AM_CFLAGS)
 test_elements_tx_LDADD = $(lib_LTLIBRARIES) @CTEST_EXTRA_STATIC@
-if PYTHON_MANYLINUX
-test_elements_tx_LDADD += $(PYTHON_LIBS)
-endif
 endif
 
 check-local: check-libwallycore check-swig-java check-swig-python
 	$(AM_V_at)! grep '^int ' $(top_srcdir)/include/*.h # Missing WALLY_CORE_API
 
 if RUN_JAVA_TESTS
-check-swig-java: $(SWIG_JAVA_TEST_DEPS) .libs/libwallycore.$(platform_dso_ext)
+check-swig-java: $(SWIG_JAVA_TEST_DEPS) libwallycore.la
 	$(AM_V_at)! grep 'native int wally_' $(sjs)/$(cblw)/Wally.java # Unwrapped Java calls
 	$(AM_V_at)! grep 'native Object wally_' $(sjs)/$(cblw)/Wally.java # Unwrapped Java calls
 if BUILD_ELEMENTS
@@ -319,7 +281,7 @@ endif # RUN_JAVA_TESTS
 
 if SHARED_BUILD_ENABLED
 if RUN_PYTHON_TESTS
-check-libwallycore: $(PYTHON_TEST_DEPS)
+check-libwallycore:
 	$(AM_V_at)$(PYTHON_TEST) test/test_address.py
 	$(AM_V_at)$(PYTHON_TEST) test/test_aes.py
 	$(AM_V_at)$(PYTHON_TEST) test/test_anti_exfil.py
@@ -356,7 +318,7 @@ if BUILD_ELEMENTS
 endif
 
 if USE_SWIG_PYTHON
-check-swig-python: check-libwallycore check-swig-java
+check-swig-python:
 	$(AM_V_at)rm -rf $(top_builddir)/venv
 	$(AM_V_at)$(PYTHON) -m virtualenv $(top_builddir)/venv
 	$(AM_V_at)$(top_builddir)/venv/bin/python -m pip install $(top_srcdir)