Build HDF5 from source in wheels to avoid vulnerablity #687
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: github-actions | |
| on: [push, pull_request] | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| windows-2022-staticLink-cs: | |
| runs-on: windows-2022 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: zlib install | |
| run: | | |
| Invoke-WebRequest https://zlib.net/current/zlib.tar.gz -OutFile ${{ runner.temp }}\zlib.tar.gz | |
| cd ${{ runner.temp }} | |
| tar -xzf ${{ runner.temp }}\zlib.tar.gz | |
| mkdir zlib-build | |
| cd zlib-build | |
| cmake -G"Visual Studio 17 2022" -A x64 -T host=x64 -Wno-dev -Wno-deprecated -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/zlib-install ${{ runner.temp }}/zlib-1.3.1 | |
| cmake --build . --config Release -j2 | |
| cmake --build . --config Release --target INSTALL | |
| - name: Minizip static lib install | |
| run: | | |
| git clone https://github.com/F2I-Consulting/Minizip.git ${{ runner.temp }}/Minizip | |
| cd ${{ runner.temp }} | |
| mkdir minizip-build | |
| cd minizip-build | |
| cmake -G"Visual Studio 17 2022" -A x64 -T host=x64 -Wno-dev -Wno-deprecated -DBUILD_SHARED_LIBS=FALSE -DZLIB_ROOT=${{ runner.temp }}/zlib-install -DZLIB_USE_STATIC_LIBS=TRUE -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/minizip-install ${{ runner.temp }}/Minizip | |
| cmake --build . --config Release -j2 | |
| cmake --build . --config Release --target INSTALL | |
| - name: HDF5 install | |
| run: | | |
| Invoke-WebRequest https://support.hdfgroup.org/releases/hdf5/v2_0/v2_0_0/downloads/hdf5-2.0.0.zip -OutFile ${{ runner.temp }}\hdf5-2.0.0.zip | |
| cd ${{ runner.temp }} | |
| Expand-Archive ${{ runner.temp }}\hdf5-2.0.0.zip -DestinationPath ${{ runner.temp }} | |
| mkdir hdf5-build | |
| cd hdf5-build | |
| cmake -G"Visual Studio 17 2022" -A x64 -T host=x64 -Wno-dev -Wno-deprecated -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=ON -DZLIB_ROOT:PATH=${{ runner.temp }}/zlib-install -DZLIB_INCLUDE_DIR:PATH=${{ runner.temp }}/zlib-install/include -DZLIB_LIBRARY:PATH=${{ runner.temp }}/zlib-install/lib/zlibstatic.lib -DBUILD_SHARED_LIBS:BOOL=OFF -DHDF5_BUILD_FORTRAN:BOOL=OFF -DHDF5_BUILD_JAVA:BOOL=OFF -DHDF5_ENABLE_PARALLEL:BOOL=OFF -DHDF5_BUILD_CPP_LIB:BOOL=OFF -DHDF5_BUILD_HL_LIB:BOOL=OFF -DHDF5_BUILD_EXAMPLES:BOOL=OFF -DHDF5_BUILD_GENERATORS:BOOL=OFF -DHDF5_BUILD_TOOLS:BOOL=OFF -DHDF5_BUILD_UTILS:BOOL=OFF -DBUILD_TESTING:BOOL=OFF -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/hdf5-install ${{ runner.temp }}/hdf5-2.0.0 | |
| cmake --build . --config Release -j2 | |
| cmake --build . --config Release --target INSTALL | |
| - name: Boost install | |
| run: | | |
| (New-Object System.Net.WebClient).DownloadFile("https://archives.boost.io/release/1.90.0/binaries/boost_1_90_0-msvc-14.3-64.exe", "${{ runner.temp }}\boost.exe") | |
| Start-Process -Wait -FilePath "${{ runner.temp }}\boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=${{ runner.temp }}\boost-install" | |
| - name: Swig install | |
| run: | | |
| (New-Object System.Net.WebClient).DownloadFile("http://prdownloads.sourceforge.net/swig/swigwin-4.4.1.zip", "${{ runner.temp }}\swigwin-4.4.1.zip") | |
| 7z x ${{ runner.temp }}\swigwin-4.4.1.zip -o${{ runner.temp }} | |
| - name: CMake build and install | |
| run: | | |
| cd ${{ github.workspace }}/.. | |
| mkdir build | |
| cd build | |
| cmake -G"Visual Studio 17 2022" -A x64 -T host=x64 -Wno-dev -Wno-deprecated -DHDF5_ROOT=${{ runner.temp }}/hdf5-install -DHDF5_USE_STATIC_LIBRARIES=TRUE -DMINIZIP_ROOT=${{ runner.temp }}/minizip-install -DZLIB_ROOT=${{ runner.temp }}/zlib-install -DZLIB_USE_STATIC_LIBS=TRUE -DBoost_INCLUDE_DIR=${{ runner.temp }}/boost-install -DWITH_EXAMPLE=TRUE -DWITH_RESQML2_2=TRUE -DWITH_DOTNET_WRAPPING=TRUE -DSWIG_EXECUTABLE=${{ runner.temp }}/swigwin-4.4.1/swig.exe -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/fesapi-install ${{ github.workspace }} | |
| cmake --build . --config Release -j2 | |
| cmake --build . --config Release --target INSTALL | |
| - name: Add msbuild to PATH | |
| uses: microsoft/setup-msbuild@v2 | |
| with: | |
| msbuild-architecture: x64 | |
| - name: Build the C# example | |
| run: msbuild ${{ github.workspace }}/cs/example/example.sln -t:rebuild /p:Configuration=Release /p:Platform="x64" | |
| - name: Run Cpp example | |
| run: | | |
| ${{ github.workspace }}\..\build\Release\example.exe | |
| - name: Copy all dlls for C# | |
| run: | | |
| Copy-Item ${{ github.workspace }}\..\build\Release\*.dll -Destination ${{ github.workspace }}\cs\example\bin\x64\Release | |
| - name: Run C# example | |
| run: | | |
| ${{ github.workspace }}\cs\example\bin\x64\Release\example.exe | |
| ubuntu-22-java11: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed. | |
| matrix: | |
| include: [ | |
| { xcc_name: 'gcc 10', xcc_pkg: gcc-10, cc: gcc-10, cxx: g++-10 }, | |
| { xcc_name: 'gcc 12', xcc_pkg: gcc-12, cc: gcc-12, cxx: g++-12 }, | |
| { xcc_name: 'clang 13', xcc_pkg: clang-13, cc: clang-13, cxx: clang++-13 }, | |
| { xcc_name: 'clang 15', xcc_pkg: clang-15, cc: clang-15, cxx: clang++-15 }, | |
| ] | |
| env: | |
| XCC: ${{ matrix.xcc_name }} | |
| CC: ${{ matrix.cc }} | |
| CXX: ${{ matrix.cxx }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-java@v5 | |
| with: | |
| distribution: 'temurin' # See 'Supported distributions' for available options | |
| java-version: '11' | |
| - name: APT install | |
| run: | | |
| sudo apt update | |
| sudo apt install -y ${{ matrix.xcc_pkg }} libminizip-dev libboost-all-dev | |
| # Build HDF5 instead of downloading the package because the package is not built with -fPIC causing relocation issue. | |
| - name: HDF5 build and install | |
| run: | | |
| cd ${{ runner.temp }} | |
| wget --no-verbose https://support.hdfgroup.org/releases/hdf5/v2_0/v2_0_0/downloads/hdf5-2.0.0.tar.gz | |
| tar -xzf hdf5-2.0.0.tar.gz | |
| cd hdf5-2.0.0 | |
| mkdir build | |
| cd build | |
| cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE:STRING=Release -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=OFF -DHDF5_BUILD_FORTRAN:BOOL=OFF -DHDF5_BUILD_JAVA:BOOL=OFF -DHDF5_ENABLE_PARALLEL:BOOL=OFF -DHDF5_BUILD_CPP_LIB:BOOL=OFF -DHDF5_BUILD_HL_LIB:BOOL=OFF -DHDF5_BUILD_EXAMPLES:BOOL=OFF -DHDF5_BUILD_GENERATORS:BOOL=OFF -DHDF5_BUILD_TOOLS:BOOL=OFF -DHDF5_BUILD_UTILS:BOOL=OFF -DBUILD_TESTING:BOOL=OFF -DCMAKE_INSTALL_PREFIX:STRING=${{ runner.temp }}/install-hdf5 .. | |
| cmake --build . --config Release | |
| make -j4 | |
| make install | |
| - name: CMake build and install | |
| run: | | |
| cd .. | |
| mkdir build | |
| cd build | |
| cmake -DHDF5_ROOT=${{ runner.temp }}/install-hdf5 -DHDF5_USE_STATIC_LIBRARIES=TRUE -DCMAKE_BUILD_TYPE=Release -DWITH_JAVA_WRAPPING=TRUE ${{ github.workspace }} -DCMAKE_C_COMPILER=${{ matrix.cc }} -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/fesapi-install | |
| cmake --build . -j2 | |
| cmake --install . | |
| - name: Run java example | |
| run: | | |
| cd ${{ github.workspace }}/java/src | |
| javac -cp `find ${{ runner.temp }}/fesapi-install -name fesapiJava*.jar` com/f2i_consulting/example/FesapiJavaExample.java | |
| java -Djava.library.path=${{ runner.temp }}/fesapi-install/lib -cp `find ${{ runner.temp }}/fesapi-install -name fesapiJava*.jar`:. com.f2i_consulting.example.FesapiJavaExample | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ubuntu-22.04-${{ matrix.cxx }} | |
| path: ${{ runner.temp }}/fesapi-install | |
| build_wheels_windows: | |
| name: Build wheels on windows-latest | |
| runs-on: windows-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Stub `setup.py` check | |
| # It will be generated during CMake run | |
| # https://github.com/pypa/cibuildwheel/issues/1139 | |
| run: touch python/setup.py | |
| - name: Build wheels | |
| uses: pypa/cibuildwheel@v3.2.1 | |
| env: | |
| CIBW_BUILD: cp38-win_amd64 cp39-win_amd64 cp310-win_amd64 cp311-win_amd64 cp312-win_amd64 cp313-win_amd64 | |
| CIBW_ARCHS: auto64 | |
| CIBW_BEFORE_ALL: > | |
| %VCPKG_INSTALLATION_ROOT%\vcpkg install boost-uuid minizip hdf5[zlib] && | |
| cd ${{ runner.temp }} && | |
| mkdir fesapi-build && | |
| cd fesapi-build && | |
| cmake -DCMAKE_TOOLCHAIN_FILE=%VCPKG_INSTALLATION_ROOT%\scripts\buildsystems\vcpkg.cmake -G"Visual Studio 17 2022" -A x64 -T host=x64 -Wno-dev -Wno-deprecated -DWITH_PYTHON_WRAPPING=TRUE -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/fesapi-install ${{ github.workspace }} && | |
| cmake --build . --config Release -j2 && | |
| cmake --build . --config Release --target INSTALL && | |
| pip install delvewheel | |
| CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: > | |
| delvewheel repair --add-path ${{ runner.temp }}\fesapi-build\Release --namespace-pkg fesapi -w {dest_dir} {wheel} | |
| CIBW_TEST_COMMAND: python ${{github.workspace}}\python\example\example.py | |
| with: | |
| package-dir: ./python | |
| output-dir: wheelhouse | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: cibw-wheels-windows | |
| path: ./wheelhouse/*.whl | |
| build_wheels_linux: | |
| name: Build wheels on ubuntu-latest | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Stub `setup.py` check | |
| # It will be generated during CMake run | |
| # https://github.com/pypa/cibuildwheel/issues/1139 | |
| run: touch python/setup.py | |
| - name: Build wheels | |
| # Cannot use a more recent version than v2.22.0 because of fetpapi wheel which uses AVRO which cannot be built with GNU 14. | |
| uses: pypa/cibuildwheel@v2.22.0 | |
| # RedHat nor Debian maintain security patches for hdf5. We consequently build the latest HDF5 version. | |
| env: | |
| CIBW_BUILD: cp38-manylinux_* cp39-manylinux_* cp310-manylinux_* cp311-manylinux_* cp312-manylinux_* cp313-manylinux_* | |
| CIBW_ARCHS: auto64 | |
| CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 | |
| CIBW_BEFORE_ALL: > | |
| yum install -y wget gcc-c++ boost-devel && | |
| yum search epel-release && | |
| yum info epel-release && | |
| yum install -y epel-release && | |
| yum --enablerepo=epel install -y minizip1.2-devel cmake3 && | |
| wget --no-verbose https://support.hdfgroup.org/releases/hdf5/v2_0/v2_0_0/downloads/hdf5-2.0.0.tar.gz && | |
| tar -xzf hdf5-2.0.0.tar.gz && | |
| cd hdf5-2.0.0 && | |
| mkdir build && | |
| cd build && | |
| cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE:STRING=Release -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=OFF -DHDF5_BUILD_FORTRAN:BOOL=OFF -DHDF5_BUILD_JAVA:BOOL=OFF -DHDF5_ENABLE_PARALLEL:BOOL=OFF -DHDF5_BUILD_CPP_LIB:BOOL=OFF -DHDF5_BUILD_HL_LIB:BOOL=OFF -DHDF5_BUILD_EXAMPLES:BOOL=OFF -DHDF5_BUILD_GENERATORS:BOOL=OFF -DHDF5_BUILD_TOOLS:BOOL=OFF -DHDF5_BUILD_UTILS:BOOL=OFF -DBUILD_TESTING:BOOL=OFF -DCMAKE_INSTALL_PREFIX:STRING=${{ github.workspace }}/../hdf5-install .. && | |
| cmake --build . --config Release && | |
| make -j4 && | |
| make install && | |
| cd ../.. && | |
| mkdir build && | |
| cd build && | |
| cmake3 -DCMAKE_BUILD_TYPE=Release -DHDF5_ROOT=${{ github.workspace }}/../hdf5-install -DWITH_PYTHON_WRAPPING=TRUE -DCMAKE_INSTALL_PREFIX:STRING=/fesapi-install {project} && | |
| cmake3 --build . -j2 --config Release && | |
| cmake3 --install . | |
| CIBW_REPAIR_WHEEL_COMMAND_LINUX: > | |
| export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/fesapi-install/lib64 && | |
| auditwheel repair -w {dest_dir} {wheel} | |
| CIBW_TEST_COMMAND: python {project}/python/example/example.py | |
| with: | |
| package-dir: ./python | |
| output-dir: wheelhouse | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: cibw-wheels-manylinux_2_28 | |
| path: ./wheelhouse/*.whl | |
| build_wheels_mac: | |
| name: Build wheels on macos-14 | |
| runs-on: macos-14 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Stub `setup.py` check | |
| # It will be generated during CMake run | |
| # https://github.com/pypa/cibuildwheel/issues/1139 | |
| run: touch python/setup.py | |
| - name: Build wheels | |
| uses: pypa/cibuildwheel@v3.2.1 | |
| env: | |
| CIBW_BUILD: cp38-macosx_* cp39-macosx_* cp310-macosx_* cp311-macosx_* cp312-macosx_* cp313-macosx_* | |
| CIBW_ARCHS: auto64 | |
| # See https://cibuildwheel.pypa.io/en/stable/cpp_standards/#macos-and-deployment-target-versions | |
| MACOSX_DEPLOYMENT_TARGET: 11.0 | |
| # Dont use brew for dependencies https://github.com/pypa/cibuildwheel/issues/1251#issuecomment-1234553537 | |
| CIBW_BEFORE_ALL: > | |
| cd ${{ github.workspace }}/.. && | |
| wget --no-verbose https://archives.boost.io/release/1.90.0/source/boost_1_90_0.tar.gz && | |
| tar xf boost_1_90_0.tar.gz && | |
| git clone https://github.com/F2I-Consulting/Minizip.git ${{ github.workspace }}/../minizip && | |
| mkdir ${{ github.workspace }}/../minizip-build && | |
| cd ${{ github.workspace }}/../minizip-build && | |
| cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/../minizip-install ${{ github.workspace }}/../minizip && | |
| cmake --build . -j2 --config Release && | |
| cmake --install . && | |
| cd ${{ github.workspace }}/.. && | |
| wget --no-verbose https://support.hdfgroup.org/releases/hdf5/v2_0/v2_0_0/downloads/hdf5-2.0.0.tar.gz && | |
| tar xf hdf5-2.0.0.tar.gz && | |
| mkdir hdf5-build && | |
| cd hdf5-build && | |
| cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS:BOOL=OFF -DHDF5_BUILD_FORTRAN:BOOL=OFF -DHDF5_BUILD_JAVA:BOOL=OFF -DHDF5_ENABLE_PARALLEL:BOOL=OFF -DHDF5_BUILD_CPP_LIB:BOOL=OFF -DHDF5_BUILD_HL_LIB:BOOL=OFF -DHDF5_BUILD_EXAMPLES:BOOL=OFF -DHDF5_BUILD_GENERATORS:BOOL=OFF -DHDF5_BUILD_TOOLS:BOOL=OFF -DHDF5_BUILD_UTILS:BOOL=OFF -DBUILD_TESTING:BOOL=OFF -DCMAKE_INSTALL_PREFIX:STRING=${{ github.workspace }}/../hdf5-install ${{ github.workspace }}/../hdf5-2.0.0 && | |
| cmake --build . -j2 --config Release && | |
| cmake --install . && | |
| brew install swig && | |
| mkdir ${{ github.workspace }}/../build && | |
| cd ${{ github.workspace }}/../build && | |
| cmake -DCMAKE_BUILD_TYPE=Release -DBOOST_ROOT=${{ github.workspace }}/../boost_1_90_0 -DMINIZIP_ROOT=${{ github.workspace }}/../minizip-install -DHDF5_ROOT=${{ github.workspace }}/../hdf5-install -DHDF5_USE_STATIC_LIBRARIES=TRUE -DWITH_PYTHON_WRAPPING=TRUE -DCMAKE_INSTALL_PREFIX:STRING=${{ github.workspace }}/../fesapi-install ${{ github.workspace }} && | |
| cmake --build . -j2 --config Release && | |
| cmake --install . | |
| # See https://cibuildwheel.pypa.io/en/stable/faq/#macos-passing-dyld_library_path-to-delocate | |
| CIBW_REPAIR_WHEEL_COMMAND_MACOS: > | |
| DYLD_LIBRARY_PATH=${{ github.workspace }}/../fesapi-install/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} | |
| CIBW_TEST_COMMAND: python ${{ github.workspace }}/python/example/example.py | |
| with: | |
| package-dir: ./python | |
| output-dir: wheelhouse | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: cibw-wheels-macosx_arm64 | |
| path: ./wheelhouse/*.whl | |
| ubuntu-20-mpi: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| - name: APT install | |
| run: | | |
| sudo apt update | |
| sudo apt install -y libhdf5-mpi-dev libminizip-dev libboost-all-dev | |
| - name: Setup MPI | |
| uses: mpi4py/setup-mpi@v1 | |
| - name: CMake build and install | |
| run: | | |
| cd ${{ runner.temp }} | |
| mkdir build | |
| cd build | |
| cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_C_COMPILER=mpicc -DHDF5_PREFER_PARALLEL:BOOL=TRUE -DWITH_EXAMPLE:BOOL=TRUE -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/fesapi-install ${{ github.workspace }} | |
| cmake --build . -j2 | |
| cmake --install . | |
| - name: Run example | |
| run: mpiexec -n 2 ${{ runner.temp }}/fesapi-install/example | |
| ubuntu-latest-valgrind-unitTest: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: APT install | |
| run: | | |
| sudo apt update | |
| sudo apt install -y libhdf5-dev libminizip-dev libboost-all-dev valgrind | |
| - name: CMake build and install | |
| run: | | |
| cd .. | |
| mkdir build | |
| cd build | |
| cmake -DCMAKE_BUILD_TYPE=Debug -DWITH_EXAMPLE=TRUE -DWITH_RESQML2_2=TRUE -DWITH_TEST=TRUE -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/fesapi-install ${{ github.workspace }} | |
| cmake --build . -j2 --config Debug | |
| cmake --install . | |
| - name: Run Valgrind on example | |
| run: | | |
| valgrind --leak-check=full --track-origins=yes --show-leak-kinds=all ${{ github.workspace }}/../build/example/exampled | |
| - name: Run Unit tests | |
| run: | | |
| ${{ github.workspace }}/../build/test/unitTest |