build.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Python Release Build
on:
  workflow_dispatch: {}
  push:
    branches: ["main*"]

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: astral-sh/setup-uv@v5
        with:
            enable-cache: true

      # Use the --no-install-package to only install the dependencies
      # but do not yet build the rust library
      - name: Install dependencies
        run: uv sync --dev --no-install-package datafusion

      # Update output format to enable automatic inline annotations.
      # - name: Run Ruff
      #   run: |
      #     uv run --no-project ruff check --output-format=github python/
      #     uv run --no-project ruff format --check python/

  # generate-license:
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v4
  #     - uses: astral-sh/setup-uv@v5
  #       with:
  #           enable-cache: true
  #
  #     - name: Generate license file
  #       run: uv run --no-project python ./dev/create_license.py
  #     - uses: actions/upload-artifact@v4
  #       with:
  #         name: python-wheel-license
  #         path: LICENSE.txt

  build-python-mac-win:
    name: Mac/Win
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10"]
        os: [macos-latest, windows-latest]
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - uses: dtolnay/rust-toolchain@stable

      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          version: "27.4"
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - uses: astral-sh/setup-uv@v5
        with:
            enable-cache: true

      - name: Build Python package
        run: |
          uv sync --dev --no-install-package datafusion
          uv run --no-project maturin build --release --strip --features substrait

      - name: List Windows wheels
        if: matrix.os == 'windows-latest'
        run: dir target\wheels\
        # since the runner is dynamic shellcheck (from actionlint) can't infer this is powershell
        # so we specify it explicitly
        shell: powershell

      - name: List Mac wheels
        if: matrix.os != 'windows-latest'
        run: find target/wheels/

      - name: Archive wheels
        uses: actions/upload-artifact@v4
        with:
          name: dist-${{ matrix.os  }}
          path: target/wheels/*

  build-macos-x86_64:
    name: Mac x86_64
    runs-on: macos-13
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10"]
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - uses: dtolnay/rust-toolchain@stable

      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          version: "27.4"
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - uses: astral-sh/setup-uv@v5
        with:
            enable-cache: true

      - name: Build Python package
        run: |
          uv sync --dev --no-install-package datafusion
          uv run --no-project maturin build --release --strip --features substrait

      - name: List Mac wheels
        run: find target/wheels/

      - name: Archive wheels
        uses: actions/upload-artifact@v4
        with:
          name: dist-macos-aarch64
          path: target/wheels/*

  build-manylinux-x86_64:
    name: Manylinux x86_64
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: libssl3 openssl
          version: 1.0
      - run: cat LICENSE.txt
      - name: Build wheels
        uses: PyO3/maturin-action@v1
        env:
          RUST_BACKTRACE: 1
        with:
          rust-toolchain: nightly
          target: x86_64
          manylinux: auto
          before-script-linux: |
            # If we're running on rhel centos, install needed packages.
            if command -v yum &> /dev/null; then
                yum update -y && yum install -y perl-core openssl openssl-devel pkgconfig libatomic

                # If we're running on i686 we need to symlink libatomic
                # in order to build openssl with -latomic flag.
                if [[ ! -d "/usr/lib64" ]]; then
                    ln -s /usr/lib/libatomic.so.1 /usr/lib/libatomic.so
                fi
            else
                # If we're running on debian-based system.
                apt update -y && apt-get install -y libssl-dev openssl pkg-config
            fi
          rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
          args: --release --manylinux 2014 --features protoc,substrait
      - name: Archive wheels
        uses: actions/upload-artifact@v4
        with:
          name: dist-manylinux-x86_64
          path: target/wheels/*

  # build-manylinux-aarch64:
  #   name: Manylinux arm64
  #   runs-on: ubuntu-latest-arm
  #   steps:
  #     - uses: actions/checkout@v4
  #     - uses: awalsh128/cache-apt-pkgs-action@latest
  #       with:
  #         packages: libssl3 openssl
  #         version: 1.0
  #     - run: cat LICENSE.txt
  #     - name: Build wheels
  #       uses: PyO3/maturin-action@v1
  #       env:
  #         RUST_BACKTRACE: full
  #       with:
  #         rust-toolchain: nightly
  #         target: aarch64
  #         # Use manylinux_2_28-cross because the manylinux2014-cross has GCC 4.8.5, which causes the build to fail
  #         manylinux: 2_28
  #         before-script-linux: |
  #           # If we're running on rhel centos, install needed packages.
  #           if command -v yum &> /dev/null; then
  #               yum update -y && yum install -y perl-core openssl openssl-devel pkgconfig libatomic
  #
  #               # If we're running on i686 we need to symlink libatomic
  #               # in order to build openssl with -latomic flag.
  #               if [[ ! -d "/usr/lib64" ]]; then
  #                   ln -s /usr/lib/libatomic.so.1 /usr/lib/libatomic.so
  #               fi
  #           else
  #               # If we're running on debian-based system.
  #               apt update -y && apt-get install -y libssl-dev openssl pkg-config librust-openssl-sys-dev
  #           fi
  #           ls -lA /usr/include/openssl/
  #         rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
  #         args: --release --features protoc,substrait
  #     - name: Archive wheels
  #       uses: actions/upload-artifact@v4
  #       with:
  #         name: dist-manylinux-aarch64
  #         path: target/wheels/*

  # build-sdist:
  #   name: Source distribution
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v4
  #     - uses: awalsh128/cache-apt-pkgs-action@latest
  #       with:
  #         packages: libssl3 openssl
  #         version: 1.0
  #     - run: cat LICENSE.txt
  #     - name: Build sdist
  #       uses: PyO3/maturin-action@v1
  #       with:
  #         rust-toolchain: stable
  #         manylinux: auto
  #         before-script-linux: |
  #           # If we're running on rhel centos, install needed packages.
  #           if command -v yum &> /dev/null; then
  #               yum update -y && yum install -y perl-core openssl openssl-devel pkgconfig libatomic
  #
  #               # If we're running on i686 we need to symlink libatomic
  #               # in order to build openssl with -latomic flag.
  #               if [[ ! -d "/usr/lib64" ]]; then
  #                   ln -s /usr/lib/libatomic.so.1 /usr/lib/libatomic.so
  #               fi
  #           else
  #               # If we're running on debian-based system.
  #               apt update -y && apt-get install -y libssl-dev openssl pkg-config
  #           fi
  #         rustup-components: rust-std rustfmt
  #         args: --release --sdist --out dist --features protoc,substrait
  #     - name: Assert sdist build does not generate wheels
  #       run: |
  #           if [ "$(ls -A target/wheels)" ]; then
  #             echo "Error: Sdist build generated wheels"
  #             exit 1
  #           else
  #             echo "Directory is clean"
  #           fi
  #       shell: bash
  
  merge-build-artifacts:
    runs-on: ubuntu-latest
    needs:
      - build-python-mac-win
      - build-macos-x86_64
      - build-manylinux-x86_64
      # - build-manylinux-aarch64
      # - build-sdist
    steps:
      - name: Merge Build Artifacts
        uses: actions/upload-artifact/merge@v4
        with:
          name: dist
          pattern: dist-*
  
  # NOTE: PyPI publish needs to be done manually for now after release passed the vote
  # release:
  #   name: Publish in PyPI
  #   needs: [build-manylinux, build-python-mac-win]
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/download-artifact@v4
  #     - name: Publish to PyPI
  #       uses: pypa/gh-action-pypi-publish@master
  #       with:
  #         user: __token__
  #         password: ${{ secrets.pypi_password }}