ENH 4.2|4.3 CI/CD Enhancements: pyproject.toml + linting + pre-commit hooks

.github/workflows/build-and-publish.yml

@@ -0,0 +1,83 @@
+name: Build and Publish Package
+
+on:
+  push:
+    branches:
+      - main
+      - master
+  release:
+    types: [created]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+    - name: Install dependencies
+      run: |
+        # Create a virtual environment for uv
+        uv venv
+        # Install pip inside the virtual environment
+        uv pip install pip
+        # Install build tools
+        uv pip install build twine setuptools wheel
+    - name: Build package
+      run: |
+        # Activate the virtual environment
+        source .venv/bin/activate
+        # Build the package using pyproject.toml
+        python -m build
+    - name: Check package
+      run: |
+        # Activate the virtual environment
+        source .venv/bin/activate
+        twine check dist/*
+    - name: Store built package
+      uses: actions/upload-artifact@v3
+      with:
+        name: dist
+        path: dist/
+        retention-days: 7
+
+  publish:
+    needs: build
+    # Only run on release
+    if: github.event_name == 'release'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+    - name: Install dependencies
+      run: |
+        # Create a virtual environment for uv
+        uv venv
+        # Install pip inside the virtual environment
+        uv pip install pip
+        # Install twine
+        uv pip install twine
+    - name: Download built package
+      uses: actions/download-artifact@v3
+      with:
+        name: dist
+        path: dist/
+    - name: Publish to PyPI
+      run: |
+        # Activate the virtual environment
+        source .venv/bin/activate
+        # Use twine to upload to PyPI
+        twine upload dist/* --username __token__ --password ${{ secrets.PYPI_API_TOKEN }} --skip-existing

.github/workflows/python-package.yml

@@ -5,28 +5,53 @@ on:
     branches:
       - '**'  # Match all branches including those with slashes
   pull_request:
-    branches: 
+    branches:
       - main
       - master  # Include master branch as well
 
 jobs:
-  # lint:
-  #   # Ruff checks are commented out for now. Uncomment to restore linting/formatting checks.
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #   - uses: actions/checkout@v3
-  #   - name: Set up Python
-  #     uses: actions/setup-python@v4
-  #     with:
-  #       python-version: '3.10'
-  #   - name: Install dependencies
-  #     run: |
-  #       python -m pip install --upgrade pip
-  #       pip install ruff
-  #   - name: Run Ruff
-  #     run: |
-  #       ruff check .
-  #       ruff format --check .
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0  # Fetch all history for proper file comparison
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+    - name: Install dependencies
+      run: |
+        # Create a virtual environment for uv
+        uv venv
+        # Install pre-commit
+        uv pip install pre-commit
+    - name: Get changed files
+      id: changed-files
+      run: |
+        # Get the list of changed files
+        if [ "${{ github.event_name }}" == "pull_request" ]; then
+          # For pull requests, compare with the base branch
+          CHANGED_FILES=$(git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | tr '\n' ' ')
+        else
+          # For pushes, compare with the previous commit
+          CHANGED_FILES=$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | tr '\n' ' ')
+        fi
+        echo "CHANGED_FILES=$CHANGED_FILES" >> $GITHUB_ENV
+    - name: Run pre-commit on changed files
+      run: |
+        # Activate the virtual environment
+        source .venv/bin/activate
+        if [ -n "$CHANGED_FILES" ]; then
+          echo "Checking files: $CHANGED_FILES"
+          pre-commit run --color always --files $CHANGED_FILES --show-diff-on-failure
+        else
+          echo "No files changed. Skipping pre-commit checks."
+        fi
 
   test:
     runs-on: ubuntu-latest
@@ -43,7 +68,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
         restore-keys: |
           ${{ runner.os }}-pip-
     - name: Install uv
@@ -54,21 +79,14 @@ jobs:
       run: |
         # Create a virtual environment for uv
         uv venv
-        # Install PyTorch first to avoid conflicts
-        uv pip install torch --index-url https://download.pytorch.org/whl/cpu
-        # Install all dependencies from requirements.txt
-        uv pip install -r requirements.txt
-        # Install package in development mode
-        uv pip install -e .
-        # Explicitly install pytest and pytest-cov in the virtual environment
-        uv pip install pytest pytest-cov
-    - name: Run tests
+        # Install the package with test dependencies
+        uv pip install -e ".[test]"
+    - name: Test with pytest
       run: |
-        # Use the Python from the virtual environment
+        # Activate the virtual environment
         source .venv/bin/activate
-        # Only run unit tests for now, integration tests will be handled in a separate MR
-        #python -m pytest tests --cov=dsipts --cov-report=xml
-        python -m pytest tests/unit/ --cov=dsipts --cov-report=xml
+        # Run unit tests with coverage
+        pytest tests/unit/ --cov=dsipts --cov-report=xml
   docs:
     needs: test
     runs-on: ubuntu-latest
@@ -78,41 +96,26 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: '3.12'
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        # Create a virtual environment for uv
+        uv venv
+        # Install pip inside the virtual environment
+        uv pip install pip
         # Install package with docs extras
-        pip install -e ".[docs]"
+        uv pip install -e ".[docs]"
         # Explicitly install Sphinx and required extensions
-        pip install sphinx>=7.0.0
-        pip install sphinx_pdj_theme>=0.4.0
+        uv pip install sphinx>=7.0.0
+        uv pip install sphinx_pdj_theme>=0.4.0
         # Install sphinx_mdinclude directly from GitHub to ensure compatibility
-        pip install git+https://github.com/omnilib/sphinx-mdinclude.git
+        uv pip install git+https://github.com/omnilib/sphinx-mdinclude.git
     - name: Build documentation
       run: |
+        # Activate the virtual environment
+        source .venv/bin/activate
         cd docs
         make html
-  build:
-    needs: [test, docs]
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.12'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install build twine setuptools wheel
-    - name: Build package
-      run: |
-        # Create a requirements.txt file if it doesn't exist
-        if [ ! -f requirements.txt ]; then
-          echo "Creating requirements.txt from setup.py core_requirements"
-          python -c "import re; from setup import core_requirements; print('\n'.join(core_requirements))" > requirements.txt
-        fi
-        # Build the package
-        python -m build
-    - name: Check package
-      run: twine check dist/*