refactoring #65
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Production Deployment | |
| on: | |
| push: | |
| branches: [ main ] | |
| tags: [ 'v*' ] | |
| pull_request: | |
| branches: [ main ] | |
| workflow_dispatch: | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| jobs: | |
| test-advanced-scraping: | |
| runs-on: ubuntu-latest | |
| name: Test Advanced Scraping Features | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install cachetools # Required for advanced scraping | |
| pip install -e . | |
| - name: Test basic functionality | |
| run: | | |
| whyml --version | |
| whyml --help | |
| - name: Test advanced scraping features | |
| run: | | |
| # Test structure simplification | |
| whyml scrape https://example.com --max-depth 2 --simplify-structure -o test-advanced.yaml | |
| # Test selective sections | |
| whyml scrape https://example.com --section metadata --section analysis -o test-selective.yaml | |
| # Verify outputs | |
| test -f test-advanced.yaml | |
| test -f test-selective.yaml | |
| # Validate YAML format | |
| python -c "import yaml; yaml.safe_load(open('test-advanced.yaml'))" | |
| python -c "import yaml; yaml.safe_load(open('test-selective.yaml'))" | |
| - name: Run test suite | |
| run: | | |
| python -m pytest tests/ -v --tb=short | |
| continue-on-error: true # Allow continuation even if some tests fail due to missing dependencies | |
| - name: Upload test artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test-outputs | |
| path: | | |
| test-advanced.yaml | |
| test-selective.yaml | |
| build-production-image: | |
| runs-on: ubuntu-latest | |
| needs: test-advanced-scraping | |
| name: Build Production Docker Image | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to Container Registry | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: docker/Dockerfile.production | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| deploy-to-staging: | |
| runs-on: ubuntu-latest | |
| needs: build-production-image | |
| if: github.ref == 'refs/heads/main' | |
| name: Deploy to Staging | |
| environment: staging | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Deploy to staging | |
| run: | | |
| echo "🚀 Deploying to staging environment" | |
| # Add actual deployment commands here | |
| # docker-compose -f docker/docker-compose.production.yml up -d | |
| - name: Run integration tests | |
| run: | | |
| echo "🧪 Running integration tests on staging" | |
| # Add integration test commands here | |
| - name: Health check | |
| run: | | |
| echo "🏥 Performing health checks" | |
| # Add health check commands here | |
| performance-benchmarks: | |
| runs-on: ubuntu-latest | |
| needs: test-advanced-scraping | |
| name: Performance Benchmarks | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install cachetools | |
| pip install -e . | |
| - name: Run performance benchmarks | |
| run: | | |
| echo "📊 Running performance benchmarks" | |
| # Benchmark structure simplification | |
| time whyml scrape https://example.com --max-depth 3 --simplify-structure -o perf-test1.yaml | |
| # Benchmark selective sections | |
| time whyml scrape https://example.com --section analysis -o perf-test2.yaml | |
| # Benchmark testing workflow | |
| time whyml scrape https://example.com --test-conversion --output-html perf-test.html -o perf-test3.yaml | |
| echo "✅ Performance benchmarks completed" | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: performance-results | |
| path: | | |
| perf-test*.yaml | |
| perf-test.html | |
| deploy-to-production: | |
| runs-on: ubuntu-latest | |
| needs: [build-production-image, deploy-to-staging, performance-benchmarks] | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| name: Deploy to Production | |
| environment: production | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Deploy to production | |
| run: | | |
| echo "🚀 Deploying to production environment" | |
| echo "Tag: ${{ github.ref_name }}" | |
| # Add actual production deployment commands here | |
| - name: Post-deployment verification | |
| run: | | |
| echo "✅ Production deployment verification" | |
| # Add production verification commands here | |
| - name: Notify deployment success | |
| run: | | |
| echo "🎉 Production deployment successful!" | |
| # Add notification commands here (Slack, email, etc.) | |
| security-scan: | |
| runs-on: ubuntu-latest | |
| name: Security Scan | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Run security scan | |
| uses: github/super-linter@v4 | |
| env: | |
| DEFAULT_BRANCH: main | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| VALIDATE_PYTHON_BLACK: false | |
| VALIDATE_PYTHON_FLAKE8: true | |
| VALIDATE_PYTHON_PYLINT: false | |
| VALIDATE_DOCKERFILE_HADOLINT: true | |
| - name: Dependency vulnerability scan | |
| run: | | |
| pip install safety | |
| safety check -r requirements.txt | |
| documentation-check: | |
| runs-on: ubuntu-latest | |
| name: Documentation Validation | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Validate documentation | |
| run: | | |
| echo "📚 Validating documentation" | |
| # Check that all referenced files exist | |
| test -f docs/USER_TESTING_GUIDE.md | |
| test -f docs/cli/scrape.md | |
| test -f examples/advanced-scraping/README.md | |
| test -f INSTALLATION.md | |
| # Check that examples in documentation are valid | |
| echo "✅ Documentation files present" | |
| - name: Test example commands | |
| run: | | |
| echo "🧪 Testing example commands from documentation" | |
| # Install dependencies | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install cachetools | |
| pip install -e . | |
| # Test basic examples work | |
| whyml --help > /dev/null | |
| whyml --version > /dev/null | |
| echo "✅ Example commands working" |