[RUN] Version Compare #30
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Compares API responses between two SeaTable versions. | |
| # | |
| # Starts each version via Docker, runs the test suite, and reports | |
| # any differences in API behavior between the two versions. | |
| # | |
| # Required GitHub Secrets: | |
| # DOCKERHUB_USERNAME - Docker Hub username (for private image access) | |
| # DOCKERHUB_TOKEN - Docker Hub Personal Access Token | |
| # SEATABLE_LICENSE - SeaTable Enterprise license file content | |
| name: "[RUN] Version Compare" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| old_version: | |
| description: "Baseline version (e.g. 6.0.10)" | |
| required: true | |
| old_image: | |
| description: "Docker Hub repository for baseline" | |
| required: true | |
| type: choice | |
| default: "seatable/seatable-enterprise" | |
| options: | |
| - "seatable/seatable-enterprise" | |
| - "seatable/seatable-enterprise-testing" | |
| new_version: | |
| description: "New version to compare (e.g. 6.1.0)" | |
| required: true | |
| new_image: | |
| description: "Docker Hub repository for new version" | |
| required: true | |
| type: choice | |
| default: "seatable/seatable-enterprise-testing" | |
| options: | |
| - "seatable/seatable-enterprise" | |
| - "seatable/seatable-enterprise-testing" | |
| jobs: | |
| compare: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repo | |
| uses: actions/checkout@v4 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install test dependencies | |
| run: pip install -r tests/requirements.txt | |
| - name: Write SeaTable license file | |
| working-directory: version-compare | |
| env: | |
| SEATABLE_LICENSE: ${{ secrets.SEATABLE_LICENSE }} | |
| run: echo "${SEATABLE_LICENSE}" > seatable-license.txt | |
| # --- Baseline (old version) --- | |
| - name: Start SeaTable ${{ inputs.old_version }} | |
| working-directory: version-compare | |
| run: | | |
| SEATABLE_IMAGE=${{ inputs.old_image }} SEATABLE_VERSION=${{ inputs.old_version }} docker compose up -d | |
| ./setup.sh | |
| - name: Run tests against ${{ inputs.old_version }} (create baseline snapshots) | |
| working-directory: tests | |
| run: | | |
| set -a && source ../version-compare/.env && set +a | |
| pytest --snapshot-update --color=yes -v 2>&1 | tee /tmp/baseline-output.txt || true | |
| - name: Stop SeaTable ${{ inputs.old_version }} | |
| working-directory: version-compare | |
| run: SEATABLE_VERSION=${{ inputs.old_version }} docker compose down -v | |
| # --- New version --- | |
| - name: Clean up seatable-data and write license file | |
| working-directory: version-compare | |
| env: | |
| SEATABLE_LICENSE: ${{ secrets.SEATABLE_LICENSE }} | |
| run: | | |
| sudo rm -r seatable-data | |
| echo "${SEATABLE_LICENSE}" > seatable-license.txt | |
| - name: Start SeaTable ${{ inputs.new_version }} | |
| working-directory: version-compare | |
| run: | | |
| SEATABLE_IMAGE=${{ inputs.new_image }} SEATABLE_VERSION=${{ inputs.new_version }} docker compose up -d | |
| ./setup.sh | |
| - name: Run tests against ${{ inputs.new_version }} (compare with baseline) | |
| working-directory: tests | |
| run: | | |
| set -a && source ../version-compare/.env && set +a | |
| pytest --color=yes -v 2>&1 | tee /tmp/compare-output.txt | |
| continue-on-error: true | |
| - name: Generate diff report | |
| if: always() | |
| run: | | |
| mkdir -p /tmp/report | |
| cat > /tmp/report/summary.md << 'HEADER' | |
| # API Version Comparison Report | |
| HEADER | |
| echo "" >> /tmp/report/summary.md | |
| echo "**Baseline:** ${{ inputs.old_image }}:${{ inputs.old_version }}" >> /tmp/report/summary.md | |
| echo "**Compared:** ${{ inputs.new_image }}:${{ inputs.new_version }}" >> /tmp/report/summary.md | |
| echo "" >> /tmp/report/summary.md | |
| # Count results from baseline | |
| if [ -f /tmp/baseline-output.txt ]; then | |
| baseline_passed=$(grep -oE '[0-9]+ passed' /tmp/baseline-output.txt | head -1 || echo "0 passed") | |
| baseline_failed=$(grep -oE '[0-9]+ failed' /tmp/baseline-output.txt | head -1 || echo "0 failed") | |
| baseline_errors=$(grep -oE '[0-9]+ error' /tmp/baseline-output.txt | head -1 || echo "0 errors") | |
| echo "**Baseline results:** ${baseline_passed}, ${baseline_failed}, ${baseline_errors}" >> /tmp/report/summary.md | |
| fi | |
| # Count results from comparison | |
| if [ -f /tmp/compare-output.txt ]; then | |
| compare_passed=$(grep -oE '[0-9]+ passed' /tmp/compare-output.txt | head -1 || echo "0 passed") | |
| compare_failed=$(grep -oE '[0-9]+ failed' /tmp/compare-output.txt | head -1 || echo "0 failed") | |
| compare_errors=$(grep -oE '[0-9]+ error' /tmp/compare-output.txt | head -1 || echo "0 errors") | |
| echo "**Comparison results:** ${compare_passed}, ${compare_failed}, ${compare_errors}" >> /tmp/report/summary.md | |
| fi | |
| echo "" >> /tmp/report/summary.md | |
| if [ -f /tmp/compare-output.txt ] && grep -qE "FAILED|ERROR" /tmp/compare-output.txt; then | |
| echo "## Differences Found" >> /tmp/report/summary.md | |
| echo "" >> /tmp/report/summary.md | |
| # List failed tests | |
| echo "### Failed Tests" >> /tmp/report/summary.md | |
| echo "" >> /tmp/report/summary.md | |
| echo '```' >> /tmp/report/summary.md | |
| grep "FAILED" /tmp/compare-output.txt >> /tmp/report/summary.md || true | |
| echo '```' >> /tmp/report/summary.md | |
| echo "" >> /tmp/report/summary.md | |
| # Extract snapshot diffs (the actual response differences) | |
| echo "### Response Differences" >> /tmp/report/summary.md | |
| echo "" >> /tmp/report/summary.md | |
| echo '```diff' >> /tmp/report/summary.md | |
| # syrupy outputs diffs between "snapshot" and "received" markers | |
| sed -n '/^____/,/^____\|^====\|^FAILED\|^---.*---$/p' /tmp/compare-output.txt \ | |
| | head -500 >> /tmp/report/summary.md || true | |
| echo '```' >> /tmp/report/summary.md | |
| echo "" >> /tmp/report/summary.md | |
| # Extract schema validation failures | |
| if grep -q "CheckFailed" /tmp/compare-output.txt; then | |
| echo "### Schema Validation Failures" >> /tmp/report/summary.md | |
| echo "" >> /tmp/report/summary.md | |
| echo '```' >> /tmp/report/summary.md | |
| grep -A 10 "Response violates schema" /tmp/compare-output.txt \ | |
| | head -100 >> /tmp/report/summary.md || true | |
| echo '```' >> /tmp/report/summary.md | |
| fi | |
| else | |
| echo "## No Differences Found" >> /tmp/report/summary.md | |
| echo "" >> /tmp/report/summary.md | |
| echo "All API responses match between versions." >> /tmp/report/summary.md | |
| fi | |
| cp /tmp/baseline-output.txt /tmp/report/ 2>/dev/null || true | |
| cp /tmp/compare-output.txt /tmp/report/ 2>/dev/null || true | |
| - name: Stop SeaTable ${{ inputs.new_version }} | |
| if: always() | |
| working-directory: version-compare | |
| run: SEATABLE_VERSION=${{ inputs.new_version }} docker compose down -v | |
| - name: Upload comparison report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: version-compare-${{ inputs.old_version }}-vs-${{ inputs.new_version }} | |
| path: /tmp/report/ | |
| retention-days: 30 |