RTLMeter #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # DESCRIPTION: Github actions config | |
| # This name is key to badges in README.rst, so we use the name build | |
| # SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 | |
| name: RTLMeter | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 2 * * *' # Daily, starting at 02:00 UTC | |
| pull_request: | |
| types: [opened, synchronize, reopened, labeled, unlabeled] | |
| permissions: | |
| contents: read | |
| defaults: | |
| run: | |
| shell: bash | |
| concurrency: | |
| # At most 1 job per branch. Auto cancel all but scheduled jobs | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name != 'schedule' }} | |
| jobs: | |
| start: | |
| name: Start | |
| # Only run scheduled jobs if explicitly enabled for that repo (e.g.: not on forks) | |
| # Only run pull request jobs if labelled as needing an RTLMeter run | |
| # Always run workflow dispatch jobs | |
| if: | | |
| (github.event_name == 'schedule' | |
| && vars.ENABLE_SCHEDULED_JOBS == 'true') || | |
| (github.event_name == 'pull_request' | |
| && contains(github.event.pull_request.labels.*.name, 'pr: rtlmeter')) || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Startup | |
| run: echo | |
| build-gcc: | |
| name: Build GCC | |
| needs: start | |
| uses: ./.github/workflows/reusable-rtlmeter-build.yml | |
| with: | |
| runs-on: ubuntu-24.04 | |
| cc: gcc | |
| build-clang: | |
| name: Build Clang | |
| needs: start | |
| uses: ./.github/workflows/reusable-rtlmeter-build.yml | |
| with: | |
| runs-on: ubuntu-24.04 | |
| cc: clang | |
| run-gcc: | |
| name: Run GCC | ${{ matrix.cases }} | |
| needs: build-gcc | |
| uses: ./.github/workflows/reusable-rtlmeter-run.yml | |
| with: | |
| tag: gcc | |
| runs-on: ubuntu-24.04 | |
| cc: gcc | |
| cases: ${{ matrix.cases }} | |
| run-name: "gcc" | |
| compileArgs: "" | |
| executeArgs: "" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: ${{ github.event == 'schedule' && 2 || 7 }} | |
| matrix: | |
| cases: | |
| - "BlackParrot:1x1:*" | |
| - "BlackParrot:2x2:*" | |
| - "BlackParrot:4x4:*" | |
| - "Caliptra:default:*" | |
| - "NVDLA:*" | |
| - "OpenPiton:1x1:*" | |
| - "OpenPiton:2x2:*" | |
| - "OpenPiton:4x4:*" | |
| - "OpenTitan:*" | |
| - "VeeR-EH1:asic*" | |
| - "VeeR-EH1:default*" | |
| - "VeeR-EH1:hiperf*" | |
| - "VeeR-EH2:asic*" | |
| - "VeeR-EH2:default*" | |
| - "VeeR-EH2:hiperf*" | |
| - "VeeR-EL2:asic*" | |
| - "VeeR-EL2:default*" | |
| - "VeeR-EL2:hiperf*" | |
| - "Vortex:mini:*" | |
| - "Vortex:sane:*" | |
| - "XiangShan:default-chisel3:* !*:linux" | |
| - "XiangShan:default-chisel6:* !*:linux" | |
| - "XiangShan:mini-chisel3:* !*:linux" | |
| - "XiangShan:mini-chisel6:* !*:linux" | |
| - "XuanTie-E902:*" | |
| - "XuanTie-E906:*" | |
| - "XuanTie-C906:*" | |
| - "XuanTie-C910:*" | |
| run-clang: | |
| name: Run Clang | ${{ matrix.cases }} | |
| needs: build-clang | |
| uses: ./.github/workflows/reusable-rtlmeter-run.yml | |
| with: | |
| tag: clang | |
| runs-on: ubuntu-24.04 | |
| cc: clang | |
| cases: ${{ matrix.cases }} | |
| run-name: "clang --threads 4" | |
| compileArgs: "--threads 4" | |
| executeArgs: "" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: ${{ github.event == 'schedule' && 2 || 7 }} | |
| matrix: | |
| cases: | |
| - "BlackParrot:1x1:*" | |
| - "BlackParrot:2x2:*" | |
| - "BlackParrot:4x4:*" | |
| - "Caliptra:default:*" | |
| - "NVDLA:*" | |
| - "OpenPiton:1x1:*" | |
| - "OpenPiton:2x2:*" | |
| - "OpenPiton:4x4:*" | |
| - "OpenTitan:*" | |
| - "VeeR-EH1:asic*" | |
| - "VeeR-EH1:default*" | |
| - "VeeR-EH1:hiperf*" | |
| - "VeeR-EH2:asic*" | |
| - "VeeR-EH2:default*" | |
| - "VeeR-EH2:hiperf*" | |
| - "VeeR-EL2:asic*" | |
| - "VeeR-EL2:default*" | |
| - "VeeR-EL2:hiperf*" | |
| - "Vortex:mini:*" | |
| - "Vortex:sane:*" | |
| - "XiangShan:default-chisel3:* !*:linux" | |
| - "XiangShan:default-chisel6:* !*:linux" | |
| - "XiangShan:mini-chisel3:* !*:linux" | |
| - "XiangShan:mini-chisel6:* !*:linux" | |
| - "XuanTie-E902:*" | |
| - "XuanTie-E906:*" | |
| - "XuanTie-C906:*" | |
| - "XuanTie-C910:*" | |
| run-gcc-hier: | |
| name: Run GCC hier | ${{ matrix.cases }} | |
| needs: build-gcc | |
| uses: ./.github/workflows/reusable-rtlmeter-run.yml | |
| with: | |
| tag: gcc-hier | |
| runs-on: ubuntu-24.04 | |
| cc: gcc | |
| cases: ${{ matrix.cases }} | |
| run-name: "gcc --hierarchical" | |
| compileArgs: "--hierarchical" | |
| executeArgs: "" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: ${{ github.event == 'schedule' && 2 || 7 }} | |
| matrix: | |
| cases: | |
| - "BlackParrot:1x1:* !-hier" | |
| - "BlackParrot:2x2:* !-hier" | |
| - "BlackParrot:4x4:* !-hier" | |
| - "NVDLA:* !-hier" | |
| - "OpenPiton:1x1:* !-hier" | |
| - "OpenPiton:2x2:* !-hier" | |
| - "OpenPiton:4x4:* !-hier" | |
| - "OpenPiton:8x8:* !-hier" | |
| - "OpenPiton:16x16:dhry !-hier" | |
| - "XuanTie-C910:* !-hier" | |
| combine-results: | |
| name: Combine results | |
| needs: [run-gcc, run-clang, run-gcc-hier] | |
| # Run if any of the dependencies have run, even if failed. | |
| # That is: do not run if all skipped, or the workflow was cancelled. | |
| if: ${{ (contains(needs.*.result, 'success') || contains(needs.*.result, 'failure')) && !cancelled() }} | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tag: [gcc, clang, gcc-hier] | |
| steps: | |
| - name: Checkout RTLMeter | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: "verilator/rtlmeter" | |
| path: rtlmeter | |
| - name: Setup RTLMeter venv | |
| working-directory: rtlmeter | |
| run: make venv | |
| - name: Download all results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: rtlmeter-${{ matrix.tag }}-results-* | |
| path: all-results-${{ matrix.tag }} | |
| merge-multiple: true | |
| - name: Combine results | |
| working-directory: rtlmeter | |
| run: | | |
| ./rtlmeter collate ../all-results-${{ matrix.tag }}/*.json > ../all-results-${{ matrix.tag }}.json | |
| - name: Upload combined results | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| path: all-results-${{ matrix.tag }}.json | |
| name: all-results-${{ matrix.tag }} | |
| overwrite: true | |
| retention-days: 30 | |
| publish-scheduled-results: | |
| name: Publish results to verilator/verilator-rtlmeter-results | |
| needs: combine-results | |
| # Only run on scheduled builds on the main repository. We also restrict | |
| # the publishing to run only on the first run_attempt. This is required | |
| # to prevent multiple uploads the same day (if rerunning), as the | |
| # dashboard UI currently assumes there is only one data point per | |
| # calendar day. Results from reruns can be imported manually if needed. | |
| if: ${{ github.event_name == 'schedule' && github.repository == 'verilator/verilator' && github.run_attempt == 1 && contains(needs.*.result, 'success') && !cancelled() }} | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Download combined results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: all-results-* | |
| path: results | |
| merge-multiple: true | |
| - name: Upload published results | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| path: results/*.json | |
| name: published-results | |
| # Pushing to verilator/verilator-rtlmeter-results requires elevated permissions | |
| - name: Generate access token | |
| id: generate-token | |
| uses: actions/create-github-app-token@v2.2.1 | |
| with: | |
| app-id: ${{ vars.VERILATOR_CI_ID }} | |
| private-key: ${{ secrets.VERILATOR_CI_KEY }} | |
| owner: verilator | |
| repositories: verilator-rtlmeter-results | |
| permission-contents: write | |
| - name: Checkout verilator-rtlmeter-results | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: "verilator/verilator-rtlmeter-results" | |
| token: ${{ steps.generate-token.outputs.token }} | |
| path: verilator-rtlmeter-results | |
| - name: Import results | |
| id: import-results | |
| working-directory: verilator-rtlmeter-results | |
| run: | | |
| for f in $(find ../results -name "*.json"); do \ | |
| echo "Importing $f"; \ | |
| ./bin/add-rtlmeter-result $f; \ | |
| done | |
| test -z "$(git status --porcelain)" || echo "valid=1" >> "$GITHUB_OUTPUT" | |
| - name: Push to verilator-rtlmeter-results | |
| if: ${{ steps.import-results.outputs.valid }} | |
| working-directory: verilator-rtlmeter-results | |
| run: | | |
| git config --global user.email "action@example.com" | |
| git config --global user.name "github action" | |
| git add . | |
| git commit -m "Verilator CI: Results of 'RTLMeter' workflow run #${{ github.run_number }}" | |
| git push origin | |
| prepare-pr-results: | |
| name: Prepare Pull Request results | |
| needs: combine-results | |
| if: ${{ github.event_name == 'pull_request' && github.repository == 'verilator/verilator' && contains(needs.*.result, 'success') && !cancelled() }} | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| actions: read | |
| steps: | |
| - name: Checkout RTLMeter | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: "verilator/rtlmeter" | |
| path: rtlmeter | |
| - name: Setup RTLMeter venv | |
| working-directory: rtlmeter | |
| run: make venv | |
| - name: Download combined results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: all-results-* | |
| path: all-results | |
| merge-multiple: true | |
| - name: Get scheduled run info | |
| id: scheduled-info | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| ID=$(gh run --repo ${{ github.repository }} list --workflow RTLMeter --event schedule --status success --limit 1 --json databaseId --jq ".[0].databaseId") | |
| echo "id=$ID" >> $GITHUB_OUTPUT | |
| URL=$(gh run --repo ${{ github.repository }} view $ID --json url --jq ".url") | |
| echo "url=$URL" >> $GITHUB_OUTPUT | |
| NUM=$(gh run --repo ${{ github.repository }} view $ID --json number --jq ".number") | |
| echo "num=$NUM" >> $GITHUB_OUTPUT | |
| DATE=$(gh run --repo ${{ github.repository }} view $ID --json createdAt --jq ".createdAt") | |
| echo "date=$DATE" >> $GITHUB_OUTPUT | |
| - name: Download scheduled run results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: published-results | |
| path: nightly-results | |
| run-id: ${{ steps.scheduled-info.outputs.id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Compare results | |
| working-directory: rtlmeter | |
| run: | | |
| for tag in gcc clang gcc-hier; do | |
| ADATA=../nightly-results/all-results-${tag}.json | |
| BDATA=../all-results/all-results-${tag}.json | |
| touch ../verilate-${tag}.txt | |
| touch ../execute-${tag}.txt | |
| touch ../cppbuild-${tag}.txt | |
| if [[ ! -e $ADATA ]]; then | |
| continue | |
| fi | |
| ./rtlmeter compare --cases '* !Example:* !*:hello' --steps "verilate" --metrics "elapsed memory" $ADATA $BDATA > ../verilate-${tag}.txt | |
| cat ../verilate-${tag}.txt | |
| ./rtlmeter compare --cases '* !Example:* !*:hello' --steps "execute" --metrics "speed memory elapsed" $ADATA $BDATA > ../execute-${tag}.txt | |
| cat ../execute-${tag}.txt | |
| ./rtlmeter compare --cases '* !Example:* !*:hello' --steps "cppbuild" --metrics "elapsed memory cpu codeSize" $ADATA $BDATA > ../cppbuild-${tag}.txt | |
| cat ../cppbuild-${tag}.txt | |
| done | |
| - name: Create report | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -x | |
| NUM=$(gh run --repo ${{ github.repository }} view ${{ github.run_id }} --json number --jq ".number") | |
| URL=$(gh run --repo ${{ github.repository }} view ${{ github.run_id }} --json url --jq ".url") | |
| echo -n "Performance metrics for PR workflow [#$NUM]($URL) (B) compared to scheduled run" > report.txt | |
| echo -n " [#${{ steps.scheduled-info.outputs.num }}](${{ steps.scheduled-info.outputs.url }}) (A)" >> report.txt | |
| echo " from ${{ steps.scheduled-info.outputs.date }}" >> report.txt | |
| for tag in gcc clang gcc-hier; do | |
| echo "" >> report.txt | |
| if [[ $tag == "gcc" ]]; then | |
| echo "<details open>" >> report.txt | |
| else | |
| echo "<details>" >> report.txt | |
| fi | |
| echo -n "<summary><strong><em>" >> report.txt | |
| jq -rj ".[0].runName" all-results/all-results-${tag}.json >> report.txt | |
| echo "</em></strong></summary>" >> report.txt | |
| awk -v RS= -v tag=${tag} '{print > sprintf("frag-%02d-verilate-%s.txt",NR,tag)}' verilate-${tag}.txt | |
| awk -v RS= -v tag=${tag} '{print > sprintf("frag-%02d-execute-%s.txt" ,NR,tag)}' execute-${tag}.txt | |
| awk -v RS= -v tag=${tag} '{print > sprintf("frag-$02d-cppbuild-%s.txt",NR,tag)}' cppbuild-${tag}.txt | |
| for f in $(ls -1 frag-*-verilate-${tag}.txt | sort) $(ls -1 frag-*-execute-${tag}.txt | sort) $(ls -1 frag-*-cppbuild-${tag}.txt | sort); do | |
| if [[ $f == frag-01-verilate-${tag}.txt || $f == frag-01-execute-${tag}.txt ]]; then | |
| echo "<details open>" >> report.txt | |
| else | |
| echo "<details>" >> report.txt | |
| fi | |
| echo -n "<summary>" >> report.txt | |
| head -n 1 $f | tr -d '\n' >> report.txt | |
| echo "</summary>" >> report.txt | |
| echo '<pre>' >> report.txt | |
| tail -n +2 $f >> report.txt | |
| echo '</pre>' >> report.txt | |
| echo "</details>" >> report.txt | |
| done | |
| echo "</details>" >> report.txt | |
| done | |
| cat report.txt | |
| - name: Upload report | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| path: report.txt | |
| name: rtlmeter-pr-results | |
| - name: Save PR number | |
| run: echo ${{ github.event.number }} > pr-number.txt | |
| - name: Upload PR number | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| path: pr-number.txt | |
| name: pr-number | |
| # Create GitHub issue for failed scheduled jobs | |
| # This should always be the last job (we want an issue if anything breaks) | |
| create-issue: | |
| name: Create issue on failure | |
| needs: publish-scheduled-results | |
| if: ${{ github.event_name == 'schedule' && github.repository == 'verilator/verilator' && github.run_attempt == 1 && failure() && !cancelled() }} | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| # Creating issues requires elevated privilege | |
| - name: Generate access token | |
| id: generate-token | |
| uses: actions/create-github-app-token@v2.2.1 | |
| with: | |
| app-id: ${{ vars.VERILATOR_CI_ID }} | |
| private-key: ${{ secrets.VERILATOR_CI_KEY }} | |
| owner: verilator | |
| repositories: verilator | |
| permission-issues: write | |
| - name: Create issue | |
| env: | |
| GH_TOKEN: ${{ steps.generate-token.outputs.token }} | |
| run: |- | |
| echo "This issue was created automatically by the GitHub Actions CI due to the failure of a scheduled RTLMeter run." >> body.txt | |
| echo "" >> body.txt | |
| echo "Workflow status: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> body.txt | |
| gh issue --repo ${{ github.repository }} create \ | |
| --title "RTLMeter run #${{ github.run_number }} Failed" \ | |
| --body-file body.txt \ | |
| --label new \ | |
| --assignee gezalore,wsnyder |