daily-compatibility.yml

# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause

name: Backwards Compatibility Tests

on:
  schedule:
    # Run daily at 8 PM PST (4 AM UTC)
    - cron: '0 4 * * *'

  workflow_dispatch:
    inputs:
      isaacsim_version:
        description: 'IsaacSim version image tag to test'
        required: true
        default: '4.5.0'
        type: string

# Concurrency control to prevent parallel runs
concurrency:
  group: compatibility-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

permissions:
  contents: read
  pull-requests: write

env:
  NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
  ISAACSIM_BASE_IMAGE: ${{ vars.ISAACSIM_BASE_IMAGE || 'nvcr.io/nvidia/isaac-sim' }}

jobs:
  setup-versions:
    runs-on: ubuntu-latest
    outputs:
      versions: ${{ steps.set-versions.outputs.versions }}
    steps:
      - name: Set Isaac Sim Versions
        id: set-versions
        run: |
          # Define all versions to test in one place
          DEFAULT_VERSIONS='["4.5.0", "5.0.0"]'

          if [ -n "${{ github.event.inputs.isaacsim_version }}" ]; then
            # If a specific version is provided via workflow_dispatch, use only that
            echo "versions=[\"${{ github.event.inputs.isaacsim_version }}\"]" >> $GITHUB_OUTPUT
          else
            # Otherwise, use all default versions
            echo "versions=$DEFAULT_VERSIONS" >> $GITHUB_OUTPUT
          fi

  test-isaaclab-tasks-compat:
    needs: setup-versions
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    continue-on-error: true
    strategy:
      matrix:
        isaacsim_version: ${{ fromJson(needs.setup-versions.outputs.versions) }}
      fail-fast: false
    env:
        CUDA_VISIBLE_DEVICES: all
        NVIDIA_VISIBLE_DEVICES: all
        NVIDIA_DRIVER_CAPABILITIES: all
        CUDA_HOME: /usr/local/cuda
        LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
        DOCKER_IMAGE_TAG: isaac-lab-compat:${{ github.ref_name }}-${{ github.sha }}-${{ matrix.isaacsim_version }}

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
        lfs: true

    - name: Build Docker Image
      uses: ./.github/actions/docker-build
      with:
        image-tag: ${{ env.DOCKER_IMAGE_TAG }}
        isaacsim-base-image: ${{ env.ISAACSIM_BASE_IMAGE }}
        isaacsim-version: ${{ matrix.isaacsim_version }}

    - name: Run IsaacLab Tasks Tests
      uses: ./.github/actions/run-tests
      with:
        test-path: "tools"
        result-file: "isaaclab-tasks-compat-report.xml"
        container-name: "isaac-lab-tasks-compat-test-$$"
        image-tag: ${{ env.DOCKER_IMAGE_TAG }}
        pytest-options: ""
        filter-pattern: "isaaclab_tasks"

    - name: Copy All Test Results from IsaacLab Tasks Container
      run: |
        CONTAINER_NAME="isaac-lab-tasks-compat-test-$$"
        if docker ps -a | grep -q $CONTAINER_NAME; then
          echo "Copying all test results from IsaacLab Tasks container..."
          docker cp $CONTAINER_NAME:/workspace/isaaclab/tests/isaaclab-tasks-compat-report.xml reports/ 2>/dev/null || echo "No test results to copy from IsaacLab Tasks container"
        fi

    - name: Upload IsaacLab Tasks Test Results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: isaaclab-tasks-compat-results-${{ matrix.isaacsim_version }}
        path: reports/isaaclab-tasks-compat-report.xml
        retention-days: 7
        compression-level: 9

  test-general-compat:
    needs: setup-versions
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    strategy:
      matrix:
        isaacsim_version: ${{ fromJson(needs.setup-versions.outputs.versions) }}
      fail-fast: false
    env:
        CUDA_VISIBLE_DEVICES: all
        NVIDIA_VISIBLE_DEVICES: all
        NVIDIA_DRIVER_CAPABILITIES: all
        CUDA_HOME: /usr/local/cuda
        LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
        DOCKER_IMAGE_TAG: isaac-lab-compat:${{ github.ref_name }}-${{ github.sha }}-${{ matrix.isaacsim_version }}

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
        lfs: true

    - name: Build Docker Image
      uses: ./.github/actions/docker-build
      with:
        image-tag: ${{ env.DOCKER_IMAGE_TAG }}
        isaacsim-base-image: ${{ env.ISAACSIM_BASE_IMAGE }}
        isaacsim-version: ${{ matrix.isaacsim_version }}

    - name: Run General Tests
      uses: ./.github/actions/run-tests
      with:
        test-path: "tools"
        result-file: "general-tests-compat-report.xml"
        container-name: "isaac-lab-general-compat-test-$$"
        image-tag: ${{ env.DOCKER_IMAGE_TAG }}
        pytest-options: ""
        filter-pattern: "not isaaclab_tasks"

    - name: Copy All Test Results from General Tests Container
      run: |
        CONTAINER_NAME="isaac-lab-general-compat-test-$$"
        if docker ps -a | grep -q $CONTAINER_NAME; then
          echo "Copying all test results from General Tests container..."
          docker cp $CONTAINER_NAME:/workspace/isaaclab/tests/general-tests-compat-report.xml reports/ 2>/dev/null || echo "No test results to copy from General Tests container"
        fi

    - name: Upload General Test Results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: general-tests-compat-results-${{ matrix.isaacsim_version }}
        path: reports/general-tests-compat-report.xml
        retention-days: 7
        compression-level: 9

  combine-compat-results:
    needs: [test-isaaclab-tasks-compat, test-general-compat]
    runs-on: [self-hosted, gpu]
    if: always()

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
        lfs: false

    - name: Create Reports Directory
      run: |
        mkdir -p reports

    - name: Download All Test Results
      uses: actions/download-artifact@v4
      with:
        pattern: '*-compat-results-*'
        path: reports/
        merge-multiple: true
      continue-on-error: true

    - name: Combine All Test Results
      uses: ./.github/actions/combine-results
      with:
        tests-dir: "reports"
        output-file: "reports/combined-compat-results.xml"

    - name: Upload Combined Test Results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: daily-compat-${{ github.run_id }}-combined-test-results
        path: reports/combined-compat-results.xml
        retention-days: 30
        compression-level: 9

    - name: Report Test Results
      uses: dorny/test-reporter@v1
      if: always()
      with:
        name: IsaacLab Compatibility Test Results (${{ github.event_name }})
        path: reports/combined-compat-results.xml
        reporter: java-junit
        max-annotations: '50'
        report-title: "IsaacLab Compatibility Test Results - ${{ github.event_name }} - ${{ github.ref_name }}"

  notify-compatibility-status:
    needs: [setup-versions, combine-compat-results]
    runs-on: [self-hosted, gpu]
    if: always()

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
        lfs: false

    - name: Create Compatibility Report
      run: |
        TRIGGER_INFO="**Trigger:** ${{ github.event_name }}"
        ISAACSIM_VERSIONS="${{ join(fromJson(needs.setup-versions.outputs.versions), ', ') }}"
        echo "## Daily Backwards Compatibility Test Results" > compatibility-report.md
        echo "" >> compatibility-report.md
        echo "$TRIGGER_INFO" >> compatibility-report.md
        echo "**IsaacSim Versions Tested:** $ISAACSIM_VERSIONS" >> compatibility-report.md
        echo "**Branch:** ${{ github.ref_name }}" >> compatibility-report.md
        echo "**Commit:** ${{ github.sha }}" >> compatibility-report.md
        echo "**Run ID:** ${{ github.run_id }}" >> compatibility-report.md
        echo "" >> compatibility-report.md
        echo "### Test Status:" >> compatibility-report.md
        echo "- Results: ${{ needs.combine-compat-results.result }}" >> compatibility-report.md
        echo "" >> compatibility-report.md
        echo "### Artifacts:" >> compatibility-report.md
        echo "- [Combined Test Results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> compatibility-report.md
        echo "" >> compatibility-report.md
        echo "---" >> compatibility-report.md
        echo "*This report was generated automatically by the daily compatibility workflow.*" >> compatibility-report.md

    - name: Upload Compatibility Report
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: compatibility-report-${{ github.run_id }}
        path: compatibility-report.md
        retention-days: 30