[DLS] Models update on self-hosted runners (by @dmichalo via workflow_dispatch) #38
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "[DLS] Models update on self-hosted runners" | |
| run-name: "[DLS] Models update on self-hosted runners (by @${{ github.actor }} via ${{ github.event_name }})" | |
| on: | |
| schedule: | |
| - cron: '0 5 * * MON' # 5:00 UTC each Monday | |
| workflow_dispatch: | |
| inputs: | |
| models_to_download: | |
| description: 'Which models to download?' | |
| required: true | |
| default: 'all' | |
| type: choice | |
| options: | |
| - all | |
| - public | |
| - omz | |
| - specific_model | |
| specific_model_name: | |
| description: 'What SPECIFIC model do you want to download?' | |
| required: false | |
| type: string | |
| specific_model_path: | |
| description: 'Where do you want to download your SPECIFIC model? (public/omz)' | |
| required: false | |
| type: string | |
| runner_labels: | |
| description: "List of runner(s) labels (example: DLS-TGL-02,DLS-ARL-01)" | |
| required: false | |
| type: string | |
| runner_labels_windows: | |
| description: "List of Windows runner(s) labels (example: DLS-ARL-10)" | |
| required: false | |
| type: string | |
| permissions: {} | |
| env: | |
| MODELS_PATH: "$HOME/models" | |
| MODELS_PATH_WINDOWS: 'C:\\models' | |
| MODELS_DIR_MIN_SIZE_MB: 10 | |
| DEFAULT_LABELS: "DLS-ARL-01,DLS-ARL-02,DLS-ARL-03,DLS-TGL-01,DLS-TGL-02,DLS-TGL-03,DLS-TGL-04,DLS-TGL-05" | |
| DEFAULT_LABELS_WINDOWS: "DLS-ARL-10" | |
| jobs: | |
| setup-runners: | |
| name: Set runners to execute update | |
| runs-on: ubuntu-latest | |
| outputs: | |
| linux_runners: ${{ steps.set-labels.outputs.linux_runners }} | |
| windows_runners: ${{ steps.set-labels.outputs.windows_runners }} | |
| steps: | |
| - name: Set runners labels | |
| id: set-labels | |
| env: | |
| RAW_LINUX_LABELS: ${{ inputs.runner_labels }} | |
| RAW_WIN_LABELS: ${{ inputs.runner_labels_windows }} | |
| DEF_LINUX_LABELS: ${{ env.DEFAULT_LABELS }} | |
| DEF_WIN_LABELS: ${{ env.DEFAULT_LABELS_WINDOWS }} | |
| run: | | |
| LINUX_LABELS="$RAW_LINUX_LABELS" | |
| if [ -z "$LINUX_LABELS" ]; then | |
| LINUX_LABELS="$DEF_LINUX_LABELS" | |
| fi | |
| LINUX_ARRAY=$(echo "$LINUX_LABELS" | jq -Rc 'split(",")') | |
| echo "linux_runners=$LINUX_ARRAY" >> $GITHUB_OUTPUT | |
| WIN_LABELS="$RAW_WIN_LABELS" | |
| if [ -z "$WIN_LABELS" ]; then | |
| WIN_LABELS="$DEF_WIN_LABELS" | |
| fi | |
| WIN_ARRAY=$(echo "$WIN_LABELS" | jq -Rc 'split(",")') | |
| echo "windows_runners=$WIN_ARRAY" >> $GITHUB_OUTPUT | |
| update_linux_hosts: | |
| name: Update on Linux runners | |
| needs: setup-runners | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: ${{ fromJson(needs.setup-runners.outputs.linux_runners) }} | |
| runs-on: | |
| - ${{ matrix.runner }} | |
| steps: | |
| - name: Get script | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 | |
| with: | |
| persist-credentials: false | |
| path: dlstreamer-repo | |
| - name: Download only specific model | |
| env: | |
| model_path: ${{ inputs.specific_model_path }} | |
| model_name: ${{ inputs.specific_model_name }} | |
| if: ${{ inputs.models_to_download == 'specific_model' }} | |
| run: | | |
| MODEL_PATH=$model_path | |
| MODEL_NAME=$model_name | |
| export MODELS_PATH=${{ env.MODELS_PATH }} | |
| echo "MODEL_PATH=$MODEL_PATH" | |
| echo "MODEL_NAME=$MODEL_NAME" | |
| echo "Downloading specific model ${{ env.MODELS_PATH }}/$MODEL_PATH/$MODEL_NAME" >> $GITHUB_STEP_SUMMARY | |
| if [ -d "${{ env.MODELS_PATH }}/$MODEL_PATH/$MODEL_NAME" ]; then | |
| echo "Removing $MODEL_NAME directory" | |
| rm -rf "${{ env.MODELS_PATH }}/$MODEL_PATH/$MODEL_NAME" | |
| if [[ "$MODEL_PATH" == "public" ]]; then | |
| ./dlstreamer-repo/samples/download_public_models.sh "$MODEL_NAME" coco128 | |
| elif [[ "$MODEL_PATH" == "omz" ]]; then | |
| mkdir -p .virtualenvs/dlstreamer | |
| python3 -m venv .virtualenvs/dlstreamer | |
| source .virtualenvs/dlstreamer/bin/activate | |
| pip3 install --no-cache-dir --upgrade tensorflow==2.19.1 openvino-dev[onnx]==2024.6.0 torch==2.8.0 | |
| export MODELS_PATH=${{ env.MODELS_PATH }} | |
| ./dlstreamer-repo/samples/download_omz_models.sh $MODEL_NAME | |
| deactivate | |
| fi | |
| fi | |
| - name: Prepare directories - public models | |
| if: ${{ inputs.models_to_download == 'public' || inputs.models_to_download == 'all' || github.event_name == 'schedule' }} | |
| run: | | |
| # Prepare directories with backup of existing ones | |
| # 1) mv public_old public_old_old | |
| # 2) mv public public_old | |
| if [ -d "${{ env.MODELS_PATH }}/public_old" ]; then | |
| echo "Changing directories names: public_old -> public_old_old" | |
| [ -d "${{ env.MODELS_PATH }}/public_old_old" ] && rm -r "${{ env.MODELS_PATH }}/public_old_old" | |
| mv "${{ env.MODELS_PATH }}/public_old" "${{ env.MODELS_PATH }}/public_old_old" | |
| fi | |
| if [ -d "${{ env.MODELS_PATH }}/public" ]; then | |
| echo "Changing directories names: public -> public_old" | |
| [ -d "${{ env.MODELS_PATH }}/public_old" ] && rm -r "${{ env.MODELS_PATH }}/public_old" | |
| mv "${{ env.MODELS_PATH }}/public" "${{ env.MODELS_PATH }}/public_old" | |
| fi | |
| - name: Prepare directories - OMZ models | |
| if: ${{ inputs.models_to_download == 'omz' || inputs.models_to_download == 'all' || github.event_name == 'schedule' }} | |
| run: | | |
| # Prepare directories with backup of existing ones | |
| # 1) mv intel intel_old | |
| # 2) mv intel_old intel_old_old | |
| if [ -d "${{ env.MODELS_PATH }}/intel_old" ]; then | |
| echo "Changing directories names: intel_old -> intel_old_old" | |
| [ -d "${{ env.MODELS_PATH }}/intel_old_old" ] && rm -r "${{ env.MODELS_PATH }}/intel_old_old" | |
| mv "${{ env.MODELS_PATH }}/intel_old" "${{ env.MODELS_PATH }}/intel_old_old" | |
| fi | |
| if [ -d "${{ env.MODELS_PATH }}/intel" ]; then | |
| echo "Changing directories names: intel -> intel_old" | |
| [ -d "${{ env.MODELS_PATH }}/intel_old" ] && rm -r "${{ env.MODELS_PATH }}/intel_old" | |
| mv "${{ env.MODELS_PATH }}/intel" "${{ env.MODELS_PATH }}/intel_old" | |
| fi | |
| - name: Download public models | |
| id: download-public | |
| if: ${{ inputs.models_to_download == 'public' || inputs.models_to_download == 'all' || github.event_name == 'schedule' }} | |
| run: | | |
| export MODELS_PATH=${{ env.MODELS_PATH }} | |
| echo "Downloading public models" | |
| ./dlstreamer-repo/samples/download_public_models.sh all coco128 | |
| - name: Download OMZ models (setup venv, dependencies and download) | |
| id: download-omz | |
| if: ${{ inputs.models_to_download == 'omz' || inputs.models_to_download == 'all' || github.event_name == 'schedule' }} | |
| run: | | |
| mkdir -p .virtualenvs/dlstreamer | |
| python3 -m venv .virtualenvs/dlstreamer | |
| source .virtualenvs/dlstreamer/bin/activate | |
| pip3 install --no-cache-dir --upgrade tensorflow==2.19.1 openvino-dev[onnx]==2024.6.0 torch==2.8.0 | |
| export MODELS_PATH=${{ env.MODELS_PATH }} | |
| ./dlstreamer-repo/samples/download_omz_models.sh | |
| deactivate | |
| - name: Verify public models downloading | |
| id: verify-public | |
| if: > | |
| steps.download-public.outcome == 'success' && | |
| (inputs.models_to_download == 'public' || inputs.models_to_download == 'all' || github.event_name == 'schedule') | |
| run: | | |
| if [ -z "$( ls -A ${{ env.MODELS_PATH }}/public )" ]; then | |
| echo "Public models not downloaded correctly - directory not created ❌" | |
| echo "Public models not downloaded - directory not created ❌" >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| else | |
| echo "Public models directory created" | |
| echo "Public models directory created" >> $GITHUB_STEP_SUMMARY | |
| DIR_SIZE_MB=$(du -sm "${{ env.MODELS_PATH }}/public" | cut -f1) | |
| echo "Public models directory size: ${DIR_SIZE_MB} MB" | |
| echo "Public models directory size: ${DIR_SIZE_MB} MB" >> $GITHUB_STEP_SUMMARY | |
| if [ "$DIR_SIZE_MB" -ge "${{ env.MODELS_DIR_MIN_SIZE_MB }}" ]; then | |
| echo "Public models downloaded ✅" | |
| echo "Public models downloaded ✅" >> $GITHUB_STEP_SUMMARY | |
| else | |
| Public models not downloaded correctly - directory too small ❌" | |
| Public models not downloaded correctly - directory too small ❌" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| fi | |
| - name: Verify OMZ models downloading | |
| id: verify-omz | |
| if: > | |
| steps.download-omz.outcome == 'success' && | |
| (inputs.models_to_download == 'omz' || inputs.models_to_download == 'all' || github.event_name == 'schedule') | |
| run: | | |
| if [ -z "$( ls -A ${{ env.MODELS_PATH }}/intel )" ]; then | |
| echo "Open Model Zoo models not downloaded correctly - directory not created ❌" | |
| echo "Open Model Zoo models not downloaded - directory not created ❌" >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| else | |
| echo "Open Model Zoo models directory created" | |
| echo "Open Model Zoo models directory created" >> $GITHUB_STEP_SUMMARY | |
| DIR_SIZE_MB=$(du -sm "${{ env.MODELS_PATH }}/intel" | cut -f1) | |
| echo "Open Model Zoo models directory size: ${DIR_SIZE_MB} MB" | |
| echo "Open Model Zoo models directory size: ${DIR_SIZE_MB} MB" >> $GITHUB_STEP_SUMMARY | |
| if [ "$DIR_SIZE_MB" -ge "${{ env.MODELS_DIR_MIN_SIZE_MB }}" ]; then | |
| echo "Open Model Zoo models downloaded ✅" | |
| echo "Open Model Zoo models downloaded ✅" >> $GITHUB_STEP_SUMMARY | |
| else | |
| Open Model Zoo models not downloaded correctly - directory too small ❌" | |
| Open Model Zoo models not downloaded correctly - directory too small ❌" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| fi | |
| - name: Manage public models directories after success | |
| if: ${{ steps.verify-public.outcome == 'success' && steps.download-public.outcome == 'success' }} | |
| run: | | |
| [ -d "${{ env.MODELS_PATH }}/public_old_old" ] && rm -rf "${{ env.MODELS_PATH }}/public_old_old" | |
| echo "New public models set up ✅" >> $GITHUB_STEP_SUMMARY | |
| - name: Manage OMZ models directories after success | |
| if: ${{ steps.verify-omz.outcome == 'success' && steps.download-omz.outcome == 'success' }} | |
| run: | | |
| [ -d "${{ env.MODELS_PATH }}/intel_old_old" ] && rm -rf "${{ env.MODELS_PATH }}/intel_old_old" | |
| echo "New Open Model Zoo models set up ✅" >> $GITHUB_STEP_SUMMARY | |
| - name: Restore public models directories after fail | |
| if: ${{ always() && steps.download-public.outcome != 'skipped' && !(steps.verify-public.outcome == 'success' && steps.download-public.outcome == 'success') }} | |
| run: | | |
| [ -d "${{ env.MODELS_PATH }}/public" ] && rm -rf "${{ env.MODELS_PATH }}/public" | |
| [ -d "${{ env.MODELS_PATH }}/public_old" ] && mv "${{ env.MODELS_PATH }}/public_old" "${{ env.MODELS_PATH }}/public" | |
| [ -d "${{ env.MODELS_PATH }}/public_old_old" ] && mv "${{ env.MODELS_PATH }}/public_old_old" "${{ env.MODELS_PATH }}/public_old" | |
| echo "Old public models restored" >> $GITHUB_STEP_SUMMARY | |
| - name: Restore OMZ models directories after fail | |
| if: ${{ always() && steps.download-omz.outcome != 'skipped' && !(steps.verify-omz.outcome == 'success' && steps.download-omz.outcome == 'success') }} | |
| run: | | |
| [ -d "${{ env.MODELS_PATH }}/intel" ] && rm -rf "${{ env.MODELS_PATH }}/intel" | |
| [ -d "${{ env.MODELS_PATH }}/intel_old" ] && mv "${{ env.MODELS_PATH }}/intel_old" "${{ env.MODELS_PATH }}/intel" | |
| [ -d "${{ env.MODELS_PATH }}/intel_old_old" ] && mv "${{ env.MODELS_PATH }}/intel_old_old" "${{ env.MODELS_PATH }}/intel_old" | |
| echo "Old Open Model Zoo models restored" >> $GITHUB_STEP_SUMMARY | |
| - name: Clean up | |
| if: always() | |
| run: | | |
| rm -rf dlstreamer-repo .virtualenvs | |
| rm -rf $HOME/.virtualenvs | |
| update_windows_hosts: | |
| name: Update on Windows runners | |
| needs: [setup-runners, update_linux_hosts] | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: ${{ fromJson(needs.setup-runners.outputs.windows_runners) }} | |
| runs-on: | |
| - ${{ matrix.runner }} | |
| steps: | |
| - name: Initialize Emojis | |
| shell: powershell | |
| run: | | |
| $emojis = @{ | |
| "EPASS" = [char]0x2705 # ✅ | |
| "EFAIL" = [char]0x274C # ❌ | |
| "EWARN" = [char]0x26A0 # ⚠️ | |
| } | |
| $lines = New-Object System.Collections.Generic.List[string] | |
| foreach ($name in $emojis.Keys) { | |
| $lines.Add("$name=$($emojis[$name])") | |
| } | |
| $utf8NoBom = New-Object System.Text.UTF8Encoding $false | |
| [System.IO.File]::AppendAllLines($env:GITHUB_ENV, $lines, $utf8NoBom) | |
| - name: Install Rclone | |
| shell: powershell | |
| run: | | |
| $installDir = "C:\rclone" | |
| if (!(Test-Path $installDir)) { New-Item -ItemType Directory -Path $installDir -Force } | |
| Write-Host "Downloading Rclone..." | |
| $zipPath = Join-Path $env:TEMP "rclone.zip" | |
| Invoke-WebRequest https://downloads.rclone.org/rclone-current-windows-amd64.zip -OutFile $zipPath | |
| Write-Host "Extracting..." | |
| Expand-Archive $zipPath -DestinationPath $env:TEMP -Force | |
| $exe = Get-ChildItem -Path $env:TEMP -Filter rclone.exe -Recurse | Select-Object -First 1 | |
| Copy-Item $exe.FullName -Destination "$installDir\rclone.exe" -Force | |
| echo "$installDir" >> $env:GITHUB_PATH | |
| Write-Host "Rclone is ready at $installDir" | |
| - name: Verify rclone installation | |
| run: rclone version | |
| - name: Prepare Windows directories (Rotation) | |
| shell: powershell | |
| run: | | |
| $rawBase = "${{ env.MODELS_PATH_WINDOWS }}" | |
| $base = Join-Path $rawBase "models" | |
| if (!(Test-Path $base)) { New-Item -ItemType Directory -Path $base } | |
| foreach ($type in "public", "intel") { | |
| $dir = Join-Path $base $type | |
| $old = Join-Path $base "$($type)_old" | |
| $old_old = Join-Path $base "$($type)_old_old" | |
| if (Test-Path $old) { | |
| if (Test-Path $old_old) { Remove-Item -Recurse -Force $old_old } | |
| Rename-Item -Path $old -NewName "$($type)_old_old" | |
| } | |
| if (Test-Path $dir) { | |
| Rename-Item -Path $dir -NewName "$($type)_old" | |
| } | |
| New-Item -ItemType Directory -Path $dir | |
| } | |
| - name: Sync Models via Rclone (Windows) | |
| shell: powershell | |
| env: | |
| REMOTE_USER: ${{ secrets.DLS_MODEL_SYNC_SSH_USER }} | |
| REMOTE_HOST: ${{ secrets.DLS_MODEL_SYNC_SSH_HOST }} | |
| SSH_KEY_CONTENT: ${{ secrets.DLS_MODEL_SYNC_SSH_PRIVATE_KEY }} | |
| run: | | |
| $ErrorActionPreference = "Continue" | |
| [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 | |
| $keyPath = Join-Path $env:TEMP "id_rsa_rclone" | |
| $cleanKey = ($env:SSH_KEY_CONTENT -replace "`r`n", "`n").Trim() | |
| Set-Content -Path $keyPath -Value $cleanKey -Encoding Ascii | |
| $folders = @("intel", "public") | |
| $fixedKeyPath = $keyPath.Replace('\', '/') | |
| $remotePrefix = ":sftp,host='$env:REMOTE_HOST',user='$env:REMOTE_USER',key_file='$fixedKeyPath',ask_password=false:" | |
| $base = Join-Path "C:\models" "models" | |
| if (Get-Content $keyPath | Select-Object -First 1 | Select-String "BEGIN") { | |
| Write-Host "[OK] Private key format looks valid." | |
| } else { | |
| Write-Error "[ERROR] Private key file is corrupted!" | |
| exit 1 | |
| } | |
| $remoteUser = $env:REMOTE_USER | |
| $remoteBase = "/home/$remoteUser/models/models" | |
| foreach ($folder in $folders) { | |
| $remotePath = "$remoteBase/$folder" | |
| $localPath = Join-Path $base $folder | |
| Write-Host "`n--- Checking Remote: $remotePath ---" | |
| $testPath = rclone lsf "$remotePrefix$remotePath" --max-depth 1 2>$null | |
| if ($null -eq $testPath -or [string]::IsNullOrWhiteSpace($testPath)) { | |
| Write-Host "[SKIP] Remote directory '$remotePath' not found or empty. Skipping." -ForegroundColor Yellow | |
| continue | |
| } | |
| Write-Host "Syncing from $remotePath to $localPath" | |
| rclone copy "$remotePrefix$remotePath" "$localPath" ` | |
| --multi-thread-streams 4 ` | |
| --transfers 4 ` | |
| --sftp-ask-password=false ` | |
| --verbose ` | |
| --stats-log-level NOTICE | |
| } | |
| if (Test-Path $keyPath) { Remove-Item $keyPath } | |
| - name: Verify and Summarize (Windows) | |
| shell: powershell | |
| run: | | |
| $summary = "" | |
| $rawBase = "${{ env.MODELS_PATH_WINDOWS }}" | |
| $base = Join-Path $rawBase "models" | |
| foreach ($type in "public", "intel") { | |
| $path = Join-Path $base $type | |
| if (Test-Path $path) { | |
| $size = (Get-ChildItem $path -Recurse | Measure-Object -Property Length -Sum).Sum / 1MB | |
| $formattedSize = "{0:N2}" -f $size | |
| if ($size -ge ${{ env.MODELS_DIR_MIN_SIZE_MB }}) { | |
| $status = "$type models directory size: $formattedSize MB $($env:EPASS) " | |
| } else { | |
| $status = "$type models directory too small: $formattedSize MB $($env:EFAIL)" | |
| } | |
| } else { | |
| $status = "$type directory not found! $($env:EWARN)" | |
| } | |
| Write-Host $status | |
| $summary += "$status`n" | |
| } | |
| $summary | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 |