ci: Optimize workflow performance with caching and parallel jobs

.dockerignore

@@ -3,7 +3,6 @@
 # dependencies
 node_modules
 
-
 # misc
 .DS_Store
 .env*
@@ -16,6 +15,16 @@ dist/
 
 /data/**/*.js
 
-
 # production
 /server/dist
+
+# CI/CD and development files
+.github/
+.cursor/
+.bruno/
+*.md
+!README.md
+*.test.ts
+.git/
+.gitignore
+.dockerignore

.github/workflows/build.yml

@@ -8,6 +8,10 @@ on:
     branches: 
       - "**"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 env:
   REGISTRY_IMAGE: |
     ghcr.io/tcgdex/server
@@ -17,6 +21,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     name: Build TCGdex Server
+    timeout-minutes: 60
     steps:
 
       - name: Checkout
@@ -65,6 +70,18 @@ jobs:
         with:
           bun-version: latest
 
+      - name: Cache Bun dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.bun/install/cache
+            ~/.bun/install/global
+            node_modules
+            server/node_modules
+          key: ${{ runner.os }}-bun-build-${{ hashFiles('bun.lockb', 'server/bun.lockb') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-build-
+
       - name: Pre build server
         run: |
           bun install --frozen-lockfile
@@ -84,6 +101,6 @@ jobs:
           file: ./Dockerfile.github-actions
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
+          cache-from: type=gha,scope=${{ github.ref_name }}
           push: ${{ !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
-          cache-to: type=gha,mode=max
+          cache-to: type=gha,mode=max,scope=${{ github.ref_name }}

.github/workflows/test.yml

@@ -2,66 +2,167 @@ name: Test the Data
 
 on:
   push:
-    branches: 
+    branches:
       - master
-  pull_request_target:
-    branches: 
-      - "**"
+  pull_request:
+    branches:
+      - '**'
 
 permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
+  export-git-metadata:
+    runs-on: ubuntu-latest
+    permissions: {}
+    timeout-minutes: 30
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Checkout full history
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+
+      - name: Cache Bun installs
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.bun/install/cache
+            ~/.bun/install/global
+          key: ${{ runner.os }}-bun-metadata-${{ hashFiles('bun.lockb', 'server/bun.lockb') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-metadata-
+
+      - name: Install server deps
+        run: |
+          cd server
+          bun install --frozen-lockfile
+
+      - name: Export git metadata
+        run: |
+          cd server
+          bun run compile --export-git-metadata
+
+      - name: Upload git metadata artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: git-metadata
+          path: server/git-metadata.json
+          retention-days: 1
+          if-no-files-found: error
+          compression-level: 9
+
   test:
+    needs: export-git-metadata
     runs-on: ${{ matrix.os }}
     permissions: {}
+    timeout-minutes: 120
+    defaults:
+      run:
+        shell: bash
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
+      fail-fast: false
 
     steps:
-    - name: Checkout
-      uses: actions/checkout@v6
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-        persist-credentials: false
-
-    - name: Setup BunJS
-      uses: oven-sh/setup-bun@v2
-
-    - name: Install deps
-      run: |
-        bun install -g @usebruno/cli
-        bun install --frozen-lockfile
-        cd server
-        bun install --frozen-lockfile
-        bun run compile
-
-    - name: Validate the data & the server
-      run: |
-        bun run validate
-        cd server
-        bun run --bun validate
-
-    - name: Validate some requests
-      shell: bash
-      run: |
-        set -euo pipefail
-        cd server
-        bun run start &
-        SERVER_PID=$!
-        cd ..
-
-        ATTEMPTS=0
-        until curl -sSf http://127.0.0.1:3000/status > /dev/null; do
-          ATTEMPTS=$((ATTEMPTS + 1))
-          if [ $ATTEMPTS -ge 60 ]; then
-            echo "Server did not become ready within 60 seconds" >&2
-            kill $SERVER_PID
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+          fetch-depth: 1
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+
+      - name: Cache Bun installs
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.bun/install/cache
+            ~/.bun/install/global
+          key: ${{ matrix.os }}-bun-${{ hashFiles('bun.lockb', 'server/bun.lockb') }}
+          restore-keys: |
+            ${{ matrix.os }}-bun-
+
+      - name: Download git metadata
+        uses: actions/download-artifact@v4
+        with:
+          name: git-metadata
+          path: server/
+
+      - name: Install CLI & root deps
+        run: |
+          bun install -g @usebruno/cli
+          bun install --frozen-lockfile
+
+      - name: Install server deps
+        run: |
+          cd server
+          bun install --frozen-lockfile
+
+      - name: Compile with imported metadata
+        run: |
+          cd server
+          set -o pipefail
+          bun run compile --import-git-metadata 2>&1 | tee ../compile.log
+          cd ..
+          if grep -q "could not load file" compile.log; then
+            echo "::error::Compiler reported missing card files" >&2
+            cat compile.log
             exit 1
           fi
-          sleep 1
-        done
+          rm -f compile.log
 
-        cd .bruno
-        bru run --env Developpement
+      - name: Run TypeScript validation (root)
+        run: bun run validate
 
-        kill $SERVER_PID
+      - name: Run TypeScript validation (server)
+        run: |
+          cd server
+          bun run --bun validate
+
+      - name: Start API server
+        run: |
+          cd server
+          MAX_WORKERS=1 bun run start > ../server.log 2>&1 &
+          echo $! > ../server.pid
+
+      - name: Wait for server readiness
+        run: |
+          for i in {1..180}; do
+            if curl -sf http://127.0.0.1:3000/status > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Server failed to start" >&2
+          cat server.log || true
+          exit 1
+
+      - name: Run Bruno integration suite
+        run: |
+          cd .bruno
+          bru run --env Developpement
+
+      - name: Stop API server
+        if: always()
+        run: |
+          if [ -f server.pid ]; then
+            kill "$(cat server.pid)" || true
+            rm -f server.pid
+          fi
+          cat server.log || true
+          rm -f server.log

.gitignore

@@ -13,3 +13,6 @@ scripts/pokedexIdFixer/pokemon-mapping.json
 scripts/pokedexIdFixer/fix-preview.txt
 scripts/pokedexIdFixer/fix-log.txt
 scripts/pokedexIdFixer/audit-report.txt
+
+# Git metadata cache (generated by compiler)
+server/git-metadata.json

server/compiler/index.ts

@@ -26,6 +26,11 @@ const DIST_FOLDER = './generated'
 	console.log('\n2. Loading informations from GIT')
 	await loadLastEdits()
 
+	if (process.argv.includes('--export-git-metadata')) {
+		console.log('\nGit metadata export complete.')
+		process.exit(0)
+	}
+
 	console.log('\n3. Compiling Files')
 
 	// Process each languages

server/compiler/utils/util.ts

@@ -2,7 +2,7 @@ import { objectSize } from '@dzeio/object-util'
 import Queue from '@dzeio/queue'
 import { glob } from 'glob'
 import { exec, spawn } from 'node:child_process'
-import { writeFileSync } from 'node:fs'
+import { readFileSync, statSync, writeFileSync } from 'node:fs'
 import { Card, Languages, Set, SupportedLanguages } from '../../../interfaces'
 import * as legals from '../../../meta/legals'
 interface fileCacheInterface {
@@ -11,6 +11,10 @@ interface fileCacheInterface {
 
 export const DB_PATH = "../"
 
+const EXPORT_METADATA = process.argv.includes('--export-git-metadata')
+const IMPORT_METADATA = process.argv.includes('--import-git-metadata')
+const METADATA_FILE = './git-metadata.json'
+
 const fileCache: fileCacheInterface = {}
 
 /**
@@ -131,6 +135,22 @@ function runCommand(command: string, useSpawn = true): Promise<string> {
 
 const lastEditsCache: Record<string, string> = {}
 export async function loadLastEdits() {
+	if (IMPORT_METADATA) {
+		console.log('Importing git metadata from file...')
+		try {
+			const data = readFileSync(METADATA_FILE, 'utf-8')
+			const imported = JSON.parse(data)
+			Object.assign(lastEditsCache, imported)
+			const stats = statSync(METADATA_FILE)
+			console.log('Loaded', objectSize(lastEditsCache), 'file timestamps from cache')
+			console.log('Metadata file size:', (stats.size / 1024 / 1024).toFixed(2), 'MB')
+			return
+		} catch (error) {
+			console.error('Failed to import git metadata:', error)
+			throw new Error('Cannot import git metadata. File missing or corrupt.')
+		}
+	}
+
 	console.log('Loading Git File Tree...')
 	const firstCommand = 'git ls-tree -r --name-only HEAD ../data'
 	const files = (await runCommand(firstCommand)).split('\n')
@@ -157,19 +177,17 @@ export async function loadLastEdits() {
 				console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`)
 			}
 		}))
-		// try {
-		// 	// don't really know why but it does not correctly execute the command when using Spawn
-		// 	lastEditsCache[file] = await runCommand(`git log -1 --pretty="format:%cd" --date=iso-strict "${file}"`, false)
-		// } catch {
-		// 	console.warn('could not load file', file, 'hope it does not break everything else lol')
-		// }
-		// processed++
-		// if (processed % 1000 === 0) {
-		// 	console.log('loaded', processed, 'out of', files.length, 'files', `(${(processed / files.length * 100).toFixed(0)}%)`)
-		// }
 	}
 	await queue.waitEnd()
 	console.log('done loading files', objectSize(lastEditsCache))
+
+	if (EXPORT_METADATA) {
+		console.log('Exporting git metadata to file...')
+		writeFileSync(METADATA_FILE, JSON.stringify(lastEditsCache))
+		const stats = statSync(METADATA_FILE)
+		console.log('Exported', objectSize(lastEditsCache), 'file timestamps')
+		console.log('Metadata file size:', (stats.size / 1024 / 1024).toFixed(2), 'MB')
+	}
 }
 
 export function getLastEdit(path: string): string {