Update Chat Replays #409
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Chat Replays | |
| on: | |
| workflow_run: | |
| workflows: ["Update Livestreams Data"] | |
| types: | |
| - completed | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| jobs: | |
| update-chat: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install Python dependencies | |
| run: pip install --quiet requests | |
| # ── Step 1: match YouTube streams to Twitch VOD IDs ─────────────────── | |
| - name: Match Twitch VODs to YouTube streams | |
| env: | |
| TWITCH_CLIENT_ID: ${{ secrets.TWITCH_CLIENT_ID }} | |
| TWITCH_CLIENT_SECRET: ${{ secrets.TWITCH_CLIENT_SECRET }} | |
| run: python scripts/match-twitch-vods.py | |
| # ── Step 2: download TwitchDownloader CLI ───────────────────────────── | |
| - name: Download TwitchDownloader CLI | |
| run: | | |
| TDCLI_VERSION=$(curl -fsSL https://api.github.com/repos/lay295/TwitchDownloader/releases/latest \ | |
| | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": "\(.*\)".*/\1/') | |
| echo "TwitchDownloader version: ${TDCLI_VERSION}" | |
| curl -fsSL \ | |
| "https://github.com/lay295/TwitchDownloader/releases/download/${TDCLI_VERSION}/TwitchDownloaderCLI-${TDCLI_VERSION}-Linux-x64.zip" \ | |
| -o tdcli.zip | |
| unzip -q tdcli.zip TwitchDownloaderCLI | |
| chmod +x TwitchDownloaderCLI | |
| # ── Step 3: download chat for any entry that has a VOD but no chat file ─ | |
| - name: Download missing chat replays | |
| env: | |
| TWITCH_CLIENT_ID: ${{ secrets.TWITCH_CLIENT_ID }} | |
| TWITCH_CLIENT_SECRET: ${{ secrets.TWITCH_CLIENT_SECRET }} | |
| run: | | |
| mkdir -p static/chats | |
| # Read livestreams.json and iterate over entries with twitchVodId but no chat file. | |
| # VODs that are still live are skipped — they will be picked up on the next run. | |
| python3 - <<'PYEOF' | |
| import json, os, subprocess, sys, requests | |
| from datetime import datetime | |
| from pathlib import Path | |
| data = json.loads(Path("data/livestreams.json").read_text()) | |
| chats = Path("static/chats") | |
| errors = 0 | |
| # ── Determine if a VOD is still being recorded ───────────────────── | |
| client_id = os.environ.get("TWITCH_CLIENT_ID", "") | |
| client_secret = os.environ.get("TWITCH_CLIENT_SECRET", "") | |
| channel = os.environ.get("TWITCH_CHANNEL", "christitustech") | |
| live_vod_id = None # VOD ID that is currently live (skip it) | |
| def parse_iso(s): | |
| return datetime.fromisoformat(s.replace("Z", "+00:00")) | |
| if client_id and client_secret: | |
| try: | |
| token_resp = requests.post( | |
| "https://id.twitch.tv/oauth2/token", | |
| data={ | |
| "client_id": client_id, | |
| "client_secret": client_secret, | |
| "grant_type": "client_credentials", | |
| }, | |
| timeout=15, | |
| ) | |
| token_resp.raise_for_status() | |
| token = token_resp.json()["access_token"] | |
| hdrs = {"Authorization": f"Bearer {token}", "Client-Id": client_id} | |
| # Check whether the channel is live right now | |
| sr = requests.get( | |
| "https://api.twitch.tv/helix/streams", | |
| params={"user_login": channel}, | |
| headers=hdrs, | |
| timeout=15, | |
| ) | |
| sr.raise_for_status() | |
| streams = sr.json().get("data", []) | |
| if streams: | |
| live_started = parse_iso(streams[0]["started_at"]) | |
| print(f"Channel is LIVE (started {streams[0]['started_at']})") | |
| # Find which pending VOD belongs to the live stream by | |
| # comparing the VOD's created_at to the stream's started_at | |
| for item in data.get("items", []): | |
| vod_id = item.get("twitchVodId") | |
| if not vod_id or (chats / f"{item['videoId']}.json").exists(): | |
| continue | |
| vr = requests.get( | |
| "https://api.twitch.tv/helix/videos", | |
| params={"id": vod_id}, | |
| headers=hdrs, | |
| timeout=15, | |
| ) | |
| vr.raise_for_status() | |
| vods = vr.json().get("data", []) | |
| if vods: | |
| vod_created = parse_iso(vods[0]["created_at"]) | |
| diff = abs((vod_created - live_started).total_seconds()) | |
| if diff < 300: # within 5 minutes → same stream | |
| live_vod_id = vod_id | |
| print(f" VOD {vod_id} is the current live stream — will skip") | |
| break | |
| except Exception as exc: | |
| print(f"Warning: could not determine live status: {exc}", file=sys.stderr) | |
| # ── Download chat for completed VODs only ────────────────────────── | |
| for item in data.get("items", []): | |
| vid_id = item.get("videoId") | |
| vod_id = item.get("twitchVodId") | |
| outfile = chats / f"{vid_id}.json" | |
| if not vod_id or outfile.exists(): | |
| continue # skip: no VOD matched, or already downloaded | |
| if vod_id == live_vod_id: | |
| print(f"Skipping {vid_id} — VOD {vod_id} is still live") | |
| continue | |
| print(f"Downloading chat for VOD {vod_id} → {outfile}") | |
| result = subprocess.run( | |
| [ | |
| "./TwitchDownloaderCLI", | |
| "chatdownload", | |
| "--id", str(vod_id), | |
| "--output", str(outfile), | |
| "--temp-path", "/tmp", | |
| ], | |
| capture_output=True, | |
| text=True, | |
| ) | |
| if result.returncode != 0: | |
| print(f" ERROR: {result.stderr.strip()}", file=sys.stderr) | |
| errors += 1 | |
| outfile.unlink(missing_ok=True) | |
| else: | |
| print(f" OK — {outfile.stat().st_size // 1024} KB") | |
| if errors: | |
| print(f"\n{errors} download(s) failed.", file=sys.stderr) | |
| sys.exit(1) | |
| PYEOF | |
| # ── Step 4: refresh hasChatReplay flags in livestreams.json ─────────── | |
| - name: Refresh livestreams.json (hasChatReplay flags) | |
| env: | |
| YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} | |
| run: python scripts/fetch-livestreams.py | |
| # ── Step 5: commit and push if anything changed ──────────────────────── | |
| - name: Commit and push changes | |
| run: | | |
| git config --local user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --local user.name "github-actions[bot]" | |
| git add data/livestreams.json static/chats/ | |
| git diff --staged --quiet || git commit -m "chore: update chat replays" | |
| git push |