Source Event Pipeline #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Source Event Pipeline | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| raw_items_path: | |
| description: "Raw post/news CSV path." | |
| required: false | |
| default: "data/live/source_items.csv" | |
| type: string | |
| aliases_path: | |
| description: "Symbol alias CSV path." | |
| required: false | |
| default: "config/core_us_equity_aliases.csv" | |
| type: string | |
| watchlist_path: | |
| description: "Watchlist CSV path." | |
| required: false | |
| default: "data/live/political_watchlist.csv" | |
| type: string | |
| commit_outputs: | |
| description: "Commit generated live CSV outputs back to data/live." | |
| required: false | |
| default: "false" | |
| type: choice | |
| options: | |
| - "false" | |
| - "true" | |
| schedule: | |
| - cron: "45 12 * * 6" | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }} | |
| cancel-in-progress: false | |
| jobs: | |
| build-source-events: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| - name: Install package | |
| run: python -m pip install -e . | |
| - name: Extract source mentions | |
| env: | |
| RAW_ITEMS_PATH: ${{ github.event.inputs.raw_items_path || 'data/live/source_items.csv' }} | |
| ALIASES_PATH: ${{ github.event.inputs.aliases_path || 'config/core_us_equity_aliases.csv' }} | |
| WATCHLIST_PATH: ${{ github.event.inputs.watchlist_path || 'data/live/political_watchlist.csv' }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p data/output/source_event_pipeline | |
| python scripts/extract_source_mentions.py \ | |
| --raw-items "${RAW_ITEMS_PATH}" \ | |
| --aliases "${ALIASES_PATH}" \ | |
| --output data/output/source_event_pipeline/source_events.csv | |
| python scripts/build_tracker.py \ | |
| --watchlist "${WATCHLIST_PATH}" \ | |
| --events data/output/source_event_pipeline/source_events.csv \ | |
| --output data/output/source_event_pipeline/source_tracker.csv | |
| - name: Publish live CSV outputs to repository | |
| env: | |
| COMMIT_OUTPUTS: ${{ github.event_name == 'schedule' && 'true' || github.event.inputs.commit_outputs || 'false' }} | |
| run: | | |
| set -euo pipefail | |
| if [ "${COMMIT_OUTPUTS}" != "true" ]; then | |
| echo "Live output commit disabled." | |
| exit 0 | |
| fi | |
| mkdir -p data/live | |
| cp data/output/source_event_pipeline/source_events.csv data/live/source_events.csv | |
| cp data/output/source_event_pipeline/source_events.csv data/live/political_events.csv | |
| cp data/output/source_event_pipeline/source_tracker.csv data/live/source_tracker.csv | |
| MANIFEST_PATHS=( | |
| data/live/source_items.csv | |
| data/live/source_events.csv | |
| data/live/political_events.csv | |
| data/live/source_tracker.csv | |
| ) | |
| if [ -f data/live/source_fetch_status.json ]; then | |
| MANIFEST_PATHS+=(data/live/source_fetch_status.json) | |
| fi | |
| python scripts/write_live_manifest.py \ | |
| --base-dir . \ | |
| --output data/live/source_manifest.json \ | |
| "${MANIFEST_PATHS[@]}" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add data/live/source_events.csv data/live/political_events.csv data/live/source_tracker.csv data/live/source_manifest.json | |
| if git diff --cached --quiet; then | |
| echo "No live source output changes to commit." | |
| else | |
| git commit -m "Update live source events [skip ci]" | |
| git push | |
| fi | |
| - name: Upload source event artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: source-event-pipeline | |
| path: data/output/source_event_pipeline/ | |
| if-no-files-found: error |