Daily update #6
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily update | |
| on: | |
| schedule: | |
| # 07:00 UTC = 15:00 Beijing time, Mon-Fri (arXiv publishes new listings on weekdays) | |
| - cron: '0 7 * * 1-5' | |
| workflow_dispatch: | |
| inputs: | |
| backfill_start: | |
| description: 'Optional: backfill start date (YYYY-MM-DD). Leave empty for normal daily run.' | |
| required: false | |
| backfill_end: | |
| description: 'Optional: backfill end date (YYYY-MM-DD).' | |
| required: false | |
| jobs: | |
| update: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Fetch Scholar Inbox (API) | |
| if: ${{ github.event.inputs.backfill_start == '' }} | |
| continue-on-error: true | |
| env: | |
| SCHOLAR_INBOX_SHA_KEY: ${{ secrets.SCHOLAR_INBOX_SHA_KEY }} | |
| run: python fetch_scholar_inbox.py --days 3 --debug | |
| - name: Fetch arxivist digest email | |
| if: ${{ github.event.inputs.backfill_start == '' }} | |
| continue-on-error: true | |
| env: | |
| GMAIL_ADDRESS: ${{ secrets.GMAIL_ADDRESS }} | |
| GMAIL_APP_PASSWORD: ${{ secrets.GMAIL_APP_PASSWORD }} | |
| run: python fetch_arxivist_email.py --debug | |
| - name: Fetch arXiv (daily) | |
| if: ${{ github.event.inputs.backfill_start == '' }} | |
| continue-on-error: true | |
| run: python fetch_arxiv.py --mode daily --days 3 --debug | |
| - name: Fetch arXiv (backfill) | |
| if: ${{ github.event.inputs.backfill_start != '' }} | |
| run: > | |
| python fetch_arxiv.py --mode backfill | |
| --start ${{ github.event.inputs.backfill_start }} | |
| --end ${{ github.event.inputs.backfill_end }} | |
| --debug | |
| - name: Classify pending papers | |
| continue-on-error: true | |
| env: | |
| GPT_API_BASE: ${{ secrets.GPT_API_BASE }} | |
| GPT_KEY: ${{ secrets.GPT_KEY }} | |
| GPT_MODEL: ${{ secrets.GPT_MODEL }} | |
| run: python classify_pending.py --limit 400 --debug | |
| - name: Generate README | |
| run: python generate_readme.py | |
| - name: Generate webpage data | |
| run: python generate_site_data.py | |
| - name: Generate semantic search embeddings | |
| continue-on-error: true | |
| run: python generate_embeddings.py | |
| - name: Commit and push changes | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| git add -A | |
| git diff --cached --quiet && echo "No changes to commit" && exit 0 | |
| git commit -m "Automated update on $(date -u '+%Y-%m-%d')" | |
| git push |