Daily update #19
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily update | |
| on: | |
| schedule: | |
| # 07:00 UTC = 15:00 Beijing time, Mon-Fri (arXiv publishes new listings on weekdays) | |
| - cron: '0 7 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| backfill_start: | |
| description: 'Optional: backfill start date (YYYY-MM-DD). Leave empty for normal daily run.' | |
| required: false | |
| backfill_end: | |
| description: 'Optional: backfill end date (YYYY-MM-DD).' | |
| required: false | |
| jobs: | |
| update: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Fetch Scholar Inbox (API) | |
| if: ${{ github.event.inputs.backfill_start == '' }} | |
| continue-on-error: true | |
| env: | |
| SCHOLAR_INBOX_SHA_KEY: ${{ secrets.SCHOLAR_INBOX_SHA_KEY }} | |
| run: python fetch_scholar_inbox.py --days 3 --debug | |
| - name: Fetch arxivist digest email | |
| if: ${{ github.event.inputs.backfill_start == '' }} | |
| continue-on-error: true | |
| env: | |
| GMAIL_ADDRESS: ${{ secrets.GMAIL_ADDRESS }} | |
| GMAIL_APP_PASSWORD: ${{ secrets.GMAIL_APP_PASSWORD }} | |
| run: python fetch_arxivist_email.py --debug | |
| - name: Fetch arXiv (daily) | |
| if: ${{ github.event.inputs.backfill_start == '' }} | |
| continue-on-error: true | |
| run: python fetch_arxiv.py --mode daily --days 3 --debug | |
| - name: Fetch arXiv (backfill) | |
| if: ${{ github.event.inputs.backfill_start != '' }} | |
| run: > | |
| python fetch_arxiv.py --mode backfill | |
| --start ${{ github.event.inputs.backfill_start }} | |
| --end ${{ github.event.inputs.backfill_end }} | |
| --debug | |
| - name: Classify pending papers | |
| continue-on-error: true | |
| env: | |
| GPT_API_BASE: ${{ secrets.GPT_API_BASE }} | |
| GPT_KEY: ${{ secrets.GPT_KEY }} | |
| GPT_MODEL: ${{ secrets.GPT_MODEL }} | |
| run: python classify_pending.py --limit 400 --debug | |
| - name: Generate README | |
| run: python generate_readme.py | |
| - name: Generate webpage data | |
| run: python generate_site_data.py | |
| - name: Generate semantic search embeddings | |
| continue-on-error: true | |
| run: python generate_embeddings.py | |
| - name: Get current date | |
| id: date | |
| run: echo "date=$(date -u '+%Y-%m-%d')" >> "$GITHUB_OUTPUT" | |
| - name: Create Pull Request | |
| uses: peter-evans/create-pull-request@v7 | |
| with: | |
| commit-message: "Automated update on ${{ steps.date.outputs.date }}" | |
| title: "Automated update on ${{ steps.date.outputs.date }}" | |
| body: | | |
| This PR was created automatically by the daily workflow. | |
| Please review the generated updates before merging. | |
| branch: chore/daily-update-${{ steps.date.outputs.date }} | |
| delete-branch: false |