|
| 1 | +name: Low-Risk PR Evaluation |
| 2 | + |
| 3 | +on: |
| 4 | + pull_request: |
| 5 | + types: [opened, reopened, synchronize] |
| 6 | + workflow_dispatch: |
| 7 | + inputs: |
| 8 | + pr_number: |
| 9 | + description: "PR number to evaluate" |
| 10 | + required: true |
| 11 | + type: number |
| 12 | + |
| 13 | +permissions: |
| 14 | + pull-requests: write |
| 15 | + contents: read |
| 16 | + |
| 17 | +jobs: |
| 18 | + evaluate: |
| 19 | + runs-on: ubuntu-latest |
| 20 | + |
| 21 | + steps: |
| 22 | + - name: Checkout repo |
| 23 | + uses: actions/checkout@v4 |
| 24 | + |
| 25 | + - name: Resolve PR number |
| 26 | + id: pr |
| 27 | + env: |
| 28 | + GH_TOKEN: ${{ github.token }} |
| 29 | + run: | |
| 30 | + if [ -n "${{ inputs.pr_number }}" ]; then |
| 31 | + echo "number=${{ inputs.pr_number }}" >> "$GITHUB_OUTPUT" |
| 32 | + else |
| 33 | + echo "number=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT" |
| 34 | + fi |
| 35 | +
|
| 36 | + - name: Remove stale low-risk-change label |
| 37 | + uses: actions/github-script@v7 |
| 38 | + with: |
| 39 | + script: | |
| 40 | + const prNumber = ${{ steps.pr.outputs.number }}; |
| 41 | + try { |
| 42 | + await github.rest.issues.removeLabel({ |
| 43 | + owner: context.repo.owner, |
| 44 | + repo: context.repo.repo, |
| 45 | + issue_number: prNumber, |
| 46 | + name: "low-risk-change", |
| 47 | + }); |
| 48 | + core.info(`Removed stale low-risk-change label from PR #${prNumber}.`); |
| 49 | + } catch (error) { |
| 50 | + if (error.status !== 404) throw error; |
| 51 | + core.info("No low-risk-change label to remove."); |
| 52 | + } |
| 53 | +
|
| 54 | + - name: Fetch PR metadata and diff |
| 55 | + id: pr-data |
| 56 | + env: |
| 57 | + GH_TOKEN: ${{ github.token }} |
| 58 | + PR_NUMBER: ${{ steps.pr.outputs.number }} |
| 59 | + run: | |
| 60 | + gh pr view "$PR_NUMBER" --json title,body --template '{{.title}}' > /tmp/pr_title.txt |
| 61 | + gh pr view "$PR_NUMBER" --json title,body --template '{{.body}}' > /tmp/pr_body.txt |
| 62 | + gh pr view "$PR_NUMBER" --json files --jq '.files[].path' > /tmp/pr_files.txt |
| 63 | + gh pr diff "$PR_NUMBER" > /tmp/pr_diff.txt |
| 64 | +
|
| 65 | + # Fail closed if diff exceeds model window — large PRs could hide risky changes |
| 66 | + DIFF_LIMIT=100000 |
| 67 | + DIFF_SIZE=$(wc -c < /tmp/pr_diff.txt) |
| 68 | + if [ "$DIFF_SIZE" -gt "$DIFF_LIMIT" ]; then |
| 69 | + echo "PR diff is ${DIFF_SIZE} chars (> ${DIFF_LIMIT}); too large for automated evaluation." |
| 70 | + echo "oversized=true" >> "$GITHUB_OUTPUT" |
| 71 | + else |
| 72 | + echo "oversized=false" >> "$GITHUB_OUTPUT" |
| 73 | + cp /tmp/pr_diff.txt /tmp/pr_diff_truncated.txt |
| 74 | + fi |
| 75 | +
|
| 76 | + - name: Fail fast for oversized diffs |
| 77 | + id: oversized |
| 78 | + if: steps.pr-data.outputs.oversized == 'true' |
| 79 | + run: | |
| 80 | + echo "qualifies=false" >> "$GITHUB_OUTPUT" |
| 81 | + echo "scope=Diff too large for automated evaluation" >> "$GITHUB_OUTPUT" |
| 82 | + echo "reasoning=This PR's diff exceeds the size limit for automated low-risk evaluation. Manual review required." >> "$GITHUB_OUTPUT" |
| 83 | +
|
| 84 | + - name: Check for restricted paths |
| 85 | + id: path-check |
| 86 | + if: steps.pr-data.outputs.oversized == 'false' |
| 87 | + run: | |
| 88 | + RESTRICTED_PATTERN='^(\.github/workflows/|(auth|security|migrations)/|.*/(auth|security|migrations)/)' |
| 89 | + if grep -qE "$RESTRICTED_PATTERN" /tmp/pr_files.txt; then |
| 90 | + echo "Restricted paths detected:" |
| 91 | + grep -E "$RESTRICTED_PATTERN" /tmp/pr_files.txt |
| 92 | + echo "blocked=true" >> "$GITHUB_OUTPUT" |
| 93 | + else |
| 94 | + echo "blocked=false" >> "$GITHUB_OUTPUT" |
| 95 | + fi |
| 96 | +
|
| 97 | + - name: Fail fast for restricted paths |
| 98 | + id: restricted |
| 99 | + if: steps.path-check.outputs.blocked == 'true' |
| 100 | + run: | |
| 101 | + echo "qualifies=false" >> "$GITHUB_OUTPUT" |
| 102 | + echo "scope=Changes include restricted paths (workflows, auth, migrations, etc.)" >> "$GITHUB_OUTPUT" |
| 103 | + echo "reasoning=This PR modifies files in restricted directories that require manual review per policy." >> "$GITHUB_OUTPUT" |
| 104 | +
|
| 105 | + - name: Evaluate PR with OpenAI |
| 106 | + id: evaluate |
| 107 | + if: steps.pr-data.outputs.oversized == 'false' && steps.path-check.outputs.blocked == 'false' |
| 108 | + env: |
| 109 | + OPENAI_API_KEY: ${{ secrets.LOW_RISK_OPENAI_API_KEY }} |
| 110 | + run: | |
| 111 | + POLICY=$(cat docs/LOW_RISK_PULL_REQUESTS.md) |
| 112 | + PR_TITLE=$(cat /tmp/pr_title.txt) |
| 113 | + PR_BODY=$(cat /tmp/pr_body.txt) |
| 114 | + PR_DIFF=$(cat /tmp/pr_diff_truncated.txt) |
| 115 | +
|
| 116 | + # Build the JSON payload using jq to handle escaping |
| 117 | + PAYLOAD=$(jq -n \ |
| 118 | + --arg policy "$POLICY" \ |
| 119 | + --arg title "$PR_TITLE" \ |
| 120 | + --arg body "$PR_BODY" \ |
| 121 | + --arg diff "$PR_DIFF" \ |
| 122 | + '{ |
| 123 | + model: "gpt-4.1-mini", |
| 124 | + temperature: 0, |
| 125 | + response_format: { |
| 126 | + type: "json_schema", |
| 127 | + json_schema: { |
| 128 | + name: "low_risk_evaluation", |
| 129 | + strict: true, |
| 130 | + schema: { |
| 131 | + type: "object", |
| 132 | + properties: { |
| 133 | + qualifies: { type: "boolean", description: "true if the PR meets ALL low-risk criteria" }, |
| 134 | + reasoning: { type: "string", description: "2-3 sentence explanation of the assessment" }, |
| 135 | + scope: { type: "string", description: "Brief summary of what the PR changes" } |
| 136 | + }, |
| 137 | + required: ["qualifies", "reasoning", "scope"], |
| 138 | + additionalProperties: false |
| 139 | + } |
| 140 | + } |
| 141 | + }, |
| 142 | + messages: [ |
| 143 | + { |
| 144 | + role: "system", |
| 145 | + content: ("You evaluate pull requests against a low-risk change policy.\n\nHere is the policy:\n\n" + $policy + "\n\nEvaluate the PR and return JSON with exactly these fields:\n- \"qualifies\": boolean (true if the PR meets ALL low-risk criteria)\n- \"reasoning\": string (2-3 sentence explanation of your assessment)\n- \"scope\": string (brief summary of what the PR changes)") |
| 146 | + }, |
| 147 | + { |
| 148 | + role: "user", |
| 149 | + content: ("PR Title: " + $title + "\n\nPR Description:\n" + $body + "\n\nDiff:\n" + $diff) |
| 150 | + } |
| 151 | + ] |
| 152 | + }') |
| 153 | +
|
| 154 | + RESPONSE=$(curl -s --max-time 60 https://api.openai.com/v1/chat/completions \ |
| 155 | + -H "Content-Type: application/json" \ |
| 156 | + -H "Authorization: Bearer $OPENAI_API_KEY" \ |
| 157 | + -d "$PAYLOAD") |
| 158 | +
|
| 159 | + # Extract the assistant's message content |
| 160 | + RESULT=$(echo "$RESPONSE" | jq -r '.choices[0].message.content') |
| 161 | +
|
| 162 | + if [ "$RESULT" = "null" ] || [ -z "$RESULT" ]; then |
| 163 | + echo "OpenAI API error:" |
| 164 | + echo "$RESPONSE" | jq . |
| 165 | + exit 1 |
| 166 | + fi |
| 167 | +
|
| 168 | + echo "$RESULT" > /tmp/evaluation.json |
| 169 | +
|
| 170 | + QUALIFIES=$(echo "$RESULT" | jq -r '.qualifies') |
| 171 | + REASONING=$(echo "$RESULT" | jq -r '.reasoning') |
| 172 | + SCOPE=$(echo "$RESULT" | jq -r '.scope') |
| 173 | +
|
| 174 | + echo "qualifies=$QUALIFIES" >> "$GITHUB_OUTPUT" |
| 175 | +
|
| 176 | + SCOPE_DELIM="$(openssl rand -hex 16)" |
| 177 | + { |
| 178 | + echo "scope<<$SCOPE_DELIM" |
| 179 | + echo "$SCOPE" |
| 180 | + echo "$SCOPE_DELIM" |
| 181 | + } >> "$GITHUB_OUTPUT" |
| 182 | +
|
| 183 | + REASONING_DELIM="$(openssl rand -hex 16)" |
| 184 | + { |
| 185 | + echo "reasoning<<$REASONING_DELIM" |
| 186 | + echo "$REASONING" |
| 187 | + echo "$REASONING_DELIM" |
| 188 | + } >> "$GITHUB_OUTPUT" |
| 189 | +
|
| 190 | + - name: Apply label and comment |
| 191 | + uses: actions/github-script@v7 |
| 192 | + env: |
| 193 | + PR_NUMBER: ${{ steps.pr.outputs.number }} |
| 194 | + QUALIFIES: ${{ steps.oversized.outputs.qualifies || steps.restricted.outputs.qualifies || steps.evaluate.outputs.qualifies }} |
| 195 | + SCOPE: ${{ steps.oversized.outputs.scope || steps.restricted.outputs.scope || steps.evaluate.outputs.scope }} |
| 196 | + REASONING: ${{ steps.oversized.outputs.reasoning || steps.restricted.outputs.reasoning || steps.evaluate.outputs.reasoning }} |
| 197 | + with: |
| 198 | + script: | |
| 199 | + const prNumber = parseInt(process.env.PR_NUMBER, 10); |
| 200 | + const qualifies = process.env.QUALIFIES === 'true'; |
| 201 | + const scope = process.env.SCOPE; |
| 202 | + const reasoning = process.env.REASONING; |
| 203 | +
|
| 204 | + if (qualifies) { |
| 205 | + await github.rest.issues.addLabels({ |
| 206 | + owner: context.repo.owner, |
| 207 | + repo: context.repo.repo, |
| 208 | + issue_number: prNumber, |
| 209 | + labels: ["low-risk-change"], |
| 210 | + }); |
| 211 | +
|
| 212 | + await github.rest.issues.createComment({ |
| 213 | + owner: context.repo.owner, |
| 214 | + repo: context.repo.repo, |
| 215 | + issue_number: prNumber, |
| 216 | + body: [ |
| 217 | + "**Automated low-risk assessment**", |
| 218 | + "", |
| 219 | + `This PR was evaluated against the repository's [Low-Risk Pull Requests](docs/LOW_RISK_PULL_REQUESTS.md) procedure.`, |
| 220 | + `- **Scope:** ${scope}`, |
| 221 | + `- **Exclusions confirmed:** no changes to auth, security settings, database schema, business-critical logic, or external integrations.`, |
| 222 | + `- **Classification:** \`low-risk-change\` under the documented policy.`, |
| 223 | + "", |
| 224 | + `> ${reasoning}`, |
| 225 | + "", |
| 226 | + "This classification allows merging without manual review once all required CI checks are passing and branch protection rules are satisfied.", |
| 227 | + ].join("\n"), |
| 228 | + }); |
| 229 | +
|
| 230 | + core.info(`PR #${prNumber} labeled as low-risk-change.`); |
| 231 | + } else { |
| 232 | + await github.rest.issues.createComment({ |
| 233 | + owner: context.repo.owner, |
| 234 | + repo: context.repo.repo, |
| 235 | + issue_number: prNumber, |
| 236 | + body: [ |
| 237 | + "**Automated low-risk assessment**", |
| 238 | + "", |
| 239 | + `This PR was evaluated against the repository's [Low-Risk Pull Requests](docs/LOW_RISK_PULL_REQUESTS.md) procedure and **does not qualify** as low risk.`, |
| 240 | + "", |
| 241 | + `> ${reasoning}`, |
| 242 | + "", |
| 243 | + "This PR requires a manual review before merging.", |
| 244 | + ].join("\n"), |
| 245 | + }); |
| 246 | +
|
| 247 | + core.info(`PR #${prNumber} does not qualify as low-risk.`); |
| 248 | + } |
0 commit comments