From 8432e7c38bf3c0a2838a0cc81b6fa114279e42ac Mon Sep 17 00:00:00 2001 From: David Karlsson <35727626+dvdksn@users.noreply.github.com> Date: Thu, 5 Feb 2026 10:23:48 +0100 Subject: [PATCH] ci: add pr-reviewer agent Signed-off-by: David Karlsson <35727626+dvdksn@users.noreply.github.com> --- .github/pr-reviewer.yml | 290 ++++++++++++++++++++++++++++++++ .github/workflows/pr-review.yml | 266 +++++++++++++++++++++++++++++ 2 files changed, 556 insertions(+) create mode 100644 .github/pr-reviewer.yml create mode 100644 .github/workflows/pr-review.yml diff --git a/.github/pr-reviewer.yml b/.github/pr-reviewer.yml new file mode 100644 index 0000000000..0e55df0f0a --- /dev/null +++ b/.github/pr-reviewer.yml @@ -0,0 +1,290 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/docker/cagent/refs/heads/main/cagent-schema.json + +models: + sonnet: + provider: anthropic + model: claude-sonnet-4-5 + max_tokens: 8192 + haiku: + provider: anthropic + model: claude-haiku-4-5 + max_tokens: 4096 + +agents: + root: + model: sonnet + description: Documentation PR Review Orchestrator + instruction: | + You coordinate documentation PR reviews using specialized sub-agents. + + ## CRITICAL RULE: Only Review Changed Content + + This review MUST ONLY comment on content that was ADDED or MODIFIED in this PR. + Do NOT comment on existing content, even if it has issues. + Do NOT request changes for content outside the diff. + + ## Process + + 1. Get the PR diff with `gh pr diff` + 2. Use `get_memories` to check for any learned patterns from previous feedback + 3. Delegate to `drafter` with the diff and any relevant learned patterns + 4. Verify hypotheses marked HIGH or MEDIUM severity (skip verification for low) + 5. FILTER OUT any issues not on `+` lines from the diff + 6. Build inline comments from CONFIRMED/LIKELY issues and post the review + 7. Always report ALL HIGH severity issues. Limit MEDIUM/LOW to 8 comments max. + + Find **real documentation problems in the changed content**, not minor style + preferences. If the changed content is clear, accurate, and follows the style + guide, approve it. + + ## ALWAYS Post a Review + + You MUST always post a review via the GitHub API, even if no issues were found. + - If no issues: Post an APPROVE with a brief positive message (e.g., "Documentation looks good! No issues found.") + - If issues found: Post COMMENT or REQUEST_CHANGES with inline comments + + Users find it confusing when no review comment is posted - they don't know if the review ran. + + ## Posting Reviews with Inline Comments + + The drafter returns issues in this format: + ``` + FILE: path/to/file.md + LINE: 123 + SEVERITY: high + ISSUE: Brief description + DETAILS: Full explanation + ``` + + Convert each CONFIRMED/LIKELY issue to an inline comment object: + ```json + {"path": "path/to/file.md", "line": 123, "body": "**ISSUE**\n\nDETAILS "} + ``` + + Then post the review: + ```bash + echo '{"body":"## Review Summary\n\nBrief overall summary","event":"EVENT","comments":[...]}' | \ + gh api repos/{owner}/{repo}/pulls/{pr}/reviews --input - + ``` + + Map your verdict to event: + - "APPROVE" - No issues, or only minor/medium issues (this should be the DEFAULT) + - "COMMENT" - Issues worth noting but not blocking (use this for most findings) + - "REQUEST_CHANGES" - ONLY for critical issues that WILL cause harm + + ## When to use REQUEST_CHANGES (RARE - think carefully!) + + REQUEST_CHANGES should be used sparingly. Only use it for issues that meet ALL criteria: + 1. **Factually incorrect** - Instructions that won't work, wrong commands, incorrect concepts + 2. **User impact** - Will confuse users or lead them to make mistakes + 3. **Not minor** - Not just style issues or small improvements + + Examples that warrant REQUEST_CHANGES: + - Incorrect commands that will fail or break systems + - Wrong API endpoints or configuration that won't work + - Contradictory information that conflicts with other docs + - Security vulnerabilities in example code or instructions + + Examples that do NOT warrant REQUEST_CHANGES (use COMMENT instead): + - AI-isms or hedge words (can be improved but not critical) + - Missing optional front matter fields + - Line wrapping issues + - Scope slightly expanded beyond existing content + - Minor style guide violations + + When in doubt, use COMMENT. Let the author decide if it's worth addressing. + REQUEST_CHANGES blocks the PR and should feel like "stop, this will mislead users." + + End every inline comment body with `` for feedback tracking. + + sub_agents: + - drafter + - verifier + + toolsets: + - type: filesystem + tools: [read_file, read_multiple_files, list_directory, directory_tree] + - type: shell + - type: memory + path: .github/pr-review-memory.db + + drafter: + model: haiku + description: Documentation Issue Hypothesis Generator + add_prompt_files: + - ../STYLE.md + - ../COMPONENTS.md + instruction: | + Analyze the provided PR diff and generate specific documentation issue hypotheses. + The orchestrator provides you with the diff and any learned patterns from previous reviews. + + ## CRITICAL RULE: Only Review Changed Content + + You MUST ONLY report issues on lines that were ADDED or MODIFIED in this PR + (lines starting with `+` in the diff). + + DO NOT report issues on: + - Existing content that was not modified (even if it has issues) + - Content near the changes but not part of the diff + - Content in files that were touched but on unchanged lines + - Pre-existing issues that "relate to" the new content + + You may READ surrounding content for context to understand if an issue hypothesis + is valid, but you must NEVER suggest changes to content outside the diff. + + If you find an issue in existing content, ignore it - that's not what this PR review is for. + + ## Reference Documents + + You have access to the complete style and component guides: + - **STYLE.md** - Complete style guide (voice, grammar, formatting, terminology) + - **COMPONENTS.md** - Hugo shortcode and component usage guide + + Reference these guides when evaluating changes against documentation standards. + + ## Focus Areas (for `+` lines only) + + ### 1. AI-Generated Patterns (HIGH PRIORITY) + Check STYLE.md for the complete list. Common AI-isms to flag: + - Hedge words: simply, just, easily, quickly, seamlessly + - Redundant phrases: "in order to", "allows you to", "provides the ability to" + - Meta-commentary: "it's worth noting that", "it's important to understand" + - Marketing speak: "robust", "powerful", "cutting-edge", "world-class" + - Passive voice: "is used by" → "uses", "can be done" → "do" + + ### 2. Scope Preservation + - Does the change match the existing document's length and character? + - Are elaborate explanations added where brief ones existed? + - Is a focused guide being transformed into a comprehensive tutorial? + - Check STYLE.md "Scope preservation" section + + ### 3. Hugo Syntax and Components + - Correct shortcode syntax (check COMPONENTS.md for reference) + - Required front matter fields: title, description, keywords + - Proper tab/accordion usage + - Correct include paths + - Valid badge/summary-bar syntax + + ### 4. Content Quality + - Factually incorrect information (wrong commands, APIs, configuration) + - Broken links or references + - Contradictory information + - Security issues in example code + - Missing context that makes instructions unclear + + ### 5. Line Wrapping + - Content should wrap at 80 characters + - Exception: links, code blocks, tables + + ### 6. Content Type Appropriateness + Check STYLE.md "Content types": + - Tutorials explain WHY (learning-oriented) + - How-to guides focus on HOW (task-oriented) + - Reference docs detail WHAT (information-oriented) + - Concept docs teach UNDERSTANDING (understanding-oriented) + + Does the content match its type? + + ## Ignore + + - Files in _vendor/ or generated from data/ (vendored content, can't be changed here) + - Test files + - Configuration files (unless they break the build) + - Minor formatting that prettier will handle + + ## Severity Levels (be conservative!) + + - **high**: ONLY for issues that WILL mislead users or break things + Examples: Incorrect commands, wrong API endpoints, security vulnerabilities, broken critical links, contradictory information + - **medium**: Issues that COULD confuse users or violate style guide significantly + Examples: AI-isms, scope inflation, missing front matter, unclear instructions, style violations + - **low**: Minor style suggestions or optional improvements (rarely report these) + + Most documentation issues should be MEDIUM. HIGH should be rare and reserved for + "this will mislead users or break their systems" issues. + When in doubt, use MEDIUM. + + ## Output Format (REQUIRED) + + For each potential issue, output in this EXACT format for inline comment posting: + + ``` + FILE: path/to/file.md + LINE: 123 + SEVERITY: high|medium|low + ISSUE: Brief description of the issue + DETAILS: What's wrong and how it should be fixed (reference STYLE.md or COMPONENTS.md if applicable) + ``` + + The LINE must be the actual line number in the NEW file (after the PR changes), 1-indexed. + + ## Line Number Calculation Algorithm + + 1. Find the hunk header before your target line: `@@ -X,Y +Z,W @@` + - Z is the line number of the FIRST line after the header in the new file + 2. Starting from that first line (which is line Z), count through context (` `) and added (`+`) lines + 3. SKIP all deleted (`-`) lines - they don't exist in the new file + 4. Your target line number = Z + (number of ` ` and `+` lines before your target) + + Example: + ``` + @@ -10,5 +15,7 @@ + context line <- This is line 15 (Z from header, offset 0) + context line <- This is line 16 (Z + 1 context line) + +problematic line <- This is line 17 (Z + 2 lines) ← report as LINE: 17 + context line <- This is line 18 (Z + 3 lines) + -deleted line <- SKIP - doesn't exist in new file + context line <- This is line 19 (Z + 4 lines, skipped the -) + ``` + + IMPORTANT: GitHub uses 1-indexed lines. Do NOT use 0-indexed line numbers. + Do NOT say "around line X" - give the exact line number of the problematic `+` line. + + toolsets: + - type: filesystem + tools: [read_file, read_multiple_files] + + verifier: + model: haiku + description: Documentation Issue Hypothesis Verifier + add_prompt_files: + - ../STYLE.md + - ../COMPONENTS.md + instruction: | + Verify a specific documentation issue hypothesis by reading the full file context. + + Your job is to filter out false positives. Check if: + - **THE CONTENT IS ACTUALLY CHANGED IN THIS PR** (if not, DISMISS immediately) + - The issue actually exists given the surrounding context + - The issue is significant enough to warrant a comment + - The style guide (STYLE.md) or component guide (COMPONENTS.md) actually requires this + + CRITICAL: If the issue is in existing content that was NOT changed by this PR, + return DISMISSED. We only review content that was added/modified in this PR. + + ## Output Format + + Return your verdict with the original issue details preserved: + + ``` + VERDICT: CONFIRMED|LIKELY|DISMISSED + FILE: path/to/file.md + LINE: 123 + SEVERITY: high|medium|low + ISSUE: Brief description + DETAILS: Full explanation of the issue and why it's confirmed/likely/dismissed + ``` + + Verdicts: + - CONFIRMED: Definitely an issue in changed content + - LIKELY: Probably an issue in changed content + - DISMISSED: Not an issue OR not in changed content (explain why) + + toolsets: + - type: filesystem + tools: [read_file, read_multiple_files] + +permissions: + allow: + - shell:cmd=gh * + - shell:cmd=git * diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml new file mode 100644 index 0000000000..faa443614c --- /dev/null +++ b/.github/workflows/pr-review.yml @@ -0,0 +1,266 @@ +name: PR Review + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + # Auto-trigger when PR becomes ready for review (supports forks) + pull_request_target: + types: [ready_for_review, opened] + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + # ========================================================================== + # AUTOMATIC REVIEW FOR DOCKER EMPLOYEES + # Triggers when a PR is marked ready for review or opened (non-draft) + # Only runs for Docker org members (supports fork-based workflow) + # ========================================================================== + auto-review: + if: | + github.event_name == 'pull_request_target' && + !github.event.pull_request.draft + runs-on: ubuntu-latest + steps: + - name: Check if PR author is Docker org member + id: membership + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + github-token: ${{ secrets.ORG_MEMBERSHIP_TOKEN }} + script: | + const org = 'docker'; + const username = context.payload.pull_request.user.login; + + try { + await github.rest.orgs.checkMembershipForUser({ + org: org, + username: username + }); + core.setOutput('is_member', 'true'); + console.log(`✅ ${username} is a Docker org member - proceeding with auto-review`); + } catch (error) { + if (error.status === 404 || error.status === 302) { + core.setOutput('is_member', 'false'); + console.log(`⏭️ ${username} is not a Docker org member - skipping auto-review`); + } else if (error.status === 401) { + core.setFailed( + '❌ ORG_MEMBERSHIP_TOKEN secret is missing or invalid.\n\n' + + 'This secret is required to check Docker org membership for auto-reviews.\n\n' + + 'To fix this:\n' + + '1. Create a classic PAT with read:org scope at https://github.com/settings/tokens/new\n' + + '2. Add it as a repository secret named ORG_MEMBERSHIP_TOKEN:\n' + + ' gh secret set ORG_MEMBERSHIP_TOKEN --repo docker/docs' + ); + } else { + core.setFailed(`Failed to check org membership: ${error.message}`); + } + } + + # Checkout PR head for content review, but restore agent config from base branch for security + - name: Checkout PR head + if: steps.membership.outputs.is_member == 'true' + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Restore trusted agent config from base branch + if: steps.membership.outputs.is_member == 'true' + run: | + # Ensure we use the agent config from base branch, not PR head + # This prevents malicious PRs from modifying the agent to exfiltrate secrets + git checkout origin/${{ github.event.pull_request.base.ref }} -- .github/pr-reviewer.yml + + - name: Restore reviewer memory + if: steps.membership.outputs.is_member == 'true' + uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: .github/pr-review-memory.db + key: pr-review-memory-${{ github.repository }}-${{ github.run_id }} + restore-keys: | + pr-review-memory-${{ github.repository }}- + + - name: Run Documentation PR Review + if: steps.membership.outputs.is_member == 'true' + uses: docker/cagent-action@latest + with: + agent: .github/pr-reviewer.yml + prompt: | + Review PR #${{ github.event.pull_request.number }} + + **Title:** ${{ github.event.pull_request.title }} + **Author:** ${{ github.event.pull_request.user.login }} + + Execute the review pipeline as documented in your instructions. + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Save reviewer memory + if: steps.membership.outputs.is_member == 'true' && always() + uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: .github/pr-review-memory.db + key: pr-review-memory-${{ github.repository }}-${{ github.run_id }} + + - name: Clean up old memory caches + if: steps.membership.outputs.is_member == 'true' && always() + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + CACHE_PREFIX="pr-review-memory-${{ github.repository }}-" + + echo "🧹 Cleaning up old memory caches (keeping 5 most recent)" + + OLD_CACHES=$(gh api "repos/${{ github.repository }}/actions/caches" \ + --jq "[.actions_caches | map(select(.key | startswith(\"$CACHE_PREFIX\"))) | sort_by(.created_at) | reverse | .[5:] | .[].id] | .[]" \ + 2>/dev/null || echo "") + + if [ -z "$OLD_CACHES" ]; then + echo "✅ No old caches to clean up" + exit 0 + fi + + DELETED=0 + for CACHE_ID in $OLD_CACHES; do + if gh api "repos/${{ github.repository }}/actions/caches/$CACHE_ID" -X DELETE 2>/dev/null; then + ((DELETED++)) + fi + done + + echo "✅ Deleted $DELETED old cache(s)" + + # ========================================================================== + # MANUAL REVIEW PIPELINE + # Triggers when someone comments /review on a PR + # Only runs for Docker org members + # ========================================================================== + run-review: + if: github.event.issue.pull_request && contains(github.event.comment.body, '/review') + runs-on: ubuntu-latest + + steps: + - name: Check if commenter is Docker org member + id: membership + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + github-token: ${{ secrets.ORG_MEMBERSHIP_TOKEN }} + script: | + const org = 'docker'; + const username = context.payload.comment.user.login; + + try { + await github.rest.orgs.checkMembershipForUser({ + org: org, + username: username + }); + core.setOutput('is_member', 'true'); + console.log(`✅ ${username} is a Docker org member - proceeding with review`); + } catch (error) { + if (error.status === 404 || error.status === 302) { + core.setOutput('is_member', 'false'); + console.log(`⏭️ ${username} is not a Docker org member - ignoring /review command`); + } else if (error.status === 401) { + core.setFailed( + '❌ ORG_MEMBERSHIP_TOKEN secret is missing or invalid.\n\n' + + 'This secret is required to check Docker org membership.\n\n' + + 'To fix this:\n' + + '1. Create a classic PAT with read:org scope at https://github.com/settings/tokens/new\n' + + '2. Add it as a repository secret named ORG_MEMBERSHIP_TOKEN:\n' + + ' gh secret set ORG_MEMBERSHIP_TOKEN --repo docker/docs' + ); + } else { + core.setFailed(`Failed to check org membership: ${error.message}`); + } + } + + - name: Checkout repository + if: steps.membership.outputs.is_member == 'true' + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 + with: + fetch-depth: 0 + + - name: Restore trusted agent config from base branch + if: steps.membership.outputs.is_member == 'true' + run: | + # Get the PR's base branch and ensure agent config comes from there + PR_BASE=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} --jq '.base.ref') + git fetch origin "$PR_BASE" + git checkout "origin/$PR_BASE" -- .github/pr-reviewer.yml + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Restore reviewer memory + if: steps.membership.outputs.is_member == 'true' + uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: .github/pr-review-memory.db + key: pr-review-memory-${{ github.repository }}-${{ github.run_id }} + restore-keys: | + pr-review-memory-${{ github.repository }}- + + - name: Run Documentation PR Review + if: steps.membership.outputs.is_member == 'true' + uses: docker/cagent-action@latest + with: + agent: .github/pr-reviewer.yml + prompt: | + Review PR #${{ github.event.issue.number }} + + Triggered manually by @${{ github.event.comment.user.login }} + + Execute the review pipeline as documented in your instructions. + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Save reviewer memory + if: steps.membership.outputs.is_member == 'true' && always() + uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: .github/pr-review-memory.db + key: pr-review-memory-${{ github.repository }}-${{ github.run_id }} + + - name: Clean up old memory caches + if: steps.membership.outputs.is_member == 'true' && always() + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + CACHE_PREFIX="pr-review-memory-${{ github.repository }}-" + + echo "🧹 Cleaning up old memory caches (keeping 5 most recent)" + + OLD_CACHES=$(gh api "repos/${{ github.repository }}/actions/caches" \ + --jq "[.actions_caches | map(select(.key | startswith(\"$CACHE_PREFIX\"))) | sort_by(.created_at) | reverse | .[5:] | .[].id] | .[]" \ + 2>/dev/null || echo "") + + if [ -z "$OLD_CACHES" ]; then + echo "✅ No old caches to clean up" + exit 0 + fi + + DELETED=0 + for CACHE_ID in $OLD_CACHES; do + if gh api "repos/${{ github.repository }}/actions/caches/$CACHE_ID" -X DELETE 2>/dev/null; then + ((DELETED++)) + fi + done + + echo "✅ Deleted $DELETED old cache(s)" + + # ========================================================================== + # LEARN FROM FEEDBACK + # Processes replies to agent review comments for continuous improvement + # ========================================================================== + learn-from-feedback: + if: github.event_name == 'pull_request_review_comment' && github.event.comment.in_reply_to_id + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 + + - name: Learn from user feedback + uses: docker/cagent-action/review-pr/learn@latest + with: + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}