ci: add pr-reviewer agent

Signed-off-by: David Karlsson <35727626+dvdksn@users.noreply.github.com>
2026-03-27 06:18:55 +07:00 · 2026-02-05 10:23:48 +01:00
parent 9f380d429e
commit 8432e7c38b
2 changed files with 556 additions and 0 deletions
--- a/.github/pr-reviewer.yml
+++ b/.github/pr-reviewer.yml
@@ -0,0 +1,290 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/docker/cagent/refs/heads/main/cagent-schema.json
+
+models:
+  sonnet:
+    provider: anthropic
+    model: claude-sonnet-4-5
+    max_tokens: 8192
+  haiku:
+    provider: anthropic
+    model: claude-haiku-4-5
+    max_tokens: 4096
+
+agents:
+  root:
+    model: sonnet
+    description: Documentation PR Review Orchestrator
+    instruction: |
+      You coordinate documentation PR reviews using specialized sub-agents.
+
+      ## CRITICAL RULE: Only Review Changed Content
+
+      This review MUST ONLY comment on content that was ADDED or MODIFIED in this PR.
+      Do NOT comment on existing content, even if it has issues.
+      Do NOT request changes for content outside the diff.
+
+      ## Process
+
+      1. Get the PR diff with `gh pr diff`
+      2. Use `get_memories` to check for any learned patterns from previous feedback
+      3. Delegate to `drafter` with the diff and any relevant learned patterns
+      4. Verify hypotheses marked HIGH or MEDIUM severity (skip verification for low)
+      5. FILTER OUT any issues not on `+` lines from the diff
+      6. Build inline comments from CONFIRMED/LIKELY issues and post the review
+      7. Always report ALL HIGH severity issues. Limit MEDIUM/LOW to 8 comments max.
+
+      Find **real documentation problems in the changed content**, not minor style
+      preferences. If the changed content is clear, accurate, and follows the style
+      guide, approve it.
+
+      ## ALWAYS Post a Review
+
+      You MUST always post a review via the GitHub API, even if no issues were found.
+      - If no issues: Post an APPROVE with a brief positive message (e.g., "Documentation looks good! No issues found.")
+      - If issues found: Post COMMENT or REQUEST_CHANGES with inline comments
+
+      Users find it confusing when no review comment is posted - they don't know if the review ran.
+
+      ## Posting Reviews with Inline Comments
+
+      The drafter returns issues in this format:
+      ```
+      FILE: path/to/file.md
+      LINE: 123
+      SEVERITY: high
+      ISSUE: Brief description
+      DETAILS: Full explanation
+      ```
+
+      Convert each CONFIRMED/LIKELY issue to an inline comment object:
+      ```json
+      {"path": "path/to/file.md", "line": 123, "body": "**ISSUE**\n\nDETAILS <!-- cagent-review -->"}
+      ```
+
+      Then post the review:
+      ```bash
+      echo '{"body":"## Review Summary\n\nBrief overall summary","event":"EVENT","comments":[...]}' | \
+      gh api repos/{owner}/{repo}/pulls/{pr}/reviews --input -
+      ```
+
+      Map your verdict to event:
+      - "APPROVE" - No issues, or only minor/medium issues (this should be the DEFAULT)
+      - "COMMENT" - Issues worth noting but not blocking (use this for most findings)
+      - "REQUEST_CHANGES" - ONLY for critical issues that WILL cause harm
+
+      ## When to use REQUEST_CHANGES (RARE - think carefully!)
+
+      REQUEST_CHANGES should be used sparingly. Only use it for issues that meet ALL criteria:
+      1. **Factually incorrect** - Instructions that won't work, wrong commands, incorrect concepts
+      2. **User impact** - Will confuse users or lead them to make mistakes
+      3. **Not minor** - Not just style issues or small improvements
+
+      Examples that warrant REQUEST_CHANGES:
+      - Incorrect commands that will fail or break systems
+      - Wrong API endpoints or configuration that won't work
+      - Contradictory information that conflicts with other docs
+      - Security vulnerabilities in example code or instructions
+
+      Examples that do NOT warrant REQUEST_CHANGES (use COMMENT instead):
+      - AI-isms or hedge words (can be improved but not critical)
+      - Missing optional front matter fields
+      - Line wrapping issues
+      - Scope slightly expanded beyond existing content
+      - Minor style guide violations
+
+      When in doubt, use COMMENT. Let the author decide if it's worth addressing.
+      REQUEST_CHANGES blocks the PR and should feel like "stop, this will mislead users."
+
+      End every inline comment body with `<!-- cagent-review -->` for feedback tracking.
+
+    sub_agents:
+      - drafter
+      - verifier
+
+    toolsets:
+      - type: filesystem
+        tools: [read_file, read_multiple_files, list_directory, directory_tree]
+      - type: shell
+      - type: memory
+        path: .github/pr-review-memory.db
+
+  drafter:
+    model: haiku
+    description: Documentation Issue Hypothesis Generator
+    add_prompt_files:
+      - ../STYLE.md
+      - ../COMPONENTS.md
+    instruction: |
+      Analyze the provided PR diff and generate specific documentation issue hypotheses.
+      The orchestrator provides you with the diff and any learned patterns from previous reviews.
+
+      ## CRITICAL RULE: Only Review Changed Content
+
+      You MUST ONLY report issues on lines that were ADDED or MODIFIED in this PR
+      (lines starting with `+` in the diff).
+
+      DO NOT report issues on:
+      - Existing content that was not modified (even if it has issues)
+      - Content near the changes but not part of the diff
+      - Content in files that were touched but on unchanged lines
+      - Pre-existing issues that "relate to" the new content
+
+      You may READ surrounding content for context to understand if an issue hypothesis
+      is valid, but you must NEVER suggest changes to content outside the diff.
+
+      If you find an issue in existing content, ignore it - that's not what this PR review is for.
+
+      ## Reference Documents
+
+      You have access to the complete style and component guides:
+      - **STYLE.md** - Complete style guide (voice, grammar, formatting, terminology)
+      - **COMPONENTS.md** - Hugo shortcode and component usage guide
+
+      Reference these guides when evaluating changes against documentation standards.
+
+      ## Focus Areas (for `+` lines only)
+
+      ### 1. AI-Generated Patterns (HIGH PRIORITY)
+      Check STYLE.md for the complete list. Common AI-isms to flag:
+      - Hedge words: simply, just, easily, quickly, seamlessly
+      - Redundant phrases: "in order to", "allows you to", "provides the ability to"
+      - Meta-commentary: "it's worth noting that", "it's important to understand"
+      - Marketing speak: "robust", "powerful", "cutting-edge", "world-class"
+      - Passive voice: "is used by" → "uses", "can be done" → "do"
+
+      ### 2. Scope Preservation
+      - Does the change match the existing document's length and character?
+      - Are elaborate explanations added where brief ones existed?
+      - Is a focused guide being transformed into a comprehensive tutorial?
+      - Check STYLE.md "Scope preservation" section
+
+      ### 3. Hugo Syntax and Components
+      - Correct shortcode syntax (check COMPONENTS.md for reference)
+      - Required front matter fields: title, description, keywords
+      - Proper tab/accordion usage
+      - Correct include paths
+      - Valid badge/summary-bar syntax
+
+      ### 4. Content Quality
+      - Factually incorrect information (wrong commands, APIs, configuration)
+      - Broken links or references
+      - Contradictory information
+      - Security issues in example code
+      - Missing context that makes instructions unclear
+
+      ### 5. Line Wrapping
+      - Content should wrap at 80 characters
+      - Exception: links, code blocks, tables
+
+      ### 6. Content Type Appropriateness
+      Check STYLE.md "Content types":
+      - Tutorials explain WHY (learning-oriented)
+      - How-to guides focus on HOW (task-oriented)
+      - Reference docs detail WHAT (information-oriented)
+      - Concept docs teach UNDERSTANDING (understanding-oriented)
+
+      Does the content match its type?
+
+      ## Ignore
+
+      - Files in _vendor/ or generated from data/ (vendored content, can't be changed here)
+      - Test files
+      - Configuration files (unless they break the build)
+      - Minor formatting that prettier will handle
+
+      ## Severity Levels (be conservative!)
+
+      - **high**: ONLY for issues that WILL mislead users or break things
+        Examples: Incorrect commands, wrong API endpoints, security vulnerabilities, broken critical links, contradictory information
+      - **medium**: Issues that COULD confuse users or violate style guide significantly
+        Examples: AI-isms, scope inflation, missing front matter, unclear instructions, style violations
+      - **low**: Minor style suggestions or optional improvements (rarely report these)
+
+      Most documentation issues should be MEDIUM. HIGH should be rare and reserved for
+      "this will mislead users or break their systems" issues.
+      When in doubt, use MEDIUM.
+
+      ## Output Format (REQUIRED)
+
+      For each potential issue, output in this EXACT format for inline comment posting:
+
+      ```
+      FILE: path/to/file.md
+      LINE: 123
+      SEVERITY: high|medium|low
+      ISSUE: Brief description of the issue
+      DETAILS: What's wrong and how it should be fixed (reference STYLE.md or COMPONENTS.md if applicable)
+      ```
+
+      The LINE must be the actual line number in the NEW file (after the PR changes), 1-indexed.
+
+      ## Line Number Calculation Algorithm
+
+      1. Find the hunk header before your target line: `@@ -X,Y +Z,W @@`
+         - Z is the line number of the FIRST line after the header in the new file
+      2. Starting from that first line (which is line Z), count through context (` `) and added (`+`) lines
+      3. SKIP all deleted (`-`) lines - they don't exist in the new file
+      4. Your target line number = Z + (number of ` ` and `+` lines before your target)
+
+      Example:
+      ```
+      @@ -10,5 +15,7 @@
+       context line       <- This is line 15 (Z from header, offset 0)
+       context line       <- This is line 16 (Z + 1 context line)
+      +problematic line   <- This is line 17 (Z + 2 lines) ← report as LINE: 17
+       context line       <- This is line 18 (Z + 3 lines)
+      -deleted line       <- SKIP - doesn't exist in new file
+       context line       <- This is line 19 (Z + 4 lines, skipped the -)
+      ```
+
+      IMPORTANT: GitHub uses 1-indexed lines. Do NOT use 0-indexed line numbers.
+      Do NOT say "around line X" - give the exact line number of the problematic `+` line.
+
+    toolsets:
+      - type: filesystem
+        tools: [read_file, read_multiple_files]
+
+  verifier:
+    model: haiku
+    description: Documentation Issue Hypothesis Verifier
+    add_prompt_files:
+      - ../STYLE.md
+      - ../COMPONENTS.md
+    instruction: |
+      Verify a specific documentation issue hypothesis by reading the full file context.
+
+      Your job is to filter out false positives. Check if:
+      - **THE CONTENT IS ACTUALLY CHANGED IN THIS PR** (if not, DISMISS immediately)
+      - The issue actually exists given the surrounding context
+      - The issue is significant enough to warrant a comment
+      - The style guide (STYLE.md) or component guide (COMPONENTS.md) actually requires this
+
+      CRITICAL: If the issue is in existing content that was NOT changed by this PR,
+      return DISMISSED. We only review content that was added/modified in this PR.
+
+      ## Output Format
+
+      Return your verdict with the original issue details preserved:
+
+      ```
+      VERDICT: CONFIRMED|LIKELY|DISMISSED
+      FILE: path/to/file.md
+      LINE: 123
+      SEVERITY: high|medium|low
+      ISSUE: Brief description
+      DETAILS: Full explanation of the issue and why it's confirmed/likely/dismissed
+      ```
+
+      Verdicts:
+      - CONFIRMED: Definitely an issue in changed content
+      - LIKELY: Probably an issue in changed content
+      - DISMISSED: Not an issue OR not in changed content (explain why)
+
+    toolsets:
+      - type: filesystem
+        tools: [read_file, read_multiple_files]
+
+permissions:
+  allow:
+    - shell:cmd=gh *
+    - shell:cmd=git *
--- a/.github/workflows/pr-review.yml
+++ b/.github/workflows/pr-review.yml
@@ -0,0 +1,266 @@
+name: PR Review
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  # Auto-trigger when PR becomes ready for review (supports forks)
+  pull_request_target:
+    types: [ready_for_review, opened]
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+
+jobs:
+  # ==========================================================================
+  # AUTOMATIC REVIEW FOR DOCKER EMPLOYEES
+  # Triggers when a PR is marked ready for review or opened (non-draft)
+  # Only runs for Docker org members (supports fork-based workflow)
+  # ==========================================================================
+  auto-review:
+    if: |
+      github.event_name == 'pull_request_target' &&
+      !github.event.pull_request.draft
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if PR author is Docker org member
+        id: membership
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        with:
+          github-token: ${{ secrets.ORG_MEMBERSHIP_TOKEN }}
+          script: |
+            const org = 'docker';
+            const username = context.payload.pull_request.user.login;
+
+            try {
+              await github.rest.orgs.checkMembershipForUser({
+                org: org,
+                username: username
+              });
+              core.setOutput('is_member', 'true');
+              console.log(`✅ ${username} is a Docker org member - proceeding with auto-review`);
+            } catch (error) {
+              if (error.status === 404 || error.status === 302) {
+                core.setOutput('is_member', 'false');
+                console.log(`⏭️ ${username} is not a Docker org member - skipping auto-review`);
+              } else if (error.status === 401) {
+                core.setFailed(
+                  '❌ ORG_MEMBERSHIP_TOKEN secret is missing or invalid.\n\n' +
+                  'This secret is required to check Docker org membership for auto-reviews.\n\n' +
+                  'To fix this:\n' +
+                  '1. Create a classic PAT with read:org scope at https://github.com/settings/tokens/new\n' +
+                  '2. Add it as a repository secret named ORG_MEMBERSHIP_TOKEN:\n' +
+                  '   gh secret set ORG_MEMBERSHIP_TOKEN --repo docker/docs'
+                );
+              } else {
+                core.setFailed(`Failed to check org membership: ${error.message}`);
+              }
+            }
+
+      # Checkout PR head for content review, but restore agent config from base branch for security
+      - name: Checkout PR head
+        if: steps.membership.outputs.is_member == 'true'
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Restore trusted agent config from base branch
+        if: steps.membership.outputs.is_member == 'true'
+        run: |
+          # Ensure we use the agent config from base branch, not PR head
+          # This prevents malicious PRs from modifying the agent to exfiltrate secrets
+          git checkout origin/${{ github.event.pull_request.base.ref }} -- .github/pr-reviewer.yml
+
+      - name: Restore reviewer memory
+        if: steps.membership.outputs.is_member == 'true'
+        uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        with:
+          path: .github/pr-review-memory.db
+          key: pr-review-memory-${{ github.repository }}-${{ github.run_id }}
+          restore-keys: |
+            pr-review-memory-${{ github.repository }}-
+
+      - name: Run Documentation PR Review
+        if: steps.membership.outputs.is_member == 'true'
+        uses: docker/cagent-action@latest
+        with:
+          agent: .github/pr-reviewer.yml
+          prompt: |
+            Review PR #${{ github.event.pull_request.number }}
+
+            **Title:** ${{ github.event.pull_request.title }}
+            **Author:** ${{ github.event.pull_request.user.login }}
+
+            Execute the review pipeline as documented in your instructions.
+          anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+      - name: Save reviewer memory
+        if: steps.membership.outputs.is_member == 'true' && always()
+        uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        with:
+          path: .github/pr-review-memory.db
+          key: pr-review-memory-${{ github.repository }}-${{ github.run_id }}
+
+      - name: Clean up old memory caches
+        if: steps.membership.outputs.is_member == 'true' && always()
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          CACHE_PREFIX="pr-review-memory-${{ github.repository }}-"
+
+          echo "🧹 Cleaning up old memory caches (keeping 5 most recent)"
+
+          OLD_CACHES=$(gh api "repos/${{ github.repository }}/actions/caches" \
+            --jq "[.actions_caches | map(select(.key | startswith(\"$CACHE_PREFIX\"))) | sort_by(.created_at) | reverse | .[5:] | .[].id] | .[]" \
+            2>/dev/null || echo "")
+
+          if [ -z "$OLD_CACHES" ]; then
+            echo "✅ No old caches to clean up"
+            exit 0
+          fi
+
+          DELETED=0
+          for CACHE_ID in $OLD_CACHES; do
+            if gh api "repos/${{ github.repository }}/actions/caches/$CACHE_ID" -X DELETE 2>/dev/null; then
+              ((DELETED++))
+            fi
+          done
+
+          echo "✅ Deleted $DELETED old cache(s)"
+
+  # ==========================================================================
+  # MANUAL REVIEW PIPELINE
+  # Triggers when someone comments /review on a PR
+  # Only runs for Docker org members
+  # ==========================================================================
+  run-review:
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/review')
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check if commenter is Docker org member
+        id: membership
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        with:
+          github-token: ${{ secrets.ORG_MEMBERSHIP_TOKEN }}
+          script: |
+            const org = 'docker';
+            const username = context.payload.comment.user.login;
+
+            try {
+              await github.rest.orgs.checkMembershipForUser({
+                org: org,
+                username: username
+              });
+              core.setOutput('is_member', 'true');
+              console.log(`✅ ${username} is a Docker org member - proceeding with review`);
+            } catch (error) {
+              if (error.status === 404 || error.status === 302) {
+                core.setOutput('is_member', 'false');
+                console.log(`⏭️ ${username} is not a Docker org member - ignoring /review command`);
+              } else if (error.status === 401) {
+                core.setFailed(
+                  '❌ ORG_MEMBERSHIP_TOKEN secret is missing or invalid.\n\n' +
+                  'This secret is required to check Docker org membership.\n\n' +
+                  'To fix this:\n' +
+                  '1. Create a classic PAT with read:org scope at https://github.com/settings/tokens/new\n' +
+                  '2. Add it as a repository secret named ORG_MEMBERSHIP_TOKEN:\n' +
+                  '   gh secret set ORG_MEMBERSHIP_TOKEN --repo docker/docs'
+                );
+              } else {
+                core.setFailed(`Failed to check org membership: ${error.message}`);
+              }
+            }
+
+      - name: Checkout repository
+        if: steps.membership.outputs.is_member == 'true'
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+        with:
+          fetch-depth: 0
+
+      - name: Restore trusted agent config from base branch
+        if: steps.membership.outputs.is_member == 'true'
+        run: |
+          # Get the PR's base branch and ensure agent config comes from there
+          PR_BASE=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} --jq '.base.ref')
+          git fetch origin "$PR_BASE"
+          git checkout "origin/$PR_BASE" -- .github/pr-reviewer.yml
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Restore reviewer memory
+        if: steps.membership.outputs.is_member == 'true'
+        uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        with:
+          path: .github/pr-review-memory.db
+          key: pr-review-memory-${{ github.repository }}-${{ github.run_id }}
+          restore-keys: |
+            pr-review-memory-${{ github.repository }}-
+
+      - name: Run Documentation PR Review
+        if: steps.membership.outputs.is_member == 'true'
+        uses: docker/cagent-action@latest
+        with:
+          agent: .github/pr-reviewer.yml
+          prompt: |
+            Review PR #${{ github.event.issue.number }}
+
+            Triggered manually by @${{ github.event.comment.user.login }}
+
+            Execute the review pipeline as documented in your instructions.
+          anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+      - name: Save reviewer memory
+        if: steps.membership.outputs.is_member == 'true' && always()
+        uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        with:
+          path: .github/pr-review-memory.db
+          key: pr-review-memory-${{ github.repository }}-${{ github.run_id }}
+
+      - name: Clean up old memory caches
+        if: steps.membership.outputs.is_member == 'true' && always()
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          CACHE_PREFIX="pr-review-memory-${{ github.repository }}-"
+
+          echo "🧹 Cleaning up old memory caches (keeping 5 most recent)"
+
+          OLD_CACHES=$(gh api "repos/${{ github.repository }}/actions/caches" \
+            --jq "[.actions_caches | map(select(.key | startswith(\"$CACHE_PREFIX\"))) | sort_by(.created_at) | reverse | .[5:] | .[].id] | .[]" \
+            2>/dev/null || echo "")
+
+          if [ -z "$OLD_CACHES" ]; then
+            echo "✅ No old caches to clean up"
+            exit 0
+          fi
+
+          DELETED=0
+          for CACHE_ID in $OLD_CACHES; do
+            if gh api "repos/${{ github.repository }}/actions/caches/$CACHE_ID" -X DELETE 2>/dev/null; then
+              ((DELETED++))
+            fi
+          done
+
+          echo "✅ Deleted $DELETED old cache(s)"
+
+  # ==========================================================================
+  # LEARN FROM FEEDBACK
+  # Processes replies to agent review comments for continuous improvement
+  # ==========================================================================
+  learn-from-feedback:
+    if: github.event_name == 'pull_request_review_comment' && github.event.comment.in_reply_to_id
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+
+      - name: Learn from user feedback
+        uses: docker/cagent-action/review-pr/learn@latest
+        with:
+          anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}