commit 05f8fad38acdaf3f272a15436e8c54b9691112a5 Author: layue13 Date: Mon Oct 26 20:40:51 2026 +0000 Dogfood: run Gatekeep on this repo's PRs Add .github/workflows/gatekeep-pr.yml with uses: ./ and OPENROUTER_API_KEY from secrets. Forge-agnostic API helper for Gitea. diff --git a/.github/workflows/gatekeep-eval.yml b/.github/workflows/gatekeep-eval.yml new file mode 100644 index 0000000..f4e836b --- /dev/null +++ b/.github/workflows/gatekeep-eval.yml @@ -0,0 +1,30 @@ +# Floor evals on the Gatekeep Action repo itself. + +name: Gatekeep evals + +on: + push: + paths: + - "defaults/**" + - "config/**" + - "evals/**" + - "scripts/**" + - "action.yml" + pull_request: + paths: + - "defaults/**" + - "config/**" + - "evals/**" + - "scripts/**" + - "action.yml" + workflow_dispatch: + +jobs: + eval-floor: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + - name: Run floor evals + run: bash scripts/eval-floor.sh evals/cases defaults/policy.json diff --git a/.github/workflows/gatekeep-pr.yml b/.github/workflows/gatekeep-pr.yml new file mode 100644 index 0000000..d94708c --- /dev/null +++ b/.github/workflows/gatekeep-pr.yml @@ -0,0 +1,45 @@ +# Dogfood: run Gatekeep on PRs to THIS repo (layue13/gatekeep). +# Gitea Actions discovers workflows under .github/workflows/ (not /workflows/). +# +# Secret: OPENROUTER_API_KEY (already configured on this repo) +# Action: in-repo composite at repository root (uses: ./) + +name: Gatekeep + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + +jobs: + gatekeep: + name: gatekeep + runs-on: ubuntu-latest + # Skip drafts on pull_request; allow manual dispatch + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false + steps: + - name: Checkout PR / default branch + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + + - name: Gatekeep review (in-repo Action) + id: gk + # Local composite action — action.yml at repo root + uses: ./ + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-verify: "true" + retain-artifacts: "true" + + - name: Job summary + if: always() + run: | + { + echo "## Gatekeep dogfood" + echo "risk_tier=${{ steps.gk.outputs.risk_tier }}" + echo "security_blocking=${{ steps.gk.outputs.security_blocking }}" + echo "agreed_count=${{ steps.gk.outputs.agreed_count }}" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..13cc780 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +.gatekeep/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..40e4d48 --- /dev/null +++ b/README.md @@ -0,0 +1,77 @@ +# Gatekeep + +AI-assisted PR review for **your own repos**: parallel multi-model review across **providers**, fail-closed PR risk tiers, agreed-only inline findings, blocking security check. + +**中文说明:** [README.zh-CN.md](./README.zh-CN.md) + +## Install (remote `uses:`) + +```yaml +# .github/workflows/gatekeep.yml ← must live under .github/workflows/ on Gitea/GitHub +- uses: actions/checkout@v4 +- uses: https://git.potatolab.uk/layue13/gatekeep@main + # placeholder: uses: https://git.example.com/OWNER/gatekeep@v1 + env: + # Names must match providers.yaml `key` — FREE-FORM (no required gatekeep_ prefix) + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} +``` + +### Dogfood (this repo) + +`layue13/gatekeep` reviews **its own** PRs via [`.github/workflows/gatekeep-pr.yml`](.github/workflows/gatekeep-pr.yml) with `uses: ./` and `secrets.OPENROUTER_API_KEY`. Open a PR against `main` to see a review (Gitea Actions must have a runner — same pool as other `layue13` repos). + +Allowlists / Gitea-on-github.com limitations: [`docs/remote-uses.md`](docs/remote-uses.md). + +## Providers YAML + +```yaml +providers: + openrouter: # only include providers you configure + key: OPENROUTER_API_KEY # free-form secret/env name — pick any + protocol: openai # openai | openai-completions | openai-responses | anthropic + # base_url: optional (defaults exist for openrouter / anthropic / openai) + models: + - google/gemma-4-31b-it:free + - google/gemma-4-26b-a4b-it:free + - cohere/north-mini-code:free + - nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free +``` + +| Field | Meaning | +| --- | --- | +| `key` | **Free-form** name of the GitHub secret / env var holding the API key. **No** required `gatekeep_` prefix. | +| `protocol` | `openai` \| `openai-completions` \| `openai-responses` \| `anthropic` | +| `models` | Names to use (validated against the provider’s model list API at runtime) | +| `base_url` | Optional; known provider ids get defaults | + +Mix providers in one review by listing several entries. Secrets are read from the **workflow step `env`** using each entry’s `key` name. + +Example suggestion only (still free-form): `key: gatekeep_openrouter` is fine if you named your secret that — not required. + +## Finding format + +**Severity** (Critical→Low) + **where** (`path:line`) + **why it matters**. Inline only when ≥2 models agree. + +## PR risk tiers + +| Tier | Humans | Auto-merge | +| --- | --- | --- | +| low | 0 after Gatekeep + security green | If kill switch on | +| medium | 1 | No | +| high | 2 | No | + +## Optional consumer config + +`config-dir: .github/gatekeep` with your `providers.yaml`. Omit to use Action `defaults/providers.yaml`. + +## Layout + +``` +action.yml +scripts/ # resolve-providers.py, call-model.sh, … +defaults/providers.yaml +config/providers.yaml # example override +workflows/ templates/ evals/ incidents/ docs/ +``` diff --git a/README.zh-CN.md b/README.zh-CN.md new file mode 100644 index 0000000..a682d20 --- /dev/null +++ b/README.zh-CN.md @@ -0,0 +1,85 @@ +# Gatekeep + +AI 辅助的 PR 审查 Action:多模型并行投票、按严重级别输出可扫读的结论,适合自己的仓库。 + +## 它做什么 + +- 对每个 PR 并行调用 **2–3 个模型**(**可来自不同供应商**),**仅当多个模型达成一致**时发 inline 评论。 +- 每条结论明确写出:**严重级别**、**代码位置**、**为什么重要**。 +- 汇总评论给出 PR **风险档**(low / medium / high);安全相关可阻断合并。 +- 不提供 CodeRabbit 式聊天 / `@命令`。 + +## 远程引用(推荐) + +工作流文件必须放在 **`.github/workflows/`**(Gitea / GitHub 都认这个路径)。 + +```yaml +- uses: actions/checkout@v4 +- uses: https://git.potatolab.uk/layue13/gatekeep@main + # 占位:uses: https://git.example.com/OWNER/gatekeep@v1 + env: + # 必须与 providers.yaml 里的 key 同名(自由命名,无强制 gatekeep_ 前缀) + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} +``` + +### 自审(本仓库 dogfood) + +`layue13/gatekeep` 用 [`.github/workflows/gatekeep-pr.yml`](.github/workflows/gatekeep-pr.yml) 对本仓库 PR 跑 Gatekeep:`uses: ./` + `secrets.OPENROUTER_API_KEY`。向 `main` 开 PR 即可触发(需有可用的 Gitea Actions runner)。 + +github.com runner 若不允许非 github.com Action 源,见 `docs/remote-uses.md`。 + +## 多供应商(providers.yaml) + +```yaml +providers: + openrouter: + key: OPENROUTER_API_KEY # 自由命名 — 任意 secret/环境变量名 + protocol: openai # openai | openai-completions | openai-responses | anthropic + models: + - google/gemma-4-31b-it:free + - google/gemma-4-26b-a4b-it:free + - cohere/north-mini-code:free + - nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free +``` + +- **`key`:自由命名**,**不要**要求必须 `gatekeep_` 前缀(你也可以叫 `gatekeep_openrouter`,只是举例)。 +- 每个供应商只需:`key` + `protocol`(`base_url` / `models` 按需;已知 id 有默认 base_url)。 +- **不内置死模型清单**;启动时拉取供应商模型列表并校验名字。 +- 可混用多个供应商;只写你真正配置了的条目。 + +## 输入面 + +| 项 | 含义 | +| --- | --- | +| `key` | Secret / 环境变量名(自由) | +| `protocol` | `openai` / `openai-completions` / `openai-responses` / `anthropic` | +| `models` | 参与并行审查的模型名 | +| step `env` | 把 `key` 同名的 secret 注入环境 | +| `github-token` | 发评论 / checks | + +## 混用示例 + +```yaml +- uses: https://git.potatolab.uk/layue13/gatekeep@main + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + config-dir: .github/gatekeep # 内含 providers.yaml,见 config/providers.yaml +``` + +## 风险档 + +| 档 | 人工 | +| --- | --- | +| low | 0(审查与安全绿后;kill switch 可自动合并) | +| medium | 1 人(Bot 不能顶替) | +| high | 2 人 | + +## 更多 + +- 英文:`README.md` +- 远程限制:`docs/remote-uses.md` diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..250a8c8 --- /dev/null +++ b/action.yml @@ -0,0 +1,126 @@ +name: Gatekeep +description: > + Parallel multi-model AI PR review across providers. Agreed findings become + inline comments; disagreements stay in the summary. Fail-closed risk tier. + Self-contained remote uses: git-url@ref. Provider secrets via free-form env names. +author: Gatekeep + +inputs: + github-token: + description: Token with pull-requests + checks write + required: true + config-dir: + description: > + Optional consumer config directory (e.g. .github/gatekeep with providers.yaml). + Empty = Action defaults/providers.yaml. + required: false + default: "" + models: + description: > + Optional override of 2–3 models as comma-separated provider:model + (or bare names unique in config). Empty = providers.yaml lists. + required: false + default: "" + skip-draft: + description: Skip draft PRs + required: false + default: "true" + run-verify: + description: Discover + run verify/benchmarks layer + required: false + default: "true" + retain-artifacts: + description: Upload last-good style artifacts from verify + required: false + default: "true" + +outputs: + risk_tier: + description: Final published risk tier + value: ${{ steps.review.outputs.risk_tier }} + security_blocking: + description: Whether security check should block + value: ${{ steps.review.outputs.security_blocking }} + agreed_count: + description: Count of multi-model agreed findings + value: ${{ steps.review.outputs.agreed_count }} + +runs: + using: composite + steps: + - name: Setup Node (stdlib only) + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install PyYAML (providers.yaml) + shell: bash + run: | + python3 -m pip install --user pyyaml -q + python3 -c "import yaml; print('pyyaml ok', yaml.__version__)" + + - name: Discover entry points + id: discover + shell: bash + env: + GATEKEEP_ROOT: ${{ github.workspace }} + GATEKEEP_ACTION: ${{ github.action_path }} + CONFIG_DIR: ${{ inputs.config-dir }} + run: bash "${{ github.action_path }}/scripts/discover.sh" + + - name: Policy floor + security scan + id: floor + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.github-token }} + CONFIG_DIR: ${{ inputs.config-dir }} + GATEKEEP_ROOT: ${{ github.workspace }} + GATEKEEP_ACTION: ${{ github.action_path }} + run: bash "${{ github.action_path }}/scripts/policy-floor.sh" + + - name: Parallel multi-model review + id: review + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.github-token }} + CONFIG_DIR: ${{ inputs.config-dir }} + GATEKEEP_MODELS: ${{ inputs.models }} + SKIP_DRAFT: ${{ inputs.skip-draft }} + GATEKEEP_ROOT: ${{ github.workspace }} + GATEKEEP_ACTION: ${{ github.action_path }} + POLICY_FLOOR: ${{ steps.floor.outputs.floor }} + POLICY_REASONS: ${{ steps.floor.outputs.reasons }} + SCAN_HARD_FAIL: ${{ steps.floor.outputs.scan_hard_fail }} + DISCOVER_JSON: ${{ steps.discover.outputs.discover_json }} + # Provider API keys: set free-form names from providers.yaml `key` + # on this step's env (inherited from the workflow step). Example: + # env: + # OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # MY_CUSTOM_KEY: ${{ secrets.MY_CUSTOM_KEY }} + run: | + export PATH="$HOME/.local/bin:$PATH" + # Ensure user-site packages (PyYAML) are importable + export PYTHONPATH="$(python3 -c 'import site; print(site.getusersitepackages())'):${PYTHONPATH:-}" + bash "${{ github.action_path }}/scripts/review.sh" + + - name: Verify layer (benchmarks) + if: inputs.run-verify == 'true' + id: verify + shell: bash + env: + GATEKEEP_ROOT: ${{ github.workspace }} + CONFIG_DIR: ${{ inputs.config-dir }} + GATEKEEP_ACTION: ${{ github.action_path }} + DISCOVER_JSON: ${{ steps.discover.outputs.discover_json }} + run: bash "${{ github.action_path }}/scripts/verify-benchmarks.sh" + + - name: Retain artifacts + if: inputs.retain-artifacts == 'true' && always() + uses: actions/upload-artifact@v4 + with: + name: gatekeep-last-good-${{ github.run_id }} + path: | + ${{ github.workspace }}/.gatekeep/out/ + ${{ github.workspace }}/.gatekeep/verify/ + if-no-files-found: ignore + retention-days: 14 diff --git a/config/README.md b/config/README.md new file mode 100644 index 0000000..0420139 --- /dev/null +++ b/config/README.md @@ -0,0 +1,16 @@ +# Example consumer overrides + +Copy `providers.yaml` (and optionally `policy.json` / `benchmarks.json`) into e.g. `.github/gatekeep/`, then: + +```yaml +- uses: https://git.potatolab.uk/layue13/gatekeep@main + env: + # Must match each providers.*.key (free-form names) + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + config-dir: .github/gatekeep +``` + +`key` is **not** required to start with `gatekeep_`. diff --git a/config/benchmarks.json b/config/benchmarks.json new file mode 100644 index 0000000..cf28e03 --- /dev/null +++ b/config/benchmarks.json @@ -0,0 +1,15 @@ +{ + "enabled": true, + "thresholds": { + "startup_ms_max": 15000, + "memory_mb_max": 1024, + "fps_min": 30 + }, + "notes": "Caller-configured verify-layer thresholds. Agent discovers how to run benches from README/package.json/Makefile; architecture-level perf debt is out of scope. Set a metric to null to skip it.", + "commands": { + "hint": "Optional explicit overrides. Empty = discover only.", + "benchmark": "", + "startup": "", + "memory": "" + } +} diff --git a/config/kill-switch.env.example b/config/kill-switch.env.example new file mode 100644 index 0000000..a15bbe1 --- /dev/null +++ b/config/kill-switch.env.example @@ -0,0 +1,9 @@ +# Repo Actions variable (Settings → Secrets and variables → Actions → Variables) +# or environment variable consumed by workflows/gatekeep-pr.yml auto-merge latch. + +GATEKEEP_MERGE_ENABLED=false +# Alias accepted by the workflow: +# AI_MERGE_ENABLED=false + +# When false (default recommended until burn-in): even risk:low PRs are not +# auto-merged by Gatekeep. Humans may still merge per tier policy. diff --git a/config/models.json b/config/models.json new file mode 100644 index 0000000..6061337 --- /dev/null +++ b/config/models.json @@ -0,0 +1,12 @@ +{ + "models": [ + "meta-llama/llama-3.3-70b-instruct:free", + "google/gemma-3-27b-it:free", + "qwen/qwen3-235b-a22b:free" + ], + "notes": "2–3 OpenRouter model IDs. Prefer free-tier (:free) for owner repos; override via action input models= or edit this file. Paid models work the same (drop :free).", + "openrouter_base_url": "https://openrouter.ai/api/v1", + "temperature": 0, + "max_tokens": 4096, + "agreement_min": 2 +} diff --git a/config/policy.json b/config/policy.json new file mode 100644 index 0000000..f37a98a --- /dev/null +++ b/config/policy.json @@ -0,0 +1,30 @@ +{ + "medium_path_globs": [ + "**/package-lock.json", + "**/pnpm-lock.yaml", + "**/yarn.lock", + "src/**", + "lib/**", + "server/**", + "packages/**" + ], + "high_path_globs": [ + ".github/workflows/**", + ".github/actions/**", + "**/Dockerfile*", + "**/*secret*", + "**/*credential*", + "**/auth/**", + "**/billing/**", + "**/payment/**", + "**/.env*", + "**/security/**", + "scripts/release/**" + ], + "low_max_files": 8, + "low_max_diff_lines": 200, + "force_high_labels": ["security", "do-not-automerge", "high-risk", "gatekeep:high"], + "force_medium_labels": ["needs-human", "medium-risk", "gatekeep:medium"], + "do_not_automerge_labels": ["do-not-automerge", "gatekeep:no-automerge"], + "security_agreement": "require_agreement_unless_scan_hard_fail" +} diff --git a/config/providers.json b/config/providers.json new file mode 100644 index 0000000..7cd50af --- /dev/null +++ b/config/providers.json @@ -0,0 +1,16 @@ +{ + "_deprecated": "Prefer providers.yaml in this folder.", + "agreement_min": 2, + "providers": { + "openrouter": { + "key": "OPENROUTER_API_KEY", + "protocol": "openai", + "models": [ + "google/gemma-4-31b-it:free", + "google/gemma-4-26b-a4b-it:free", + "cohere/north-mini-code:free", + "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free" + ] + } + } +} diff --git a/config/providers.yaml b/config/providers.yaml new file mode 100644 index 0000000..6ee7af3 --- /dev/null +++ b/config/providers.yaml @@ -0,0 +1,16 @@ +# Example / owner consumer override — OpenRouter only. +# `key` is free-form; map it in the workflow step env from secrets. + +agreement_min: 2 +temperature: 0 +max_tokens: 4096 + +providers: + openrouter: + key: OPENROUTER_API_KEY + protocol: openai + models: + - google/gemma-4-31b-it:free + - google/gemma-4-26b-a4b-it:free + - cohere/north-mini-code:free + - nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free diff --git a/defaults/benchmarks.json b/defaults/benchmarks.json new file mode 100644 index 0000000..cf28e03 --- /dev/null +++ b/defaults/benchmarks.json @@ -0,0 +1,15 @@ +{ + "enabled": true, + "thresholds": { + "startup_ms_max": 15000, + "memory_mb_max": 1024, + "fps_min": 30 + }, + "notes": "Caller-configured verify-layer thresholds. Agent discovers how to run benches from README/package.json/Makefile; architecture-level perf debt is out of scope. Set a metric to null to skip it.", + "commands": { + "hint": "Optional explicit overrides. Empty = discover only.", + "benchmark": "", + "startup": "", + "memory": "" + } +} diff --git a/defaults/models.json b/defaults/models.json new file mode 100644 index 0000000..a1c82ca --- /dev/null +++ b/defaults/models.json @@ -0,0 +1,11 @@ +{ + "_deprecated": "Use providers.json. Kept only so old config-dir copies still resolve; resolve-providers.py migrates this shape.", + "models": [ + "meta-llama/llama-3.3-70b-instruct:free", + "google/gemma-3-27b-it:free" + ], + "openrouter_base_url": "https://openrouter.ai/api/v1", + "agreement_min": 2, + "temperature": 0, + "max_tokens": 4096 +} diff --git a/defaults/policy.json b/defaults/policy.json new file mode 100644 index 0000000..f37a98a --- /dev/null +++ b/defaults/policy.json @@ -0,0 +1,30 @@ +{ + "medium_path_globs": [ + "**/package-lock.json", + "**/pnpm-lock.yaml", + "**/yarn.lock", + "src/**", + "lib/**", + "server/**", + "packages/**" + ], + "high_path_globs": [ + ".github/workflows/**", + ".github/actions/**", + "**/Dockerfile*", + "**/*secret*", + "**/*credential*", + "**/auth/**", + "**/billing/**", + "**/payment/**", + "**/.env*", + "**/security/**", + "scripts/release/**" + ], + "low_max_files": 8, + "low_max_diff_lines": 200, + "force_high_labels": ["security", "do-not-automerge", "high-risk", "gatekeep:high"], + "force_medium_labels": ["needs-human", "medium-risk", "gatekeep:medium"], + "do_not_automerge_labels": ["do-not-automerge", "gatekeep:no-automerge"], + "security_agreement": "require_agreement_unless_scan_hard_fail" +} diff --git a/defaults/providers.json b/defaults/providers.json new file mode 100644 index 0000000..33c4619 --- /dev/null +++ b/defaults/providers.json @@ -0,0 +1,16 @@ +{ + "_deprecated": "Use providers.yaml.", + "agreement_min": 2, + "providers": { + "openrouter": { + "key": "OPENROUTER_API_KEY", + "protocol": "openai", + "models": [ + "google/gemma-4-31b-it:free", + "google/gemma-4-26b-a4b-it:free", + "cohere/north-mini-code:free", + "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free" + ] + } + } +} diff --git a/defaults/providers.yaml b/defaults/providers.yaml new file mode 100644 index 0000000..be39da5 --- /dev/null +++ b/defaults/providers.yaml @@ -0,0 +1,17 @@ +# Bundled defaults — only providers you actually use. +# `key` is a FREE-FORM secret / env var name (no required gatekeep_ prefix). +# Put that name in the workflow step env, e.g. env: { OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} } + +agreement_min: 2 +temperature: 0 +max_tokens: 4096 + +providers: + openrouter: + key: OPENROUTER_API_KEY + protocol: openai + models: + - google/gemma-4-31b-it:free + - google/gemma-4-26b-a4b-it:free + - cohere/north-mini-code:free + - nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free diff --git a/docs/deploy-slots.md b/docs/deploy-slots.md new file mode 100644 index 0000000..dc888a0 --- /dev/null +++ b/docs/deploy-slots.md @@ -0,0 +1,12 @@ +# Deploy slots (stub — later) + +Gatekeep’s PR Action does **not** perform production rollback. Ops keeps two slots: + +| Slot | Role | +| --- | --- | +| `current` / `active` | What players/users hit now | +| `previous` | Last known-good artifact for one-command swap | + +**v1 note only:** wire symlink or compose services outside this Action; record `previous_good_sha` on every deploy; PR template “How to revert” points operators here. Auto-rollback stays human-confirm by default except crash-loop burn-in (see Project rollback design). + +Artifact retention for review/verify jobs uses `actions/upload-artifact` (`gatekeep-last-good-`, 14 days) — that is **CI evidence**, not the production slot. diff --git a/docs/dogfood.md b/docs/dogfood.md new file mode 100644 index 0000000..721dc53 --- /dev/null +++ b/docs/dogfood.md @@ -0,0 +1,18 @@ +# Dogfood — Gatekeep reviews Gatekeep + +The Action repo itself runs Gatekeep on its own pull requests. + +| Piece | Path / value | +| --- | --- | +| Live workflow | `.github/workflows/gatekeep-pr.yml` | +| Action | `uses: ./` (repo-root `action.yml`) | +| Secret | `OPENROUTER_API_KEY` → step `env` | +| Trigger | `pull_request` (non-draft) + `workflow_dispatch` | + +## How to see a review + +1. Branch off `main`, make a tiny change, open a PR on https://git.potatolab.uk/layue13/gatekeep +2. Wait for the **Gatekeep** workflow (same Actions runner pool as other `layue13` repos) +3. Look for the summary comment `` on the PR + +Consumer sample workflows remain under top-level `workflows/` for copy-paste into *other* repos’ `.github/workflows/`. diff --git a/docs/remote-uses.md b/docs/remote-uses.md new file mode 100644 index 0000000..e10841f --- /dev/null +++ b/docs/remote-uses.md @@ -0,0 +1,68 @@ +# Remote `uses:` — limitations & workarounds + +Gatekeep is a **self-contained composite Action** (`action.yml` at repo root + `scripts/` + `defaults/`). Consumers should prefer: + +```yaml +uses: https://git.example.com/OWNER/gatekeep@v1 +# Project example: +uses: https://git.potatolab.uk/layue13/gatekeep@main +``` + +No copy of Action sources into the consumer repo is required. Optional `.github/gatekeep/` overrides still work via `config-dir`. + +## What must be true + +| Requirement | Notes | +| --- | --- | +| Runner can fetch the Action git repo | Network + credentials if the Action repo is private | +| Runner can reach OpenRouter | `OPENROUTER_API_KEY` secret | +| `GITHUB_TOKEN` permissions | `pull-requests: write`, `checks: write` (see sample workflow) | +| Org / repo Actions policy | External Actions allowed | + +## GitHub.com-hosted consumers + Gitea-hosted Action + +**Honest status:** GitHub Actions’ first-class `uses: owner/repo@ref` syntax is for **github.com**. Full URL `uses: https://…@ref` is documented for public Actions, but **enterprise / org allowlists** often restrict non-`github.com` Action sources. Behavior can also differ by plan and policy version. + +If `uses: https://git.potatolab.uk/layue13/gatekeep@main` fails on github.com runners with an allowlist / “not permitted” / fetch error: + +### Workaround A — GitHub mirror (recommended) + +1. Mirror `layue13/gatekeep` to a github.com repo you control (e.g. `your-org/gatekeep`). +2. Point consumers at: + ```yaml + uses: your-org/gatekeep@main + # or: uses: your-org/gatekeep@v1 + ``` +3. Keep the Gitea repo as source of truth; sync tags on release. + +### Workaround B — Vendor checkout (still no Marketplace) + +If remotes are fully blocked, checkout Gatekeep into the job and run the composite via local path: + +```yaml +- uses: actions/checkout@v4 + with: + repository: layue13/gatekeep # or mirror + # For Gitea, use a custom remote + token: + # (or actions/checkout with sparse / git mirror step) + path: .gatekeep-action + ref: main +- uses: ./.gatekeep-action + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + openrouter-api-key: ${{ secrets.OPENROUTER_API_KEY }} +``` + +For a pure Gitea URL when `actions/checkout` only speaks github.com, use an explicit `git clone` step with a deploy token, then `uses: ./.gatekeep-action`. + +### Workaround C — Copy-in (appendix) + +Copy Action files into `.github/actions/gatekeep` (see README appendix). Same runtime; more drift risk. + +## Gitea / Forgejo Actions runners + +If the **consumer** also runs on Gitea Actions (not GitHub-hosted), use that forge’s documented `uses:` / local action syntax. Paths remain `github.action_path`-relative so the same Action tree works once the runner can resolve it. + +## Secrets never travel with the Action + +Remote fetch does **not** include consumer secrets. Always set `OPENROUTER_API_KEY` (and any forge token for private Action clones) on the **consumer** repo/org. diff --git a/docs/security-checklist.md b/docs/security-checklist.md new file mode 100644 index 0000000..9e8420a --- /dev/null +++ b/docs/security-checklist.md @@ -0,0 +1,23 @@ +# Gatekeep security seed checklist + +Used in model prompts **and** the deterministic scanner (`action/scripts/security-scan.sh`). + +| Item | Deterministic scan | Model review | +| --- | --- | --- | +| Hardcoded secrets/tokens | Added-line regex | Yes | +| Prompt injection via repo files | Instruction-override phrases in diff | Yes | +| Auth bypass | Path globs (`**/auth/**`) floor high | Yes | +| Unauthorized API access | — | Yes | +| Billing logic bypass | Path globs (`**/billing/**`, `**/payment/**`) | Yes | +| CI workflow tampering | `.github/workflows/**` → high + scan hit | Yes | +| Unsafe deserialization | pickle/yaml.load/unserialize patterns | Yes | +| SQL/command injection | Concat / `shell=True` patterns | Yes | + +## Blocking policy (default) + +`policy.json` → `security_agreement`: **`require_agreement_unless_scan_hard_fail`** + +- **Inline + block:** security / **Critical** findings where **≥2 models agree**, or any **deterministic scan hard-fail**. +- **Summary only:** lone-model security flags (disputed) — do not inline; humans can still escalate. +- Scan hard-fail always floors PR tier to **high** and fails check `gatekeep / security`. +- Every finding (including security) still shows **severity + where + why it matters**. diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..7e2eaea --- /dev/null +++ b/evals/README.md @@ -0,0 +1,9 @@ +# Gatekeep floor evals + +Pins the **deterministic policy floor** (fail-closed). Multi-model LLM regression is optional later; MVP evals do not call OpenRouter. + +```bash +bash scripts/eval-floor.sh evals/cases defaults/policy.json +``` + +Add a case after every incident (`expect_min_tier` required). Seed set: ≥3 must-stay-low, ≥2 must-stay-high. diff --git a/evals/cases/_template.json b/evals/cases/_template.json new file mode 100644 index 0000000..dca7ac5 --- /dev/null +++ b/evals/cases/_template.json @@ -0,0 +1,17 @@ +{ + "id": "unique-kebab-id", + "description": "What this pins", + "expect_min_tier": "high", + "expect_tier": "high", + "incident_ref": null, + "fixture": { + "title": "PR title", + "body": "", + "labels": [], + "files": ["path/to/file"], + "additions": 10, + "deletions": 2, + "ci_status": "success", + "diff": "diff --git a/path/to/file b/path/to/file\n" + } +} diff --git a/evals/cases/must-stay-high-secrets.json b/evals/cases/must-stay-high-secrets.json new file mode 100644 index 0000000..b93b566 --- /dev/null +++ b/evals/cases/must-stay-high-secrets.json @@ -0,0 +1,14 @@ +{ + "id": "must-stay-high-secrets", + "description": "auth paths never low", + "expect_min_tier": "high", + "expect_tier": "high", + "fixture": { + "title": "chore: comment in auth module", + "files": ["src/auth/session.ts"], + "additions": 3, + "deletions": 0, + "ci_status": "success", + "diff": "diff --git a/src/auth/session.ts b/src/auth/session.ts\n" + } +} diff --git a/evals/cases/must-stay-high-workflow.json b/evals/cases/must-stay-high-workflow.json new file mode 100644 index 0000000..6d1abe4 --- /dev/null +++ b/evals/cases/must-stay-high-workflow.json @@ -0,0 +1,15 @@ +{ + "id": "must-stay-high-workflow", + "description": ".github/workflows always high", + "expect_min_tier": "high", + "expect_tier": "high", + "incident_ref": "INC-EXAMPLE-001", + "fixture": { + "title": "ci: speed up unit job", + "files": [".github/workflows/ci.yml"], + "additions": 5, + "deletions": 1, + "ci_status": "success", + "diff": "diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml\n" + } +} diff --git a/evals/cases/must-stay-low-docs.json b/evals/cases/must-stay-low-docs.json new file mode 100644 index 0000000..aa300b9 --- /dev/null +++ b/evals/cases/must-stay-low-docs.json @@ -0,0 +1,14 @@ +{ + "id": "must-stay-low-docs", + "description": "Docs-only stays low", + "expect_min_tier": "low", + "expect_tier": "low", + "fixture": { + "title": "docs: clarify setup", + "files": ["README.md", "docs/setup.md"], + "additions": 12, + "deletions": 4, + "ci_status": "success", + "diff": "diff --git a/README.md b/README.md\n" + } +} diff --git a/evals/cases/must-stay-low-test-helper.json b/evals/cases/must-stay-low-test-helper.json new file mode 100644 index 0000000..6d11370 --- /dev/null +++ b/evals/cases/must-stay-low-test-helper.json @@ -0,0 +1,14 @@ +{ + "id": "must-stay-low-test-helper", + "description": "Isolated test fixture stays low", + "expect_min_tier": "low", + "expect_tier": "low", + "fixture": { + "title": "test: rename fixture id", + "files": ["tests/fixtures/world-seed.json"], + "additions": 2, + "deletions": 2, + "ci_status": "success", + "diff": "diff --git a/tests/fixtures/world-seed.json b/tests/fixtures/world-seed.json\n" + } +} diff --git a/evals/cases/must-stay-low-typo.json b/evals/cases/must-stay-low-typo.json new file mode 100644 index 0000000..db5dfc7 --- /dev/null +++ b/evals/cases/must-stay-low-typo.json @@ -0,0 +1,14 @@ +{ + "id": "must-stay-low-typo", + "description": "Tiny non-sensitive typo stays low", + "expect_min_tier": "low", + "expect_tier": "low", + "fixture": { + "title": "fix: typo", + "files": ["tools/format-notes.txt"], + "additions": 1, + "deletions": 1, + "ci_status": "success", + "diff": "diff --git a/tools/format-notes.txt b/tools/format-notes.txt\n" + } +} diff --git a/incidents/README.md b/incidents/README.md new file mode 100644 index 0000000..6037ce4 --- /dev/null +++ b/incidents/README.md @@ -0,0 +1,7 @@ +# Incidents → evals + +1. Restore service (revert / slot swap). Flip `GATEKEEP_MERGE_ENABLED=false` if needed. +2. File **gatekeep-incident** issue (template in `templates/gatekeep-incident.yml`). +3. Add `incidents/seed/.json` + `evals/cases/.json` with `expect_min_tier`. +4. Tighten `defaults/policy.json` (or consumer override) / security checklist; never delete a failing case to go green. +5. `bash scripts/eval-floor.sh evals/cases defaults/policy.json` diff --git a/incidents/SCHEMA.md b/incidents/SCHEMA.md new file mode 100644 index 0000000..aaebcf9 --- /dev/null +++ b/incidents/SCHEMA.md @@ -0,0 +1,21 @@ +# Incident record schema + +```json +{ + "id": "INC-2026-001", + "date": "2026-10-26", + "severity": "low|medium|high|critical", + "summary": "One-line what went wrong", + "pr_url": "https://github.com/org/repo/pull/123", + "merge_sha": "abc123", + "ai_tier_at_merge": "low|medium|high|null", + "should_have_been_tier": "low|medium|high", + "root_cause": "classifier_under_tier|missed_agreed_finding|agreement_split|missing_oracle|human_skip|prompt_miss|other", + "models_at_review": ["model-a", "model-b"], + "revert_pr_url": "https://github.com/org/repo/pull/124", + "revert_minutes": 15, + "eval_case_id": "must-stay-high-from-inc-2026-001", + "policy_change": "What glob/label/checklist changed", + "notes": "Freeform" +} +``` diff --git a/incidents/seed/example-incident.json b/incidents/seed/example-incident.json new file mode 100644 index 0000000..a422f9b --- /dev/null +++ b/incidents/seed/example-incident.json @@ -0,0 +1,20 @@ +{ + "id": "INC-EXAMPLE-001", + "date": "2026-10-01", + "severity": "high", + "summary": "Workflow change under-tiered; unpinned action merged.", + "pr_url": "https://github.com/example/repo/pull/0", + "merge_sha": "0000000000000000000000000000000000000000", + "ai_tier_at_merge": "low", + "should_have_been_tier": "high", + "root_cause": "classifier_under_tier", + "models_at_review": [ + "meta-llama/llama-3.3-70b-instruct:free", + "google/gemma-3-27b-it:free" + ], + "revert_pr_url": "https://github.com/example/repo/pull/1", + "revert_minutes": 12, + "eval_case_id": "must-stay-high-workflow", + "policy_change": "Keep .github/workflows/** in high_path_globs (seed policy).", + "notes": "Seed example only." +} diff --git a/scripts/call-model.sh b/scripts/call-model.sh new file mode 100644 index 0000000..275bde7 --- /dev/null +++ b/scripts/call-model.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Call one model via provider protocol. +# Protocols: openai | openai-completions | openai-responses | anthropic +# (openai / openai-compatible → chat completions) +# Args: +set -euo pipefail + +PROVIDER="${1:?provider}" +MODEL="${2:?model}" +PROTOCOL="${3:?protocol}" +BASE_URL="${4:?base_url}" +PROMPT_FILE="${5:?prompt}" +OUT_FILE="${6:?out}" +SECRETS_FILE="${7:?secrets}" + +KEY=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get(sys.argv[2],""))' "$SECRETS_FILE" "$PROVIDER") +if [[ -z "$KEY" ]]; then + echo "missing key for $PROVIDER" >&2 + echo "{\"error\":\"missing_key\",\"model\":\"$MODEL\",\"provider\":\"$PROVIDER\"}" > "$OUT_FILE" + exit 0 +fi + +MAX_TOKENS="${GATEKEEP_MAX_TOKENS:-4096}" +SYSTEM=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["system"])' "$PROMPT_FILE") +USER=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["user"])' "$PROMPT_FILE") + +LABEL="${PROVIDER}:${MODEL}" + +# Normalize protocol aliases +WIRE=$(python3 -c 'import sys +p=sys.argv[1].lower() +m={"openai":"openai-completions","openai-compatible":"openai-completions", + "openai-completions":"openai-completions","openai-responses":"openai-responses", + "anthropic":"anthropic"} +print(m.get(p,""))' "$PROTOCOL") + +if [[ -z "$WIRE" ]]; then + echo "{\"error\":\"bad_protocol\",\"protocol\":\"$PROTOCOL\",\"model\":\"$MODEL\"}" > "$OUT_FILE" + exit 0 +fi + +if [[ "$WIRE" == "openai-completions" ]]; then + BODY=$(python3 - "$MODEL" "$SYSTEM" "$USER" "$MAX_TOKENS" <<'PY' +import json,sys +model, system, user, max_tokens = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) +print(json.dumps({ + "model": model, + "temperature": 0, + "max_tokens": max_tokens, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], +})) +PY +) + HTTP_CODE=$(curl -sS -o "$OUT_FILE.raw" -w "%{http_code}" \ + "${BASE_URL%/}/chat/completions" \ + -H "Authorization: Bearer $KEY" \ + -H "Content-Type: application/json" \ + -H "HTTP-Referer: https://github.com/${GITHUB_REPOSITORY:-gatekeep}" \ + -H "X-Title: Gatekeep" \ + -d "$BODY") + EXTRACT=openai +elif [[ "$WIRE" == "openai-responses" ]]; then + BODY=$(python3 - "$MODEL" "$SYSTEM" "$USER" "$MAX_TOKENS" <<'PY' +import json,sys +model, system, user, max_tokens = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) +print(json.dumps({ + "model": model, + "temperature": 0, + "max_output_tokens": max_tokens, + "input": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], +})) +PY +) + HTTP_CODE=$(curl -sS -o "$OUT_FILE.raw" -w "%{http_code}" \ + "${BASE_URL%/}/responses" \ + -H "Authorization: Bearer $KEY" \ + -H "Content-Type: application/json" \ + -d "$BODY") + EXTRACT=openai-responses +elif [[ "$WIRE" == "anthropic" ]]; then + BODY=$(python3 - "$MODEL" "$SYSTEM" "$USER" "$MAX_TOKENS" <<'PY' +import json,sys +model, system, user, max_tokens = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) +print(json.dumps({ + "model": model, + "max_tokens": max_tokens, + "temperature": 0, + "system": system, + "messages": [{"role": "user", "content": user}], +})) +PY +) + HTTP_CODE=$(curl -sS -o "$OUT_FILE.raw" -w "%{http_code}" \ + "${BASE_URL%/}/v1/messages" \ + -H "x-api-key: $KEY" \ + -H "anthropic-version: 2023-06-01" \ + -H "Content-Type: application/json" \ + -d "$BODY") + EXTRACT=anthropic +fi + +if [[ "$HTTP_CODE" != "200" ]]; then + echo "$LABEL failed HTTP $HTTP_CODE" >&2 + head -c 800 "$OUT_FILE.raw" >&2 || true + echo "{\"error\":\"http\",\"model\":\"$MODEL\",\"provider\":\"$PROVIDER\",\"http\":$HTTP_CODE}" > "$OUT_FILE" + exit 0 +fi + +python3 - "$OUT_FILE.raw" "$OUT_FILE" "$MODEL" "$PROVIDER" "$EXTRACT" <<'PY' +import json,sys +raw=json.load(open(sys.argv[1])) +out, model, provider, kind = sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] +if kind == "anthropic": + text = "".join( + (c.get("text") or "") + for c in (raw.get("content") or []) + if c.get("type") == "text" + ) +elif kind == "openai-responses": + # Best-effort extract from Responses API + text = raw.get("output_text") or "" + if not text: + chunks = [] + for item in raw.get("output") or []: + for c in item.get("content") or []: + if c.get("type") in ("output_text", "text"): + chunks.append(c.get("text") or "") + text = "".join(chunks) +else: + text = (((raw.get("choices") or [{}])[0].get("message") or {}).get("content")) or "" +start, end = text.find("{"), text.rfind("}") +payload = {} +if start >= 0 and end > start: + try: + payload = json.loads(text[start:end+1]) + except Exception as e: + payload = {"parse_error": str(e), "raw_text": text[:2000]} +else: + payload = {"parse_error": "no_json", "raw_text": text[:2000]} +payload["_model"] = f"{provider}:{model}" +payload["_provider"] = provider +json.dump(payload, open(out,"w"), indent=2) +PY + +echo "$LABEL → $OUT_FILE" >&2 diff --git a/scripts/discover.sh b/scripts/discover.sh new file mode 100644 index 0000000..2640024 --- /dev/null +++ b/scripts/discover.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Discover build/test/benchmark entry points from common manifests. +# No Minecraft-hardcoded commands — agent/caller repos vary. +set -euo pipefail + +ROOT="${GATEKEEP_ROOT:-.}" +OUT_DIR="$ROOT/.gatekeep/out" +mkdir -p "$OUT_DIR" + +json_escape() { + python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))' <<<"$1" +} + +entries=() + +add() { + local kind="$1" cmd="$2" src="$3" + entries+=("{\"kind\":$(json_escape "$kind"),\"command\":$(json_escape "$cmd"),\"source\":$(json_escape "$src")}") +} + +if [[ -f "$ROOT/package.json" ]]; then + # Prefer known script names without requiring jq + for s in test check lint build verify bench benchmark start; do + if python3 - "$ROOT/package.json" "$s" <<'PY' +import json,sys +p=json.load(open(sys.argv[1])) +scripts=p.get("scripts") or {} +sys.exit(0 if sys.argv[2] in scripts else 1) +PY + then + add "npm:$s" "npm run $s" "package.json" + fi + done +fi + +if [[ -f "$ROOT/Makefile" ]]; then + for t in test check verify bench benchmark build; do + if grep -qE "^${t}[[:space:]]*:" "$ROOT/Makefile"; then + add "make:$t" "make $t" "Makefile" + fi + done +fi + +if [[ -f "$ROOT/gradlew" ]]; then + add "gradle:test" "./gradlew test" "gradlew" + add "gradle:build" "./gradlew build" "gradlew" +fi + +if [[ -f "$ROOT/pom.xml" ]]; then + add "mvn:test" "mvn -B test" "pom.xml" +fi + +if [[ -f "$ROOT/Cargo.toml" ]]; then + add "cargo:test" "cargo test" "Cargo.toml" + add "cargo:bench" "cargo bench" "Cargo.toml" +fi + +if [[ -f "$ROOT/go.mod" ]]; then + add "go:test" "go test ./..." "go.mod" +fi + +if [[ -f "$ROOT/pyproject.toml" ]] || [[ -f "$ROOT/pytest.ini" ]]; then + add "pytest" "pytest -q" "python" +fi + +# README hints (non-executing notes for the model/verify layer) +readme_hints="" +for f in README.md README.rst docs/README.md; do + if [[ -f "$ROOT/$f" ]]; then + readme_hints+=$(grep -iE '^(#+ ).*(test|build|bench|verify|ci)' "$ROOT/$f" | head -20 || true) + readme_hints+=$'\n' + fi +done + +joined=$(IFS=,; echo "${entries[*]}") +discover_json="{\"entries\":[${joined}],\"readme_hints\":$(json_escape "$readme_hints")}" +printf '%s\n' "$discover_json" > "$OUT_DIR/discover.json" + +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + { + echo "discover_json<> "$GITHUB_OUTPUT" +fi + +echo "Discovered ${#entries[@]} entry point(s)" >&2 diff --git a/scripts/eval-floor.sh b/scripts/eval-floor.sh new file mode 100644 index 0000000..4c923a1 --- /dev/null +++ b/scripts/eval-floor.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Offline eval of deterministic policy floor (fail-closed). No OpenRouter calls. +set -euo pipefail + +CASES_DIR="${1:?evals/cases dir}" +POLICY="${2:-}" +ACTION_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +export GATEKEEP_ACTION="$ACTION_ROOT" +export GATEKEEP_ROOT="${GATEKEEP_ROOT:-$ACTION_ROOT}" +export CONFIG_DIR="${CONFIG_DIR:-}" +# shellcheck source=lib-config.sh +source "$ACTION_ROOT/scripts/lib-config.sh" +POLICY="${POLICY:-$(gatekeep_config_file policy.json)}" + +failed=0 +total=0 + +for case in "$CASES_DIR"/*.json; do + [[ -f "$case" ]] || continue + [[ "$(basename "$case")" == _template.json ]] && continue + total=$((total + 1)) + python3 - "$case" "$POLICY" <<'PY' || failed=$((failed + 1)) +import json, fnmatch, sys + +case = json.load(open(sys.argv[1])) +policy = json.load(open(sys.argv[2])) +fx = case["fixture"] +files = fx.get("files") or [] +labels = [x.lower() for x in (fx.get("labels") or [])] +add = int(fx.get("additions") or 10) +delete = int(fx.get("deletions") or 2) +ci = fx.get("ci_status") or "success" + +rank = {"low": 0, "medium": 1, "high": 2} +floor = "low" +reasons = [] + +def bump(t, why): + global floor + if rank[t] > rank[floor]: + floor = t + reasons.append(why) + +def match(path, glob): + g = glob.replace("**/", "*").replace("**", "*") + return fnmatch.fnmatch(path, glob) or fnmatch.fnmatch(path, g) + +for lab in labels: + if lab in [x.lower() for x in policy.get("force_high_labels", [])]: + bump("high", f"label {lab}") + if lab in [x.lower() for x in policy.get("force_medium_labels", [])]: + bump("medium", f"label {lab}") + +for f in files: + for g in policy.get("high_path_globs", []): + if match(f, g): + bump("high", f"{f}~{g}") + for g in policy.get("medium_path_globs", []): + if match(f, g): + bump("medium", f"{f}~{g}") + +if len(files) > int(policy.get("low_max_files", 8)): + bump("medium", "file count") +if add + delete > int(policy.get("low_max_diff_lines", 200)): + bump("medium", "diff lines") +if ci in ("failure", "error"): + bump("high", f"ci {ci}") + +min_t = case["expect_min_tier"] +exact = case.get("expect_tier") +ok = rank[floor] >= rank[min_t] and (exact is None or floor == exact) +status = "PASS" if ok else "FAIL" +print(f"{status} {case['id']}: floor={floor} min={min_t}" + (f" expect={exact}" if exact else "")) +if not ok: + print(" " + "; ".join(reasons) or "(none)") + print(" " + case.get("description", "")) + sys.exit(1) +PY +done + +echo "" +echo "$((total - failed))/$total passed" +exit "$failed" diff --git a/scripts/forge_api.py b/scripts/forge_api.py new file mode 100644 index 0000000..390253a --- /dev/null +++ b/scripts/forge_api.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +Minimal GitHub / Gitea API helper for Gatekeep (no gh CLI required). +Uses GITHUB_API_URL + GITHUB_TOKEN (Gitea Actions sets these to the forge). +""" +from __future__ import annotations + +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + + +def api_base() -> str: + base = (os.environ.get("GITHUB_API_URL") or "https://api.github.com").rstrip("/") + return base + + +def is_gitea() -> bool: + b = api_base().lower() + return "api.github.com" not in b + + +def token() -> str: + return ( + os.environ.get("GITHUB_TOKEN") + or os.environ.get("GITEA_TOKEN") + or os.environ.get("GH_TOKEN") + or "" + ) + + +def request(method: str, path: str, body: dict | None = None, accept: str | None = None) -> tuple[int, str]: + path = path if path.startswith("/") else f"/{path}" + url = f"{api_base()}{path}" + data = None if body is None else json.dumps(body).encode("utf-8") + headers = { + "Authorization": f"token {token()}" if is_gitea() else f"Bearer {token()}", + "Accept": accept or ("application/json" if is_gitea() else "application/vnd.github+json"), + "User-Agent": "gatekeep-action", + } + if body is not None: + headers["Content-Type"] = "application/json" + # GitHub also accepts token prefix + if not is_gitea() and token(): + headers["Authorization"] = f"Bearer {token()}" + req = urllib.request.Request(url, data=data, headers=headers, method=method.upper()) + try: + with urllib.request.urlopen(req, timeout=120) as resp: + return resp.status, resp.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as e: + return e.code, e.read().decode("utf-8", errors="replace") + + +def request_json(method: str, path: str, body: dict | None = None): + code, text = request(method, path, body) + if code >= 400: + raise RuntimeError(f"{method} {path} → HTTP {code}: {text[:500]}") + if not text.strip(): + return None + return json.loads(text) + + +def main(argv: list[str]) -> int: + """CLI: forge_api.py METHOD PATH [json-body] → prints response body""" + if len(argv) < 3: + print("usage: forge_api.py METHOD /path [json-body]", file=sys.stderr) + return 2 + method, path = argv[1], argv[2] + body = json.loads(argv[3]) if len(argv) > 3 else None + code, text = request(method, path, body) + sys.stdout.write(text) + return 0 if code < 400 else 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/scripts/lib-config.sh b/scripts/lib-config.sh new file mode 100644 index 0000000..c45783b --- /dev/null +++ b/scripts/lib-config.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Resolve Gatekeep config files for remote-referenceable Action installs. +# Prefer optional consumer override dir; else bundled defaults under action_path. +# +# Expects: GATEKEEP_ROOT, GATEKEEP_ACTION, CONFIG_DIR (may be empty) +# Usage: source "$GATEKEEP_ACTION/scripts/lib-config.sh" +# cfg=$(gatekeep_config_file models.json) + +gatekeep_config_file() { + local name="$1" + local root="${GATEKEEP_ROOT:-.}" + local action="${GATEKEEP_ACTION:-.}" + local cdir="${CONFIG_DIR:-}" + + # Optional consumer override (only if config-dir set and file exists) + if [[ -n "$cdir" && -f "$root/$cdir/$name" ]]; then + printf '%s\n' "$root/$cdir/$name" + return 0 + fi + + # Bundled defaults shipped inside the Action (remote uses: / copy-in) + if [[ -f "$action/defaults/$name" ]]; then + printf '%s\n' "$action/defaults/$name" + return 0 + fi + + # Legacy pack layouts (dev / old copy-in) + if [[ -f "$action/../config/$name" ]]; then + printf '%s\n' "$action/../config/$name" + return 0 + fi + if [[ -f "$action/../defaults/$name" ]]; then + printf '%s\n' "$action/../defaults/$name" + return 0 + fi + + echo "gatekeep: missing config $name (no consumer override, no Action defaults)" >&2 + return 1 +} + +# Prefer providers.yaml, then providers.yml, then legacy providers.json / models.json +gatekeep_providers_config() { + local f + for f in providers.yaml providers.yml providers.json models.json; do + if fpath="$(gatekeep_config_file "$f" 2>/dev/null)"; then + printf '%s\n' "$fpath" + return 0 + fi + done + echo "gatekeep: missing providers.yaml (or legacy providers.json / models.json)" >&2 + return 1 +} diff --git a/scripts/merge-findings.js b/scripts/merge-findings.js new file mode 100644 index 0000000..09bb741 --- /dev/null +++ b/scripts/merge-findings.js @@ -0,0 +1,230 @@ +#!/usr/bin/env node +/** + * Merge parallel model outputs into agreed findings + disputed + fail-closed PR risk tier. + * Zero npm dependencies (Node 20 stdlib only). + * + * Finding-level severity (Critical/High/Medium/Low) is distinct from PR risk tier. + * Every finding carries: severity, exact location, why_it_matters. + * + * Agreement: fingerprint must appear in >= agreementMin distinct models (default 2). + * Security: agreed security findings block; scan hard-fail always blocks. + */ +import { readFileSync, writeFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; + +const args = parseArgs(process.argv.slice(2)); +const dir = args.dir; +const floor = args.floor || "low"; +const agreementMin = Number(args["agreement-min"] || 2); +const scanHardFail = args["scan-hard-fail"] === "true"; +const outPath = args.out; + +const tierRank = { low: 0, medium: 1, high: 2 }; +const sevRank = { Critical: 0, High: 1, Medium: 2, Low: 3 }; +const maxTier = (a, b) => (tierRank[a] >= tierRank[b] ? a : b); +const normTier = (t) => { + const s = String(t || "").toLowerCase(); + if (s === "low" || s === "medium" || s === "high") return s; + if (s === "easy") return "low"; + if (s === "hard") return "high"; + return "high"; +}; + +const files = readdirSync(dir).filter((f) => f.startsWith("model-") && f.endsWith(".json")); +const models = []; +for (const f of files) { + try { + const j = JSON.parse(readFileSync(join(dir, f), "utf8")); + if (j.error || j.parse_error) { + models.push({ + model: j._model || f, + error: j.error || j.parse_error, + findings: [], + risk_tier: "high", + }); + continue; + } + models.push({ + model: j._model || f, + risk_tier: normTier(j.risk_tier), + summary: j.summary || "", + rationale: j.rationale || "", + findings: Array.isArray(j.findings) ? j.findings : [], + human_focus: Array.isArray(j.human_focus) ? j.human_focus : [], + }); + } catch (e) { + models.push({ model: f, error: String(e), findings: [], risk_tier: "high" }); + } +} + +function fingerprint(f) { + const path = String(f.path || "").replace(/\\/g, "/").toLowerCase(); + const lineBucket = f.line ? String(Math.floor(Number(f.line) / 5) * 5) : "file"; + const kind = String(f.kind || "other").toLowerCase(); + const title = String(f.title || "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim() + .split(/\s+/) + .slice(0, 8) + .join(" "); + return `${kind}|${path}|${lineBucket}|${title}`; +} + +function normSeverity(raw, kind, tag) { + const s = String(raw || "").trim(); + const map = { + critical: "Critical", + crit: "Critical", + high: "High", + medium: "Medium", + med: "Medium", + low: "Low", + important: "High", + nit: "Low", + }; + if (map[s.toLowerCase()]) return map[s.toLowerCase()]; + // Infer from kind/tag when model omitted severity + if (String(kind).toLowerCase() === "security") return "Critical"; + if (tag === "Important") return "High"; + if (tag === "Nit") return "Low"; + if (tag === "Pre-existing") return "Medium"; + return "Medium"; +} + +function formatLocation(path, line, lineEnd) { + const p = String(path || "").trim() || "(unknown file)"; + if (line == null || line === "") return p; + if (lineEnd != null && Number(lineEnd) > Number(line)) return `${p}:${line}-${lineEnd}`; + return `${p}:${line}`; +} + +function normalizeFinding(f, model) { + const kind = ["security", "correctness", "behavior", "other"].includes(String(f.kind).toLowerCase()) + ? String(f.kind).toLowerCase() + : "other"; + let tag = ["Important", "Nit", "Pre-existing"].includes(f.tag) ? f.tag : "Important"; + if (f.pre_existing === true) tag = "Pre-existing"; + const severity = normSeverity(f.severity, kind, tag); + const path = String(f.path || ""); + const line = typeof f.line === "number" ? f.line : undefined; + const line_end = typeof f.line_end === "number" ? f.line_end : undefined; + const why = + String(f.why_it_matters || f.why || f.impact || "").trim() || + // Fall back: last sentence-ish of body, else body itself + (() => { + const body = String(f.body || "").trim(); + if (!body) return "Impact not spelled out — treat as unresolved until clarified."; + const parts = body.split(/\n+/).map((x) => x.trim()).filter(Boolean); + return parts[parts.length - 1] || body; + })(); + const title = String(f.title || "Finding"); + const body = String(f.body || ""); + return { + path, + line, + line_end, + location: formatLocation(path, line, line_end), + severity, + tag, + kind, + title, + body, + why_it_matters: why, + blocking: Boolean(f.blocking) || kind === "security" || severity === "Critical", + models: [model], + fingerprint: fingerprint({ ...f, kind, title }), + }; +} + +function preferSeverity(a, b) { + return (sevRank[a] ?? 9) <= (sevRank[b] ?? 9) ? a : b; +} + +/** @type {Map} */ +const bucket = new Map(); +for (const m of models) { + for (const raw of m.findings) { + const n = normalizeFinding(raw, m.model); + const prev = bucket.get(n.fingerprint); + if (!prev) { + bucket.set(n.fingerprint, n); + } else { + if (!prev.models.includes(m.model)) prev.models.push(m.model); + prev.severity = preferSeverity(prev.severity, n.severity); + if (n.tag === "Important") prev.tag = "Important"; + if (n.tag === "Pre-existing" && prev.tag !== "Important") prev.tag = "Pre-existing"; + if ((n.why_it_matters || "").length > (prev.why_it_matters || "").length) { + prev.why_it_matters = n.why_it_matters; + } + if ((n.body || "").length > (prev.body || "").length) prev.body = n.body; + if (n.blocking) prev.blocking = true; + if (!prev.line && n.line) prev.line = n.line; + if (!prev.line_end && n.line_end) prev.line_end = n.line_end; + prev.location = formatLocation(prev.path, prev.line, prev.line_end); + } + } +} + +const agreed = []; +const disputed = []; +for (const f of bucket.values()) { + if (f.models.length >= agreementMin) agreed.push(f); + else disputed.push(f); +} + +const bySeverity = (a, b) => (sevRank[a.severity] ?? 9) - (sevRank[b.severity] ?? 9); +agreed.sort(bySeverity); +disputed.sort(bySeverity); + +let risk_tier = floor; +for (const m of models) risk_tier = maxTier(risk_tier, m.risk_tier); +for (const f of agreed) { + if (f.kind === "security" || f.severity === "Critical") risk_tier = maxTier(risk_tier, "high"); + else if (f.severity === "High" || f.tag === "Important") risk_tier = maxTier(risk_tier, "medium"); +} +if (scanHardFail) risk_tier = maxTier(risk_tier, "high"); + +const openSecurityBlocking = + scanHardFail || agreed.some((f) => (f.kind === "security" || f.severity === "Critical") && f.blocking); + +const result = { + risk_tier, + policy_floor: floor, + agreement_min: agreementMin, + models: models.map((m) => ({ + id: m.model, + risk_tier: m.risk_tier, + finding_count: m.findings.length, + error: m.error || null, + })), + agreed_findings: agreed, + disputed_findings: disputed, + open_security_blocking: openSecurityBlocking, + scan_hard_fail: scanHardFail, + human_focus: [...new Set(models.flatMap((m) => m.human_focus || []))].slice(0, 12), + summaries: models.map((m) => ({ model: m.model, summary: m.summary, rationale: m.rationale })), +}; + +writeFileSync(outPath, JSON.stringify(result, null, 2)); +process.stdout.write( + JSON.stringify({ + risk_tier: result.risk_tier, + security_blocking: openSecurityBlocking, + agreed_count: agreed.length, + disputed_count: disputed.length, + }) + "\n", +); + +function parseArgs(argv) { + const out = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith("--")) { + const key = a.slice(2); + const val = argv[i + 1] && !argv[i + 1].startsWith("--") ? argv[++i] : "true"; + out[key] = val; + } + } + return out; +} diff --git a/scripts/policy-floor.sh b/scripts/policy-floor.sh new file mode 100644 index 0000000..0d98ae6 --- /dev/null +++ b/scripts/policy-floor.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Deterministic fail-closed policy floor + lightweight security scan. +set -euo pipefail + +ROOT="${GATEKEEP_ROOT:-.}" +ACTION="${GATEKEEP_ACTION:-.}" +OUT="$ROOT/.gatekeep/out" +mkdir -p "$OUT" + +# shellcheck source=lib-config.sh +source "$ACTION/scripts/lib-config.sh" +POLICY="$(gatekeep_config_file policy.json)" + +EVENT="${GITHUB_EVENT_PATH:-}" +if [[ -z "$EVENT" || ! -f "$EVENT" ]]; then + echo "GITHUB_EVENT_PATH missing" >&2 + exit 1 +fi + +PR_NUMBER=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["pull_request"]["number"])' "$EVENT") +REPO="${GITHUB_REPOSITORY:?}" +OWNER="${REPO%%/*}" +NAME="${REPO##*/}" +FORGE="$ACTION/scripts/forge_api.py" + +python3 - "$FORGE" "$OWNER" "$NAME" "$PR_NUMBER" "$OUT" <<'PY' +import json, pathlib, sys, subprocess + +forge, owner, name, pr, out = sys.argv[1:6] +out = pathlib.Path(out) + +def api(method, path): + p = subprocess.run( + [sys.executable, forge, method, path], + capture_output=True, text=True, + ) + if p.returncode != 0: + return None + try: + return json.loads(p.stdout) if p.stdout.strip() else None + except Exception: + return None + +files_data = api("GET", f"/repos/{owner}/{name}/pulls/{pr}/files") or [] +# Gitea may paginate poorly; accept list +if isinstance(files_data, dict): + files_data = files_data.get("files") or [] +filenames = [] +add = delete = 0 +for f in files_data: + filenames.append(f.get("filename") or f.get("name") or "") + add += int(f.get("additions") or 0) + delete += int(f.get("deletions") or 0) +filenames = [f for f in filenames if f] +(out / "changed-files.txt").write_text("\n".join(filenames) + ("\n" if filenames else "")) + +pr_data = api("GET", f"/repos/{owner}/{name}/pulls/{pr}") or {} +if not add and not delete: + add = int(pr_data.get("additions") or 0) + delete = int(pr_data.get("deletions") or 0) +labels = [] +for l in pr_data.get("labels") or []: + if isinstance(l, dict): + labels.append(l.get("name") or "") + else: + labels.append(str(l)) +head_sha = ((pr_data.get("head") or {}).get("sha")) or "" +ci = "unknown" +if head_sha: + st = api("GET", f"/repos/{owner}/{name}/commits/{head_sha}/statuses") or [] + # Gitea returns list; derive combined + if isinstance(st, list) and st: + states = [s.get("state") for s in st] + if any(s == "failure" or s == "error" for s in states): + ci = "failure" + elif any(s == "pending" for s in states): + ci = "pending" + elif all(s == "success" for s in states): + ci = "success" + else: + ci = states[0] or "unknown" + elif isinstance(st, dict): + ci = st.get("state") or "unknown" + +meta = {"additions": add, "deletions": delete, "labels": labels, "ci": ci, "head_sha": head_sha} +(out / "pr-meta.json").write_text(json.dumps(meta, indent=2)) +print(json.dumps({"files": len(filenames), "add": add, "del": delete, "ci": ci})) +PY + +ADD=$(python3 -c 'import json; print(json.load(open("'"$OUT"'/pr-meta.json")).get("additions",0))') +DEL=$(python3 -c 'import json; print(json.load(open("'"$OUT"'/pr-meta.json")).get("deletions",0))') +LABELS=$(python3 -c 'import json; print(",".join(json.load(open("'"$OUT"'/pr-meta.json")).get("labels") or []))') +CI_STATE=$(python3 -c 'import json; print(json.load(open("'"$OUT"'/pr-meta.json")).get("ci","unknown"))') + +python3 - "$POLICY" "$OUT/changed-files.txt" "$ADD" "$DEL" "$LABELS" "$CI_STATE" "$OUT" <<'PY' +import json, fnmatch, sys, pathlib + +policy = json.load(open(sys.argv[1])) +files = [l.strip() for l in open(sys.argv[2]) if l.strip()] +add, delete = int(sys.argv[3] or 0), int(sys.argv[4] or 0) +labels = [x.strip().lower() for x in (sys.argv[5] or "").split(",") if x.strip()] +ci = sys.argv[6] +out = pathlib.Path(sys.argv[7]) + +rank = {"low": 0, "medium": 1, "high": 2} +floor = "low" +reasons = [] + +def bump(t, why): + global floor + if rank[t] > rank[floor]: + floor = t + reasons.append(why) + +for lab in labels: + if lab in [x.lower() for x in policy.get("force_high_labels", [])]: + bump("high", f"label forces high: {lab}") + if lab in [x.lower() for x in policy.get("force_medium_labels", [])]: + bump("medium", f"label forces medium: {lab}") + +def match(path, glob): + g = glob.replace("**/", "*").replace("**", "*") + return fnmatch.fnmatch(path, glob) or fnmatch.fnmatch(path, g) or fnmatch.fnmatch(path.split("/")[-1], glob) + +for f in files: + for g in policy.get("high_path_globs", []): + if match(f, g): + bump("high", f"high-path {f} ~ {g}") + for g in policy.get("medium_path_globs", []): + if match(f, g): + bump("medium", f"medium-path {f} ~ {g}") + +if len(files) > int(policy.get("low_max_files", 8)): + bump("medium", f"file count {len(files)} > low cap") +if add + delete > int(policy.get("low_max_diff_lines", 200)): + bump("medium", f"diff lines {add+delete} > low cap") +if ci in ("failure", "error"): + bump("high", f"CI status {ci}") + +seen=set(); uniq=[] +for r in reasons: + if r not in seen: + seen.add(r); uniq.append(r) + +result = {"floor": floor, "reasons": uniq, "files": files, "additions": add, "deletions": delete, "labels": labels, "ci": ci} +(out / "floor.json").write_text(json.dumps(result, indent=2)) +print(floor) +print("---") +print("\n".join(uniq)) +PY + +FLOOR_JSON="$OUT/floor.json" +FLOOR=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["floor"])' "$FLOOR_JSON") +REASONS=$(python3 -c 'import json,sys; print(" | ".join(json.load(open(sys.argv[1]))["reasons"]))' "$FLOOR_JSON") + +SCAN_OUT="$OUT/security-scan.json" +bash "$ACTION/scripts/security-scan.sh" "$OUT/changed-files.txt" "$SCAN_OUT" || true +SCAN_HARD=$(python3 -c 'import json,sys; print("true" if json.load(open(sys.argv[1])).get("hard_fail") else "false")' "$SCAN_OUT") + +if [[ "$SCAN_HARD" == "true" && "$FLOOR" != "high" ]]; then + FLOOR=high + REASONS="${REASONS} | deterministic security scan hard-fail" + python3 - "$FLOOR_JSON" <<'PY' +import json,sys +p=json.load(open(sys.argv[1])) +p["floor"]="high" +p["reasons"].append("deterministic security scan hard-fail") +json.dump(p, open(sys.argv[1],"w"), indent=2) +PY +fi + +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "floor=$FLOOR" >> "$GITHUB_OUTPUT" + echo "reasons=$REASONS" >> "$GITHUB_OUTPUT" + echo "scan_hard_fail=$SCAN_HARD" >> "$GITHUB_OUTPUT" +fi + +echo "Policy floor: $FLOOR (scan_hard_fail=$SCAN_HARD)" >&2 diff --git a/scripts/publish.js b/scripts/publish.js new file mode 100644 index 0000000..45c0c03 --- /dev/null +++ b/scripts/publish.js @@ -0,0 +1,293 @@ +#!/usr/bin/env node +/** + * Publish Gatekeep summary + agreed inline comments. + * Forge-agnostic (GitHub + Gitea) via fetch — no gh CLI required. + */ +import { readFileSync, writeFileSync } from "node:fs"; + +const mergedPath = process.argv[2]; +const floorPath = process.argv[3]; +const scanPath = process.argv[4]; +const merged = JSON.parse(readFileSync(mergedPath, "utf8")); +const floor = JSON.parse(readFileSync(floorPath, "utf8")); +const scan = scanPath ? JSON.parse(readFileSync(scanPath, "utf8")) : { hits: [] }; + +const event = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")); +const pr = event.pull_request; +const repo = process.env.GITHUB_REPOSITORY; +const [owner, name] = repo.split("/"); +const token = + process.env.GITHUB_TOKEN || process.env.GITEA_TOKEN || process.env.GH_TOKEN || ""; +const marker = ""; +const checkName = "gatekeep / security"; + +const apiBase = (process.env.GITHUB_API_URL || "https://api.github.com").replace(/\/$/, ""); +const gitea = !apiBase.includes("api.github.com"); + +const SEV_EMOJI = { Critical: "🔴", High: "🟠", Medium: "🟡", Low: "🟢" }; + +async function forgeApi(method, path, body) { + const p = path.startsWith("/") ? path : `/${path}`; + const url = `${apiBase}${p}`; + const headers = { + Accept: gitea ? "application/json" : "application/vnd.github+json", + "User-Agent": "gatekeep-action", + Authorization: gitea ? `token ${token}` : `Bearer ${token}`, + }; + if (body !== undefined) headers["Content-Type"] = "application/json"; + const res = await fetch(url, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }); + const text = await res.text(); + if (!res.ok) { + throw new Error(`${method} ${p} → HTTP ${res.status}: ${text.slice(0, 400)}`); + } + return text.trim() ? JSON.parse(text) : null; +} + +function approvals(tier) { + if (tier === "low") + return "Looks low-risk — no human needed once Gatekeep + security are green (auto-merge only if the kill switch is on)."; + if (tier === "medium") return "Needs **1 human** look. Bot thumbs-up don't count as that look."; + return "Needs **2 human** approvals. Bots never fill those seats."; +} + +function loc(f) { + return f.location || (f.path ? `${f.path}${f.line ? `:${f.line}` : ""}` : "(no location)"); +} + +function formatFindingBlock(f, { disputed = false } = {}) { + const sev = f.severity || "Medium"; + const emoji = SEV_EMOJI[sev] || "⚪"; + const pre = f.tag === "Pre-existing" ? " · pre-existing" : ""; + const block = f.blocking ? " · blocking" : ""; + const dispute = disputed ? " · disputed (not inlined)" : ""; + const tagNote = f.tag && f.tag !== "Important" ? ` · ${f.tag}` : ""; + const why = f.why_it_matters || "Impact not spelled out — please clarify before dismissing."; + const detail = (f.body || "").trim(); + const detailBit = detail && detail !== why ? `\n\n${detail}` : ""; + return ( + `#### ${emoji} ${sev}${pre}${block}${dispute}${tagNote}: ${f.title}\n` + + `- **Where:** \`${loc(f)}\`\n` + + `- **Why it matters:** ${why}` + + detailBit + + `\n- _Models:_ ${f.models.join(", ")}` + ); +} + +function formatInline(f) { + const sev = f.severity || "Medium"; + const emoji = SEV_EMOJI[sev] || "⚪"; + const pre = f.tag === "Pre-existing" ? " (pre-existing)" : ""; + const why = f.why_it_matters || "Impact not spelled out — please clarify before dismissing."; + const detail = (f.body || "").trim(); + let out = + `### ${emoji} ${sev}${pre} — ${f.title}\n\n` + + `**Where:** \`${loc(f)}\`\n\n` + + `**Why it matters:** ${why}\n`; + if (detail && detail !== why) out += `\n${detail}\n`; + out += + `\n---\n` + + `_Agreed by ${f.models.join(", ")}. Fix it, or reply \`dismissed: \` and resolve the thread._\n` + + `_No chat commands — Gatekeep doesn't take @-mentions._`; + return out; +} + +const sortedAgreed = [...(merged.agreed_findings || [])]; +const sortedDisputed = [...(merged.disputed_findings || [])]; + +const agreedList = + sortedAgreed.length === 0 + ? "_Nothing the models agreed on — nice, or spooky. Glance at disputed below._" + : sortedAgreed.map((f) => formatFindingBlock(f)).join("\n\n"); + +const disputedList = + sortedDisputed.length === 0 + ? "_No disagreements worth parking._" + : sortedDisputed.map((f) => formatFindingBlock(f, { disputed: true })).join("\n\n"); + +const modelRows = merged.models + .map( + (m) => + `| \`${m.id}\` | \`${m.risk_tier}\` | ${m.finding_count}${m.error ? ` (err: ${m.error})` : ""} |`, + ) + .join("\n"); + +const critCount = sortedAgreed.filter((f) => f.severity === "Critical").length; +const highCount = sortedAgreed.filter((f) => f.severity === "High").length; + +const body = `${marker} +## Gatekeep review + +Hey — here's the multi-model pass. **Severity first** so you can skim Critical → Low in seconds. + +| | | +| --- | --- | +| **PR risk tier** | \`${merged.risk_tier}\` | +| Policy floor | \`${merged.policy_floor}\` | +| Agreement bar | ≥${merged.agreement_min} models | +| Agreed / disputed | ${sortedAgreed.length} / ${sortedDisputed.length} | +| Critical / High (agreed) | ${critCount} / ${highCount} | + +${approvals(merged.risk_tier)} + +### Models (parallel) +| Model | Their PR tier | Findings | +| --- | --- | --- | +${modelRows} + +### Agreed findings +Sorted by severity. Each one spells out **severity**, **where**, and **why it matters**. +Fix or dismiss with \`dismissed: \` (then resolve the thread). Gatekeep has **no** chat / @-command surface. + +${agreedList} + +### Disputed (summary only — not posted inline) +One model saw it; others didn't. Useful signal, not a merge blocker by itself. + +${disputedList} + +### Deterministic security scan +${(scan.hits || []).length ? (scan.hits || []).map((h) => `- ${h}`).join("\n") : "_No hard-fail hits._"} + +### Policy floor signals +${(floor.reasons || []).map((r) => `- ${r}`).join("\n") || "- none"} + +### Still on humans +${(merged.human_focus || []).map((h) => `- ${h}`).join("\n") || "- taste, architecture, billing/CI/auth, reading the evidence"} + +--- +Missed something real? Open a **gatekeep-incident** issue — that's how it becomes an eval. +_Inline comments only when models agree. PR tier is fail-closed. Findings lead with severity + location + why._ +`; + +async function main() { + const comments = + (await forgeApi( + "GET", + `/repos/${owner}/${name}/issues/${pr.number}/comments?limit=100&per_page=100`, + )) || []; + const list = Array.isArray(comments) ? comments : []; + const existing = list.find((c) => (c.body || "").includes(marker)); + if (existing) { + await forgeApi("PATCH", `/repos/${owner}/${name}/issues/comments/${existing.id}`, { body }); + } else { + await forgeApi("POST", `/repos/${owner}/${name}/issues/${pr.number}/comments`, { body }); + } + + const inline = sortedAgreed + .filter((f) => f.path && f.line && f.severity !== "Low" && f.tag !== "Nit") + .slice(0, 30) + .map((f) => ({ + path: f.path, + // Gitea uses old_position/new_position in some versions; line works on GitHub + line: f.line, + body: formatInline(f), + })); + + if (inline.length) { + try { + await forgeApi("POST", `/repos/${owner}/${name}/pulls/${pr.number}/reviews`, { + commit_id: pr.head.sha, + event: "COMMENT", + body: "Gatekeep agreed findings (severity · where · why). Fix or dismiss — no @-commands.", + comments: inline, + }); + } catch (e) { + console.error("Inline review failed; falling back to issue comments:", e.message); + for (const f of inline) { + await forgeApi("POST", `/repos/${owner}/${name}/issues/${pr.number}/comments`, { + body: f.body + `\n\n_\`${f.path}:${f.line}\`_`, + }); + } + } + } + + for (const f of sortedAgreed) { + if ( + (f.severity === "Critical" || f.severity === "High" || f.tag === "Important") && + !f.line && + f.path + ) { + await forgeApi("POST", `/repos/${owner}/${name}/issues/${pr.number}/comments`, { + body: formatInline(f) + `\n\n_File-level (no single line) on \`${f.path}\`._`, + }); + } + } + + const want = `risk:${merged.risk_tier}`; + for (const lab of ["risk:low", "risk:medium", "risk:high"]) { + if (lab !== want) { + try { + await forgeApi( + "DELETE", + `/repos/${owner}/${name}/issues/${pr.number}/labels/${encodeURIComponent(lab)}`, + ); + } catch { + /* label may not exist */ + } + } + } + try { + // Gitea expects { labels: ["name"] }; GitHub same for issues API + await forgeApi("POST", `/repos/${owner}/${name}/issues/${pr.number}/labels`, { + labels: [want], + }); + } catch (e) { + console.error("label:", e.message); + } + + const conclusion = merged.open_security_blocking ? "failure" : "success"; + const statusState = merged.open_security_blocking ? "failure" : "success"; + const desc = merged.open_security_blocking + ? "Open security / Critical findings — merge blocked" + : `PR risk ${merged.risk_tier} — no open security blockers`; + + // Prefer Checks API (GitHub); fall back to commit status (Gitea-friendly) + try { + await forgeApi("POST", `/repos/${owner}/${name}/check-runs`, { + name: checkName, + head_sha: pr.head.sha, + status: "completed", + conclusion, + output: { + title: desc, + summary: desc, + }, + }); + } catch { + try { + await forgeApi("POST", `/repos/${owner}/${name}/statuses/${pr.head.sha}`, { + state: statusState, + context: checkName, + description: desc.slice(0, 140), + }); + } catch (e2) { + console.error("status/check publish failed:", e2.message); + } + } + + if (process.env.GITHUB_OUTPUT) { + writeFileSync( + process.env.GITHUB_OUTPUT, + [ + `risk_tier=${merged.risk_tier}`, + `security_blocking=${merged.open_security_blocking}`, + `agreed_count=${sortedAgreed.length}`, + ].join("\n") + "\n", + { flag: "a" }, + ); + } + + if (merged.open_security_blocking) { + console.error("Security blocking — exiting non-zero for job red X"); + process.exit(1); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/resolve-providers.py b/scripts/resolve-providers.py new file mode 100644 index 0000000..178b9e3 --- /dev/null +++ b/scripts/resolve-providers.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +""" +Resolve Gatekeep providers from YAML (or legacy JSON). + +Each provider entry needs: + key: FREE-FORM secret/env var name (NO required gatekeep_ prefix) + protocol: openai | openai-completions | openai-responses | anthropic +Optional: + base_url, models (defaults applied for known provider ids) + +Discovers model lists from the provider API; validates requested names; writes +provider-plan.json for mixed parallel review. +""" +from __future__ import annotations + +import json +import os +import sys +import urllib.error +import urllib.request +from pathlib import Path + +PROTOCOLS = ("openai", "openai-completions", "openai-responses", "anthropic") +# Wire aliases used by call-model.sh +PROTOCOL_WIRE = { + "openai": "openai-completions", + "openai-completions": "openai-completions", + "openai-compatible": "openai-completions", # legacy + "openai-responses": "openai-responses", + "anthropic": "anthropic", +} + +KNOWN_DEFAULTS = { + "openrouter": { + "base_url": "https://openrouter.ai/api/v1", + "protocol": "openai", + }, + "anthropic": { + "base_url": "https://api.anthropic.com", + "protocol": "anthropic", + }, + "openai": { + "base_url": "https://api.openai.com/v1", + "protocol": "openai", + }, +} + + +def die(msg: str, code: int = 1) -> None: + print(f"gatekeep: {msg}", file=sys.stderr) + sys.exit(code) + + +def load_raw(path: Path) -> dict: + text = path.read_text() + if path.suffix.lower() in (".yaml", ".yml"): + try: + import yaml # type: ignore + except ImportError: + die("PyYAML required to load providers.yaml (pip install pyyaml)") + raw = yaml.safe_load(text) or {} + else: + raw = json.loads(text) + + # Legacy models.json → single openrouter provider + if "providers" not in raw and "models" in raw: + base = raw.get("openrouter_base_url") or KNOWN_DEFAULTS["openrouter"]["base_url"] + raw = { + "agreement_min": raw.get("agreement_min", 2), + "temperature": raw.get("temperature", 0), + "max_tokens": raw.get("max_tokens", 4096), + "providers": { + "openrouter": { + "key": "OPENROUTER_API_KEY", + "protocol": "openai", + "base_url": base, + "models": raw["models"], + } + }, + } + + # Legacy JSON providers without `key` + providers = raw.get("providers") or {} + for pid, pconf in list(providers.items()): + if not isinstance(pconf, dict): + continue + if "key" not in pconf: + # migrate old shape: invent free-form env name suggestion from provider id + # (not forcing gatekeep_ prefix) + pconf = {**pconf, "key": f"{pid.upper()}_API_KEY"} + if pconf.get("protocol") == "openai-compatible": + pconf["protocol"] = "openai" + providers[pid] = pconf + raw["providers"] = providers + + if not providers: + die(f"no providers in {path}") + return raw + + +def secret_value(key_name: str) -> str: + """Look up FREE-FORM secret name in the process environment. No prefix rules.""" + name = (key_name or "").strip() + if not name: + return "" + val = os.environ.get(name, "") + if val and val.strip(): + return val.strip() + return "" + + +def http_json(url: str, headers: dict) -> dict | list: + req = urllib.request.Request(url, headers=headers, method="GET") + with urllib.request.urlopen(req, timeout=60) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def normalize_protocol(p: str) -> str: + p = (p or "").strip().lower() + if p not in PROTOCOL_WIRE: + die(f"unknown protocol {p!r} (want {' | '.join(PROTOCOLS)})") + return p + + +def wire_protocol(p: str) -> str: + return PROTOCOL_WIRE[normalize_protocol(p)] + + +def discover_model_ids(base_url: str, protocol: str, api_key: str) -> set[str]: + base = base_url.rstrip("/") + wire = wire_protocol(protocol) + ids: set[str] = set() + try: + if wire in ("openai-completions", "openai-responses"): + data = http_json( + f"{base}/models", + {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + ) + for m in data.get("data") or []: + mid = m.get("id") + if mid: + ids.add(mid) + elif wire == "anthropic": + data = http_json( + f"{base}/v1/models", + { + "x-api-key": api_key, + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + }, + ) + for m in data.get("data") or []: + mid = m.get("id") + if mid: + ids.add(mid) + else: + die(f"unhandled protocol wire {wire!r}") + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace")[:400] + die(f"model discovery failed for {base}: HTTP {e.code} {body}") + except Exception as e: + die(f"model discovery failed for {base}: {e}") + return ids + + +def enrich_provider(pid: str, pconf: dict) -> dict: + known = KNOWN_DEFAULTS.get(pid, {}) + out = {**pconf} + if not out.get("base_url"): + if known.get("base_url"): + out["base_url"] = known["base_url"] + else: + die(f"provider {pid}: base_url required (no default for this id)") + if not out.get("protocol"): + out["protocol"] = known.get("protocol") or "openai" + out["protocol"] = normalize_protocol(out["protocol"]) + if not out.get("key"): + die(f"provider {pid}: `key` (free-form secret/env name) is required") + if not isinstance(out.get("models"), list) or not out["models"]: + die(f"provider {pid}: `models` list required (names validated against provider API)") + return out + + +def main() -> None: + if len(sys.argv) < 3: + die("usage: resolve-providers.py [override-models-csv]") + cfg_path = Path(sys.argv[1]) + out_path = Path(sys.argv[2]) + override_csv = sys.argv[3] if len(sys.argv) > 3 else "" + + cfg = load_raw(cfg_path) + providers_in = cfg["providers"] + providers: dict[str, dict] = { + pid: enrich_provider(pid, pconf) for pid, pconf in providers_in.items() + } + + override_items: list[tuple[str | None, str]] = [] + if override_csv.strip(): + for part in override_csv.split(","): + part = part.strip() + if not part: + continue + head, _, rest = part.partition(":") + if rest and head in providers: + override_items.append((head, rest)) + else: + override_items.append((None, part)) + + requested: list[dict] = [] + if override_items: + catalog: dict[str, list[str]] = {} + for pid, pconf in providers.items(): + for m in pconf["models"]: + catalog.setdefault(m, []).append(pid) + for prov, model in override_items: + if prov: + requested.append({"provider": prov, "model": model}) + else: + hits = catalog.get(model) or [] + if len(hits) == 1: + requested.append({"provider": hits[0], "model": model}) + elif len(hits) > 1: + die(f"model {model!r} under multiple providers {hits}; use provider:model") + elif len(providers) == 1: + requested.append({"provider": next(iter(providers)), "model": model}) + else: + die(f"model {model!r} not in config; use provider:model") + else: + for pid, pconf in providers.items(): + for m in pconf["models"]: + requested.append({"provider": pid, "model": m}) + + seen: set[tuple[str, str]] = set() + uniq: list[dict] = [] + for r in requested: + key = (r["provider"], r["model"]) + if key not in seen: + seen.add(key) + uniq.append(r) + requested = uniq + + if len(requested) < 2: + die(f"need 2–3 models for parallel review; got {len(requested)}") + if len(requested) > 3: + print(f"gatekeep: truncating to first 3 of {len(requested)} models", file=sys.stderr) + requested = requested[:3] + + provider_meta: dict[str, dict] = {} + for pid in {r["provider"] for r in requested}: + pconf = providers[pid] + key_name = str(pconf["key"]) + api_key = secret_value(key_name) + if not api_key: + die( + f"missing secret/env for provider {pid!r}: expected env var named {key_name!r} " + f"(from providers.*.key — free-form; set it on the workflow step env from secrets)." + ) + available = discover_model_ids(pconf["base_url"], pconf["protocol"], api_key) + if not available: + die(f"provider {pid}: model list empty — check base_url/protocol/key") + provider_meta[pid] = { + "base_url": pconf["base_url"].rstrip("/"), + "protocol": pconf["protocol"], + "wire_protocol": wire_protocol(pconf["protocol"]), + "key_name": key_name, + "api_key": api_key, + "available_count": len(available), + "_available": available, + } + + plan_models = [] + for r in requested: + pid, model = r["provider"], r["model"] + avail = provider_meta[pid]["_available"] + if model not in avail: + sample = sorted(avail)[:8] + die( + f"model {model!r} not available on provider {pid} " + f"({provider_meta[pid]['available_count']} listed). Examples: {sample}" + ) + plan_models.append( + { + "provider": pid, + "model": model, + "base_url": provider_meta[pid]["base_url"], + "protocol": provider_meta[pid]["protocol"], + "wire_protocol": provider_meta[pid]["wire_protocol"], + "key_name": provider_meta[pid]["key_name"], + } + ) + + secrets_path = out_path.with_suffix(".secrets.json") + secrets_path.write_text( + json.dumps({pid: provider_meta[pid]["api_key"] for pid in provider_meta}, indent=2) + ) + os.chmod(secrets_path, 0o600) + + plan = { + "agreement_min": int(cfg.get("agreement_min", 2)), + "temperature": cfg.get("temperature", 0), + "max_tokens": int(cfg.get("max_tokens", 4096)), + "models": plan_models, + "secrets_file": str(secrets_path), + "providers_used": sorted(provider_meta.keys()), + } + out_path.write_text(json.dumps(plan, indent=2)) + print( + json.dumps( + { + "providers": plan["providers_used"], + "models": [f"{m['provider']}:{m['model']}" for m in plan_models], + "keys": {m["provider"]: m["key_name"] for m in plan_models}, + "agreement_min": plan["agreement_min"], + } + ), + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/review.sh b/scripts/review.sh new file mode 100644 index 0000000..74ef3f1 --- /dev/null +++ b/scripts/review.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Orchestrate parallel multi-provider model calls → merge → publish. +set -euo pipefail + +ROOT="${GATEKEEP_ROOT:-.}" +ACTION="${GATEKEEP_ACTION:-.}" +OUT="$ROOT/.gatekeep/out" +MODELS_DIR="$OUT/models" +mkdir -p "$MODELS_DIR" + +EVENT="${GITHUB_EVENT_PATH:?}" +export GATEKEEP_ROOT ACTION CONFIG_DIR OUT + +# shellcheck source=lib-config.sh +source "$ACTION/scripts/lib-config.sh" + +if [[ "${SKIP_DRAFT:-true}" == "true" ]]; then + DRAFT=$(python3 -c 'import json,sys; print(str(json.load(open(sys.argv[1]))["pull_request"].get("draft", False)).lower())' "$EVENT") + if [[ "$DRAFT" == "true" ]]; then + echo "Draft PR — skip" + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "risk_tier=skipped" >> "$GITHUB_OUTPUT" + echo "security_blocking=false" >> "$GITHUB_OUTPUT" + echo "agreed_count=0" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi +fi + +REPO="${GITHUB_REPOSITORY:?}" +PR=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["pull_request"]["number"])' "$EVENT") + +PROVIDERS_CFG="$(gatekeep_providers_config)" + +PLAN="$OUT/provider-plan.json" +python3 "$ACTION/scripts/resolve-providers.py" \ + "$PROVIDERS_CFG" \ + "$PLAN" \ + "${GATEKEEP_MODELS:-}" + +AGREEMENT_MIN=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["agreement_min"])' "$PLAN") +export GATEKEEP_MAX_TOKENS +GATEKEEP_MAX_TOKENS=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("max_tokens",4096))' "$PLAN") +SECRETS_FILE=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["secrets_file"])' "$PLAN") +cleanup_secrets() { rm -f "$SECRETS_FILE"; } +trap cleanup_secrets EXIT + +# Diff (truncated) — forge-agnostic (.diff URL) +: > "$OUT/diff.full" +SERVER="${GITHUB_SERVER_URL:-}" +TOK="${GITHUB_TOKEN:-${GITEA_TOKEN:-}}" +if [[ -n "$SERVER" ]]; then + curl -sS -H "Authorization: token ${TOK}" \ + "${SERVER%/}/${REPO}/pulls/${PR}.diff" -o "$OUT/diff.full" 2>/dev/null || true +fi +if [[ ! -s "$OUT/diff.full" ]]; then + python3 "$ACTION/scripts/forge_api.py" GET "/repos/${REPO}/pulls/${PR}.diff" >"$OUT/diff.full" 2>/dev/null || true +fi +python3 -c 'import pathlib; p=pathlib.Path("'"$OUT"'/diff.full"); t=p.read_text(errors="replace"); pathlib.Path("'"$OUT"'/diff.txt").write_text(t if len(t)<=200000 else t[:200000]+"\n\n… [truncated]")' + +FLOOR="${POLICY_FLOOR:-$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["floor"])' "$OUT/floor.json")}" +SCAN_HARD="${SCAN_HARD_FAIL:-false}" +export POLICY_FLOOR="$FLOOR" POLICY_REASONS="${POLICY_REASONS:-}" + +python3 <<'PY' +import json, os, pathlib +out = pathlib.Path(os.environ["OUT"]) +floor = os.environ.get("POLICY_FLOOR") or json.loads((out / "floor.json").read_text())["floor"] +reasons = os.environ.get("POLICY_REASONS", "") +event = json.loads(pathlib.Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) +pr = event["pull_request"] +files = [l.strip() for l in (out / "changed-files.txt").read_text().splitlines() if l.strip()] +floor_j = json.loads((out / "floor.json").read_text()) +diff = (out / "diff.txt").read_text() if (out / "diff.txt").exists() else "" +discover = json.loads((out / "discover.json").read_text()) if (out / "discover.json").exists() else {} +plan = json.loads((out / "provider-plan.json").read_text()) + +system = """You are one reviewer in Gatekeep, a multi-model PR review system for the repo owner's own projects. +Your job is useful review content that makes human triage cheap — not chat UX. + +Return ONLY valid JSON (no markdown fences) with: +{ + "risk_tier": "low|medium|high", + "summary": "short chatty overview", + "rationale": "why this PR risk tier", + "findings": [{ + "severity": "Critical|High|Medium|Low", + "path": "file path", + "line": 1, + "line_end": null, + "tag": "Important|Nit|Pre-existing", + "kind": "security|correctness|behavior|other", + "title": "short headline", + "why_it_matters": "concrete impact if ignored", + "body": "optional extra detail in a friendly tone", + "blocking": false + }], + "human_focus": ["string"] +} + +Finding rules (required on EVERY finding): +1. severity — Critical/High/Medium/Low so humans see critical vs minor instantly. (Distinct from PR risk_tier.) +2. exact location — path + line (use line_end for a range when needed). +3. why_it_matters — the impact/reason, not just \"please fix\". +Tone: clear and human; structured fields, not robotic checklist-speak. +tag is secondary (Important/Nit/Pre-existing); severity dominates. +Prefer real harm over style nits. Drop pure style. kind=security or severity=Critical ⇒ blocking true. +Pre-existing issues: still include severity + location + why if the PR touches or worsens them. + +PR risk_tier rules: +- From the change (paths, blast radius, security, CI), not author trust. +- MUST be >= the provided policy_floor. +- Security checklist seed: hardcoded secrets/tokens; prompt injection via repo files; auth bypass; unauthorized API access; billing logic bypass; CI workflow tampering; unsafe deserialization; SQL/command injection. +- You never approve as a human. Bots never fill human slots. +- Do not invent chat commands or ask humans to @-mention you. +""" +user = { + "policy_floor": floor, + "policy_floor_reasons": [r.strip() for r in reasons.split("|") if r.strip()], + "discover": discover, + "reviewers": [f"{m['provider']}:{m['model']}" for m in plan.get("models", [])], + "pull_request": { + "title": pr.get("title"), + "body": pr.get("body") or "", + "author": (pr.get("user") or {}).get("login"), + "base": (pr.get("base") or {}).get("ref"), + "head": (pr.get("head") or {}).get("ref"), + "labels": [l.get("name") if isinstance(l, dict) else l for l in (pr.get("labels") or [])], + "ci_status": floor_j.get("ci"), + "changed_files": files, + "additions": floor_j.get("additions"), + "deletions": floor_j.get("deletions"), + }, + "diff": diff, +} +json.dump({"system": system, "user": json.dumps(user, indent=2)}, open(out / "prompt.json", "w")) +print(f"Prompt ready; floor={floor}; plan={len(plan.get('models',[]))} models", flush=True) +PY + +# Fire models in parallel (may be mixed providers) +pids=() +i=0 +while IFS= read -r line; do + i=$((i + 1)) + provider=$(printf '%s' "$line" | python3 -c 'import json,sys; print(json.load(sys.stdin)["provider"])') + model=$(printf '%s' "$line" | python3 -c 'import json,sys; print(json.load(sys.stdin)["model"])') + protocol=$(printf '%s' "$line" | python3 -c 'import json,sys; print(json.load(sys.stdin)["protocol"])') + base_url=$(printf '%s' "$line" | python3 -c 'import json,sys; print(json.load(sys.stdin)["base_url"])') + safe=$(echo "${provider}_${model}" | tr '/:' '__') + bash "$ACTION/scripts/call-model.sh" \ + "$provider" "$model" "$protocol" "$base_url" \ + "$OUT/prompt.json" "$MODELS_DIR/model-$i-$safe.json" "$SECRETS_FILE" & + pids+=($!) +done < <(python3 -c 'import json,sys; [print(json.dumps(m)) for m in json.load(open(sys.argv[1]))["models"]]' "$PLAN") + +ec=0 +for pid in "${pids[@]}"; do + wait "$pid" || ec=1 +done +echo "Parallel model calls finished (wait_ec=$ec)" >&2 + +# Scrub secrets file from workspace artifact path if present +rm -f "$SECRETS_FILE" + +node "$ACTION/scripts/merge-findings.js" \ + --dir "$MODELS_DIR" \ + --floor "$FLOOR" \ + --agreement-min "$AGREEMENT_MIN" \ + --scan-hard-fail "$SCAN_HARD" \ + --out "$OUT/merged.json" | tee "$OUT/merge-meta.json" + +set +e +node "$ACTION/scripts/publish.js" "$OUT/merged.json" "$OUT/floor.json" "$OUT/security-scan.json" +pub_ec=$? +set -e + +META=$(cat "$OUT/merge-meta.json") +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + python3 -c 'import json,os,sys +m=json.loads(sys.argv[1]) +open(os.environ["GITHUB_OUTPUT"],"a").write( + f"risk_tier={m[\"risk_tier\"]}\nsecurity_blocking={str(m[\"security_blocking\"]).lower()}\nagreed_count={m[\"agreed_count\"]}\n" +)' "$META" +fi + +exit "$pub_ec" diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh new file mode 100644 index 0000000..fbace00 --- /dev/null +++ b/scripts/security-scan.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Deterministic security checklist seed scanner (paths + diff hunks). +# Hits here are hard-fail regardless of model agreement (see policy.security_agreement). +set -euo pipefail + +FILES_LIST="${1:?changed-files list}" +OUT="${2:?out json}" +ROOT="${GATEKEEP_ROOT:-.}" +REPO="${GITHUB_REPOSITORY:-}" +EVENT="${GITHUB_EVENT_PATH:-}" + +hits=() + +while IFS= read -r f; do + [[ -z "$f" ]] && continue + case "$f" in + .github/workflows/*|.github/actions/*) + hits+=("CI workflow tampering surface: $f") + ;; + *secret*|*credential*|.env*|.env) + hits+=("Secrets/credential path touched: $f") + ;; + esac +done < "$FILES_LIST" + +# Diff scan for high-signal patterns (best-effort; forge-agnostic) +if [[ -n "$EVENT" && -f "$EVENT" && -n "$REPO" ]]; then + PR=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["pull_request"]["number"])' "$EVENT") + ACTION_DIR="${GATEKEEP_ACTION:-$(cd "$(dirname "$0")/.." && pwd)}" + TMP=$(mktemp) + # Prefer .diff endpoint (works on Gitea + GitHub) + SERVER="${GITHUB_SERVER_URL:-}" + TOKEN="${GITHUB_TOKEN:-${GITEA_TOKEN:-}}" + if [[ -n "$SERVER" ]]; then + curl -sS -H "Authorization: token ${TOKEN}" \ + "${SERVER%/}/${REPO}/pulls/${PR}.diff" -o "$TMP" 2>/dev/null \ + || curl -sS -H "Authorization: token ${TOKEN}" \ + "${GITHUB_API_URL%/}/repos/${REPO}/pulls/${PR}.diff" -o "$TMP" 2>/dev/null \ + || true + fi + if [[ ! -s "$TMP" ]]; then + python3 "$ACTION_DIR/scripts/forge_api.py" GET "/repos/${REPO}/pulls/${PR}.diff" >"$TMP" 2>/dev/null || true + fi + DIFF=$(cat "$TMP" 2>/dev/null || true) + # Only look at added lines + ADDED=$(grep -E '^\+' "$TMP" | grep -vE '^\+\+\+' || true) + check() { + local name="$1" pat="$2" + if printf '%s\n' "$ADDED" | grep -Eiq -- "$pat"; then + hits+=("$name") + fi + } + check "Hardcoded secrets/tokens pattern" '(api[_-]?key|secret|token|password|private[_-]?key)\s*[:=]\s*['\''\"]?[A-Za-z0-9_\-]{16,}' + check "Prompt injection via repo instruction files" '(ignore (all )?(previous|above) instructions|you are now|system prompt)' + check "Possible SQL injection concat" '(execute\(|query\(|raw\().*\+|(SELECT|DROP|UNION).*\+' + check "Possible command injection" '(exec\(|spawn\(|os\.system\(|subprocess\.[a-z]+\([^)]*shell\s*=\s*True)' + check "Unsafe deserialization" '(pickle\.loads|yaml\.load\(|unserialize\(|ObjectInputStream)' + rm -f "$TMP" +fi + +HARD=false +if ((${#hits[@]} > 0)); then + HARD=true +fi + +python3 - "$OUT" "$HARD" "${hits[@]}" <<'PY' +import json,sys +out=sys.argv[1] +hard=sys.argv[2]=="true" +hits=sys.argv[3:] +json.dump({"hard_fail": hard, "hits": hits, "checklist": [ + "hardcoded secrets/tokens", + "prompt injection via repo files", + "auth bypass", + "unauthorized API access", + "billing logic bypass", + "CI workflow tampering", + "unsafe deserialization", + "SQL/command injection" +]}, open(out,"w"), indent=2) +print(f"security-scan hard_fail={hard} hits={len(hits)}", file=sys.stderr) +PY diff --git a/scripts/verify-benchmarks.sh b/scripts/verify-benchmarks.sh new file mode 100644 index 0000000..d5bebaf --- /dev/null +++ b/scripts/verify-benchmarks.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Verify layer: discover or run caller-configured benchmarks vs thresholds. +# Architecture-level performance debt is out of scope — metrics only. +set -euo pipefail + +ROOT="${GATEKEEP_ROOT:-.}" +ACTION="${GATEKEEP_ACTION:-.}" +OUT="$ROOT/.gatekeep/verify" +mkdir -p "$OUT" + +# shellcheck source=lib-config.sh +source "$ACTION/scripts/lib-config.sh" +BENCH_CFG="$(gatekeep_config_file benchmarks.json)" + +ENABLED=$(python3 -c 'import json,sys; print(str(json.load(open(sys.argv[1])).get("enabled", True)).lower())' "$BENCH_CFG") +if [[ "$ENABLED" != "true" ]]; then + echo "Benchmarks disabled" | tee "$OUT/result.txt" + exit 0 +fi + +python3 - "$BENCH_CFG" "$OUT" "$ROOT" <<'PY' +import json, os, pathlib, subprocess, time, re, sys + +cfg = json.load(open(sys.argv[1])) +out = pathlib.Path(sys.argv[2]) +root = pathlib.Path(sys.argv[3]) +thr = cfg.get("thresholds") or {} +cmds = cfg.get("commands") or {} +discover = {} +dpath = root / ".gatekeep/out/discover.json" +if dpath.exists(): + discover = json.loads(dpath.read_text()) + +def pick_command(kind_prefixes): + explicit = cmds.get("benchmark") or cmds.get(kind_prefixes[0].split(":")[0]) or "" + if explicit: + return explicit, "config" + for e in discover.get("entries") or []: + k = e.get("kind") or "" + if any(k.startswith(p) or p in k for p in kind_prefixes): + return e.get("command"), e.get("source") + return None, None + +results = {"metrics": {}, "thresholds": thr, "ran": [], "skipped": [], "pass": True} + +# Optional explicit startup / memory / fps harnesses — callers wire real collectors. +# Default: run discovered bench/test if present; parse common patterns when printed. + +bench_cmd, src = pick_command(["npm:bench", "npm:benchmark", "make:bench", "make:benchmark", "cargo:bench"]) +test_cmd, tsrc = pick_command(["npm:test", "make:test", "gradle:test", "cargo:test", "go:test", "pytest", "mvn:test"]) + +def run(label, cmd): + if not cmd: + results["skipped"].append(label) + return "" + results["ran"].append({"label": label, "command": cmd}) + t0 = time.time() + p = subprocess.run(cmd, shell=True, cwd=root, capture_output=True, text=True) + dt = int((time.time() - t0) * 1000) + (out / f"{label}.stdout.txt").write_text(p.stdout[-50000:]) + (out / f"{label}.stderr.txt").write_text(p.stderr[-20000:]) + results["metrics"][f"{label}_exit"] = p.returncode + results["metrics"][f"{label}_wall_ms"] = dt + if p.returncode != 0: + results["pass"] = False + return p.stdout + "\n" + p.stderr + +log = "" +if bench_cmd: + log += run("benchmark", bench_cmd) +elif test_cmd: + # Fall back: treat test suite wall time as coarse startup/regression signal only when no bench + log += run("verify_tests", test_cmd) +else: + results["skipped"].append("no discoverable benchmark/test command") + +# Parse metrics if present in output +def grab(name, patterns): + for pat in patterns: + m = re.search(pat, log, re.I | re.M) + if m: + try: + results["metrics"][name] = float(m.group(1)) + return + except Exception: + pass + +grab("fps", [r"fps[=:\s]+([0-9.]+)", r"frames?/s[=:\s]+([0-9.]+)"]) +grab("startup_ms", [r"startup[^0-9]*([0-9.]+)\s*ms", r"started in ([0-9.]+)\s*ms"]) +grab("memory_mb", [r"memory[^0-9]*([0-9.]+)\s*mb", r"rss[=:\s]+([0-9.]+)\s*mb"]) + +# Threshold checks (null skips) +def check(metric, op, key): + lim = thr.get(key) + if lim is None: + return + if metric not in results["metrics"]: + results["skipped"].append(f"missing metric for {key}") + return + val = results["metrics"][metric] + ok = val <= lim if op == "max" else val >= lim + results["metrics"][f"{key}_ok"] = ok + if not ok: + results["pass"] = False + +check("startup_ms", "max", "startup_ms_max") +check("memory_mb", "max", "memory_mb_max") +check("fps", "min", "fps_min") + +(out / "benchmarks.json").write_text(json.dumps(results, indent=2)) +print(json.dumps({"pass": results["pass"], "metrics": results["metrics"]}, indent=2)) +if not results["pass"]: + sys.exit(1) +PY diff --git a/templates/PULL_REQUEST_TEMPLATE.md b/templates/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..8a00314 --- /dev/null +++ b/templates/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,36 @@ +## Summary + + + +## Testing notes + +- [ ] Discovered build/test entry points run locally or in CI (paste command / job link) +- [ ] Benchmarks vs thresholds (fps / startup / memory) if behavior-sensitive — or N/A +- [ ] Client-visible paths: noted if **not** verified on GitHub-hosted runners + +Commands / evidence: + +``` + +``` + +## Risk (author hint — Gatekeep may raise the PR tier) + +Author self-assessment (optional): low / medium / high + +Sensitive paths touched? (auth, billing, `.github/workflows`, secrets, release): yes / no + + + +## How to revert + +Rollback speed is the load-bearing wall. Fill this in before merge. + +1. **One-step revert:** `git revert ` (or GitHub "Revert") then merge the revert PR. +2. **Artifact / slot rollback:** point `active` at previous known-good artifact (see Gatekeep deploy-slots note); restart. +3. **Feature flag / kill switch:** +4. **Data / world migrations:** +5. **Owner if this causes an incident:** + +If this change is hard to revert, treat it as **high** risk and get the required human approvals. diff --git a/templates/gatekeep-incident.yml b/templates/gatekeep-incident.yml new file mode 100644 index 0000000..15521b5 --- /dev/null +++ b/templates/gatekeep-incident.yml @@ -0,0 +1,70 @@ +name: Gatekeep incident +description: Record a review/merge miss so it becomes a fail-closed eval case +title: "[gatekeep-incident] " +labels: ["gatekeep-incident"] +body: + - type: markdown + attributes: + value: | + Service restore first. Then file this so Gatekeep learns (incident → eval). + Set `GATEKEEP_MERGE_ENABLED=false` while firefighting if auto-merge was involved. + - type: input + id: pr_url + attributes: + label: PR URL + placeholder: https://github.com/org/repo/pull/123 + validations: + required: true + - type: dropdown + id: miss_type + attributes: + label: What did Gatekeep get wrong? + options: + - Missed finding (security / Important not raised or not agreed) + - Wrong tier (under-tiered) + - Verify/benchmark miss + - Auto-merge latch error + - Other + validations: + required: true + - type: dropdown + id: ai_tier + attributes: + label: Tier at merge + options: [low, medium, high, unknown] + validations: + required: true + - type: dropdown + id: should_tier + attributes: + label: Should have been tier + options: [low, medium, high] + validations: + required: true + - type: textarea + id: failure + attributes: + label: Actual failure / incident summary + description: What broke, how you detected it, revert minutes if known. + validations: + required: true + - type: textarea + id: review_snapshot + attributes: + label: Gatekeep summary snapshot + description: Paste the Gatekeep review comment (agreed/disputed/models) at the time. + validations: + required: false + - type: input + id: revert_pr + attributes: + label: Revert PR / rollback command + validations: + required: false + - type: textarea + id: proposed_eval + attributes: + label: Proposed eval case / policy change + description: Paths, truncated diff, expect_min_tier, checklist item to pin. + validations: + required: false diff --git a/workflows/gatekeep-eval.yml b/workflows/gatekeep-eval.yml new file mode 100644 index 0000000..c761952 --- /dev/null +++ b/workflows/gatekeep-eval.yml @@ -0,0 +1,14 @@ +# SAMPLE for the Action repo or consumers — prefer .github/workflows/gatekeep-eval.yml on Gitea. +# Live dogfood eval workflow: .github/workflows/gatekeep-eval.yml + +name: Gatekeep evals + +on: + workflow_dispatch: + +jobs: + eval-floor: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + - run: bash scripts/eval-floor.sh evals/cases defaults/policy.json diff --git a/workflows/gatekeep-pr.yml b/workflows/gatekeep-pr.yml new file mode 100644 index 0000000..4c552cf --- /dev/null +++ b/workflows/gatekeep-pr.yml @@ -0,0 +1,33 @@ +# SAMPLE for *other* consumer repos — copy to .github/workflows/gatekeep-pr.yml +# +# On the Gatekeep dogfood repo itself, the live workflow is: +# .github/workflows/gatekeep-pr.yml (uses: ./) +# +# Secrets: free-form names from providers.yaml `key` (e.g. OPENROUTER_API_KEY) + +name: Gatekeep + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review, labeled] + +jobs: + gatekeep: + name: gatekeep + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + + - name: Gatekeep review + verify + id: gk + # Consumer remote reference (not used for dogfood — dogfood uses ./ ) + uses: https://git.potatolab.uk/layue13/gatekeep@main + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-verify: "true" + retain-artifacts: "true"