Add .github/workflows/gatekeep-pr.yml with uses: ./ and OPENROUTER_API_KEY from secrets. Forge-agnostic API helper for Gitea.
This commit is contained in:
layue13 committed
commit
05f8fad38a
48 files changed
+2844
No files matched your search
@@ -0,0 +1,30 @@
|
|||||||
|
# Floor evals on the Gatekeep Action repo itself.
|
||||||
|
|
||||||
|
name: Gatekeep evals
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
paths:
|
||||||
|
- "defaults/**"
|
||||||
|
- "config/**"
|
||||||
|
- "evals/**"
|
||||||
|
- "scripts/**"
|
||||||
|
- "action.yml"
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- "defaults/**"
|
||||||
|
- "config/**"
|
||||||
|
- "evals/**"
|
||||||
|
- "scripts/**"
|
||||||
|
- "action.yml"
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
eval-floor:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- name: Run floor evals
|
||||||
|
run: bash scripts/eval-floor.sh evals/cases defaults/policy.json
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
# Dogfood: run Gatekeep on PRs to THIS repo (layue13/gatekeep).
|
||||||
|
# Gitea Actions discovers workflows under .github/workflows/ (not /workflows/).
|
||||||
|
#
|
||||||
|
# Secret: OPENROUTER_API_KEY (already configured on this repo)
|
||||||
|
# Action: in-repo composite at repository root (uses: ./)
|
||||||
|
|
||||||
|
name: Gatekeep
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
types: [opened, synchronize, reopened, ready_for_review]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
gatekeep:
|
||||||
|
name: gatekeep
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
# Skip drafts on pull_request; allow manual dispatch
|
||||||
|
if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false
|
||||||
|
steps:
|
||||||
|
- name: Checkout PR / default branch
|
||||||
|
uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
|
||||||
|
- name: Gatekeep review (in-repo Action)
|
||||||
|
id: gk
|
||||||
|
# Local composite action — action.yml at repo root
|
||||||
|
uses: ./
|
||||||
|
env:
|
||||||
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
run-verify: "true"
|
||||||
|
retain-artifacts: "true"
|
||||||
|
|
||||||
|
- name: Job summary
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
{
|
||||||
|
echo "## Gatekeep dogfood"
|
||||||
|
echo "risk_tier=${{ steps.gk.outputs.risk_tier }}"
|
||||||
|
echo "security_blocking=${{ steps.gk.outputs.security_blocking }}"
|
||||||
|
echo "agreed_count=${{ steps.gk.outputs.agreed_count }}"
|
||||||
|
} >> "$GITHUB_STEP_SUMMARY"
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
.gatekeep/
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
# Gatekeep
|
||||||
|
|
||||||
|
AI-assisted PR review for **your own repos**: parallel multi-model review across **providers**, fail-closed PR risk tiers, agreed-only inline findings, blocking security check.
|
||||||
|
|
||||||
|
**中文说明：** [README.zh-CN.md](./README.zh-CN.md)
|
||||||
|
|
||||||
|
## Install (remote `uses:`)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# .github/workflows/gatekeep.yml ← must live under .github/workflows/ on Gitea/GitHub
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: https://git.potatolab.uk/layue13/gatekeep@main
|
||||||
|
# placeholder: uses: https://git.example.com/OWNER/gatekeep@v1
|
||||||
|
env:
|
||||||
|
# Names must match providers.yaml `key` — FREE-FORM (no required gatekeep_ prefix)
|
||||||
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dogfood (this repo)
|
||||||
|
|
||||||
|
`layue13/gatekeep` reviews **its own** PRs via [`.github/workflows/gatekeep-pr.yml`](.github/workflows/gatekeep-pr.yml) with `uses: ./` and `secrets.OPENROUTER_API_KEY`. Open a PR against `main` to see a review (Gitea Actions must have a runner — same pool as other `layue13` repos).
|
||||||
|
|
||||||
|
Allowlists / Gitea-on-github.com limitations: [`docs/remote-uses.md`](docs/remote-uses.md).
|
||||||
|
|
||||||
|
## Providers YAML
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
providers:
|
||||||
|
openrouter: # only include providers you configure
|
||||||
|
key: OPENROUTER_API_KEY # free-form secret/env name — pick any
|
||||||
|
protocol: openai # openai | openai-completions | openai-responses | anthropic
|
||||||
|
# base_url: optional (defaults exist for openrouter / anthropic / openai)
|
||||||
|
models:
|
||||||
|
- google/gemma-4-31b-it:free
|
||||||
|
- google/gemma-4-26b-a4b-it:free
|
||||||
|
- cohere/north-mini-code:free
|
||||||
|
- nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free
|
||||||
|
```
|
||||||
|
|
||||||
|
| Field | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `key` | **Free-form** name of the GitHub secret / env var holding the API key. **No** required `gatekeep_` prefix. |
|
||||||
|
| `protocol` | `openai` \| `openai-completions` \| `openai-responses` \| `anthropic` |
|
||||||
|
| `models` | Names to use (validated against the provider’s model list API at runtime) |
|
||||||
|
| `base_url` | Optional; known provider ids get defaults |
|
||||||
|
|
||||||
|
Mix providers in one review by listing several entries. Secrets are read from the **workflow step `env`** using each entry’s `key` name.
|
||||||
|
|
||||||
|
Example suggestion only (still free-form): `key: gatekeep_openrouter` is fine if you named your secret that — not required.
|
||||||
|
|
||||||
|
## Finding format
|
||||||
|
|
||||||
|
**Severity** (Critical→Low) + **where** (`path:line`) + **why it matters**. Inline only when ≥2 models agree.
|
||||||
|
|
||||||
|
## PR risk tiers
|
||||||
|
|
||||||
|
| Tier | Humans | Auto-merge |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| low | 0 after Gatekeep + security green | If kill switch on |
|
||||||
|
| medium | 1 | No |
|
||||||
|
| high | 2 | No |
|
||||||
|
|
||||||
|
## Optional consumer config
|
||||||
|
|
||||||
|
`config-dir: .github/gatekeep` with your `providers.yaml`. Omit to use Action `defaults/providers.yaml`.
|
||||||
|
|
||||||
|
## Layout
|
||||||
|
|
||||||
|
```
|
||||||
|
action.yml
|
||||||
|
scripts/ # resolve-providers.py, call-model.sh, …
|
||||||
|
defaults/providers.yaml
|
||||||
|
config/providers.yaml # example override
|
||||||
|
workflows/ templates/ evals/ incidents/ docs/
|
||||||
|
```
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
# Gatekeep
|
||||||
|
|
||||||
|
AI 辅助的 PR 审查 Action：多模型并行投票、按严重级别输出可扫读的结论，适合自己的仓库。
|
||||||
|
|
||||||
|
## 它做什么
|
||||||
|
|
||||||
|
- 对每个 PR 并行调用 **2–3 个模型**（**可来自不同供应商**），**仅当多个模型达成一致**时发 inline 评论。
|
||||||
|
- 每条结论明确写出：**严重级别**、**代码位置**、**为什么重要**。
|
||||||
|
- 汇总评论给出 PR **风险档**（low / medium / high）；安全相关可阻断合并。
|
||||||
|
- 不提供 CodeRabbit 式聊天 / `@命令`。
|
||||||
|
|
||||||
|
## 远程引用（推荐）
|
||||||
|
|
||||||
|
工作流文件必须放在 **`.github/workflows/`**（Gitea / GitHub 都认这个路径）。
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: https://git.potatolab.uk/layue13/gatekeep@main
|
||||||
|
# 占位：uses: https://git.example.com/OWNER/gatekeep@v1
|
||||||
|
env:
|
||||||
|
# 必须与 providers.yaml 里的 key 同名（自由命名，无强制 gatekeep_ 前缀）
|
||||||
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 自审（本仓库 dogfood）
|
||||||
|
|
||||||
|
`layue13/gatekeep` 用 [`.github/workflows/gatekeep-pr.yml`](.github/workflows/gatekeep-pr.yml) 对本仓库 PR 跑 Gatekeep：`uses: ./` + `secrets.OPENROUTER_API_KEY`。向 `main` 开 PR 即可触发（需有可用的 Gitea Actions runner）。
|
||||||
|
|
||||||
|
github.com runner 若不允许非 github.com Action 源，见 `docs/remote-uses.md`。
|
||||||
|
|
||||||
|
## 多供应商（providers.yaml）
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
providers:
|
||||||
|
openrouter:
|
||||||
|
key: OPENROUTER_API_KEY # 自由命名 — 任意 secret/环境变量名
|
||||||
|
protocol: openai # openai | openai-completions | openai-responses | anthropic
|
||||||
|
models:
|
||||||
|
- google/gemma-4-31b-it:free
|
||||||
|
- google/gemma-4-26b-a4b-it:free
|
||||||
|
- cohere/north-mini-code:free
|
||||||
|
- nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free
|
||||||
|
```
|
||||||
|
|
||||||
|
- **`key`：自由命名**，**不要**要求必须 `gatekeep_` 前缀（你也可以叫 `gatekeep_openrouter`，只是举例）。
|
||||||
|
- 每个供应商只需：`key` + `protocol`（`base_url` / `models` 按需；已知 id 有默认 base_url）。
|
||||||
|
- **不内置死模型清单**；启动时拉取供应商模型列表并校验名字。
|
||||||
|
- 可混用多个供应商；只写你真正配置了的条目。
|
||||||
|
|
||||||
|
## 输入面
|
||||||
|
|
||||||
|
| 项 | 含义 |
|
||||||
|
| --- | --- |
|
||||||
|
| `key` | Secret / 环境变量名（自由） |
|
||||||
|
| `protocol` | `openai` / `openai-completions` / `openai-responses` / `anthropic` |
|
||||||
|
| `models` | 参与并行审查的模型名 |
|
||||||
|
| step `env` | 把 `key` 同名的 secret 注入环境 |
|
||||||
|
| `github-token` | 发评论 / checks |
|
||||||
|
|
||||||
|
## 混用示例
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- uses: https://git.potatolab.uk/layue13/gatekeep@main
|
||||||
|
env:
|
||||||
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||||
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
config-dir: .github/gatekeep # 内含 providers.yaml，见 config/providers.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
## 风险档
|
||||||
|
|
||||||
|
| 档 | 人工 |
|
||||||
|
| --- | --- |
|
||||||
|
| low | 0（审查与安全绿后；kill switch 可自动合并） |
|
||||||
|
| medium | 1 人（Bot 不能顶替） |
|
||||||
|
| high | 2 人 |
|
||||||
|
|
||||||
|
## 更多
|
||||||
|
|
||||||
|
- 英文：`README.md`
|
||||||
|
- 远程限制：`docs/remote-uses.md`
|
||||||
+126
@@ -0,0 +1,126 @@
|
|||||||
|
name: Gatekeep
|
||||||
|
description: >
|
||||||
|
Parallel multi-model AI PR review across providers. Agreed findings become
|
||||||
|
inline comments; disagreements stay in the summary. Fail-closed risk tier.
|
||||||
|
Self-contained remote uses: git-url@ref. Provider secrets via free-form env names.
|
||||||
|
author: Gatekeep
|
||||||
|
|
||||||
|
inputs:
|
||||||
|
github-token:
|
||||||
|
description: Token with pull-requests + checks write
|
||||||
|
required: true
|
||||||
|
config-dir:
|
||||||
|
description: >
|
||||||
|
Optional consumer config directory (e.g. .github/gatekeep with providers.yaml).
|
||||||
|
Empty = Action defaults/providers.yaml.
|
||||||
|
required: false
|
||||||
|
default: ""
|
||||||
|
models:
|
||||||
|
description: >
|
||||||
|
Optional override of 2–3 models as comma-separated provider:model
|
||||||
|
(or bare names unique in config). Empty = providers.yaml lists.
|
||||||
|
required: false
|
||||||
|
default: ""
|
||||||
|
skip-draft:
|
||||||
|
description: Skip draft PRs
|
||||||
|
required: false
|
||||||
|
default: "true"
|
||||||
|
run-verify:
|
||||||
|
description: Discover + run verify/benchmarks layer
|
||||||
|
required: false
|
||||||
|
default: "true"
|
||||||
|
retain-artifacts:
|
||||||
|
description: Upload last-good style artifacts from verify
|
||||||
|
required: false
|
||||||
|
default: "true"
|
||||||
|
|
||||||
|
outputs:
|
||||||
|
risk_tier:
|
||||||
|
description: Final published risk tier
|
||||||
|
value: ${{ steps.review.outputs.risk_tier }}
|
||||||
|
security_blocking:
|
||||||
|
description: Whether security check should block
|
||||||
|
value: ${{ steps.review.outputs.security_blocking }}
|
||||||
|
agreed_count:
|
||||||
|
description: Count of multi-model agreed findings
|
||||||
|
value: ${{ steps.review.outputs.agreed_count }}
|
||||||
|
|
||||||
|
runs:
|
||||||
|
using: composite
|
||||||
|
steps:
|
||||||
|
- name: Setup Node (stdlib only)
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "20"
|
||||||
|
|
||||||
|
- name: Install PyYAML (providers.yaml)
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
python3 -m pip install --user pyyaml -q
|
||||||
|
python3 -c "import yaml; print('pyyaml ok', yaml.__version__)"
|
||||||
|
|
||||||
|
- name: Discover entry points
|
||||||
|
id: discover
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
GATEKEEP_ROOT: ${{ github.workspace }}
|
||||||
|
GATEKEEP_ACTION: ${{ github.action_path }}
|
||||||
|
CONFIG_DIR: ${{ inputs.config-dir }}
|
||||||
|
run: bash "${{ github.action_path }}/scripts/discover.sh"
|
||||||
|
|
||||||
|
- name: Policy floor + security scan
|
||||||
|
id: floor
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ inputs.github-token }}
|
||||||
|
CONFIG_DIR: ${{ inputs.config-dir }}
|
||||||
|
GATEKEEP_ROOT: ${{ github.workspace }}
|
||||||
|
GATEKEEP_ACTION: ${{ github.action_path }}
|
||||||
|
run: bash "${{ github.action_path }}/scripts/policy-floor.sh"
|
||||||
|
|
||||||
|
- name: Parallel multi-model review
|
||||||
|
id: review
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ inputs.github-token }}
|
||||||
|
CONFIG_DIR: ${{ inputs.config-dir }}
|
||||||
|
GATEKEEP_MODELS: ${{ inputs.models }}
|
||||||
|
SKIP_DRAFT: ${{ inputs.skip-draft }}
|
||||||
|
GATEKEEP_ROOT: ${{ github.workspace }}
|
||||||
|
GATEKEEP_ACTION: ${{ github.action_path }}
|
||||||
|
POLICY_FLOOR: ${{ steps.floor.outputs.floor }}
|
||||||
|
POLICY_REASONS: ${{ steps.floor.outputs.reasons }}
|
||||||
|
SCAN_HARD_FAIL: ${{ steps.floor.outputs.scan_hard_fail }}
|
||||||
|
DISCOVER_JSON: ${{ steps.discover.outputs.discover_json }}
|
||||||
|
# Provider API keys: set free-form names from providers.yaml `key`
|
||||||
|
# on this step's env (inherited from the workflow step). Example:
|
||||||
|
# env:
|
||||||
|
# OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||||
|
# MY_CUSTOM_KEY: ${{ secrets.MY_CUSTOM_KEY }}
|
||||||
|
run: |
|
||||||
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
|
# Ensure user-site packages (PyYAML) are importable
|
||||||
|
export PYTHONPATH="$(python3 -c 'import site; print(site.getusersitepackages())'):${PYTHONPATH:-}"
|
||||||
|
bash "${{ github.action_path }}/scripts/review.sh"
|
||||||
|
|
||||||
|
- name: Verify layer (benchmarks)
|
||||||
|
if: inputs.run-verify == 'true'
|
||||||
|
id: verify
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
GATEKEEP_ROOT: ${{ github.workspace }}
|
||||||
|
CONFIG_DIR: ${{ inputs.config-dir }}
|
||||||
|
GATEKEEP_ACTION: ${{ github.action_path }}
|
||||||
|
DISCOVER_JSON: ${{ steps.discover.outputs.discover_json }}
|
||||||
|
run: bash "${{ github.action_path }}/scripts/verify-benchmarks.sh"
|
||||||
|
|
||||||
|
- name: Retain artifacts
|
||||||
|
if: inputs.retain-artifacts == 'true' && always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: gatekeep-last-good-${{ github.run_id }}
|
||||||
|
path: |
|
||||||
|
${{ github.workspace }}/.gatekeep/out/
|
||||||
|
${{ github.workspace }}/.gatekeep/verify/
|
||||||
|
if-no-files-found: ignore
|
||||||
|
retention-days: 14
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# Example consumer overrides
|
||||||
|
|
||||||
|
Copy `providers.yaml` (and optionally `policy.json` / `benchmarks.json`) into e.g. `.github/gatekeep/`, then:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- uses: https://git.potatolab.uk/layue13/gatekeep@main
|
||||||
|
env:
|
||||||
|
# Must match each providers.*.key (free-form names)
|
||||||
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||||
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
config-dir: .github/gatekeep
|
||||||
|
```
|
||||||
|
|
||||||
|
`key` is **not** required to start with `gatekeep_`.
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"enabled": true,
|
||||||
|
"thresholds": {
|
||||||
|
"startup_ms_max": 15000,
|
||||||
|
"memory_mb_max": 1024,
|
||||||
|
"fps_min": 30
|
||||||
|
},
|
||||||
|
"notes": "Caller-configured verify-layer thresholds. Agent discovers how to run benches from README/package.json/Makefile; architecture-level perf debt is out of scope. Set a metric to null to skip it.",
|
||||||
|
"commands": {
|
||||||
|
"hint": "Optional explicit overrides. Empty = discover only.",
|
||||||
|
"benchmark": "",
|
||||||
|
"startup": "",
|
||||||
|
"memory": ""
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# Repo Actions variable (Settings → Secrets and variables → Actions → Variables)
|
||||||
|
# or environment variable consumed by workflows/gatekeep-pr.yml auto-merge latch.
|
||||||
|
|
||||||
|
GATEKEEP_MERGE_ENABLED=false
|
||||||
|
# Alias accepted by the workflow:
|
||||||
|
# AI_MERGE_ENABLED=false
|
||||||
|
|
||||||
|
# When false (default recommended until burn-in): even risk:low PRs are not
|
||||||
|
# auto-merged by Gatekeep. Humans may still merge per tier policy.
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"models": [
|
||||||
|
"meta-llama/llama-3.3-70b-instruct:free",
|
||||||
|
"google/gemma-3-27b-it:free",
|
||||||
|
"qwen/qwen3-235b-a22b:free"
|
||||||
|
],
|
||||||
|
"notes": "2–3 OpenRouter model IDs. Prefer free-tier (:free) for owner repos; override via action input models= or edit this file. Paid models work the same (drop :free).",
|
||||||
|
"openrouter_base_url": "https://openrouter.ai/api/v1",
|
||||||
|
"temperature": 0,
|
||||||
|
"max_tokens": 4096,
|
||||||
|
"agreement_min": 2
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"medium_path_globs": [
|
||||||
|
"**/package-lock.json",
|
||||||
|
"**/pnpm-lock.yaml",
|
||||||
|
"**/yarn.lock",
|
||||||
|
"src/**",
|
||||||
|
"lib/**",
|
||||||
|
"server/**",
|
||||||
|
"packages/**"
|
||||||
|
],
|
||||||
|
"high_path_globs": [
|
||||||
|
".github/workflows/**",
|
||||||
|
".github/actions/**",
|
||||||
|
"**/Dockerfile*",
|
||||||
|
"**/*secret*",
|
||||||
|
"**/*credential*",
|
||||||
|
"**/auth/**",
|
||||||
|
"**/billing/**",
|
||||||
|
"**/payment/**",
|
||||||
|
"**/.env*",
|
||||||
|
"**/security/**",
|
||||||
|
"scripts/release/**"
|
||||||
|
],
|
||||||
|
"low_max_files": 8,
|
||||||
|
"low_max_diff_lines": 200,
|
||||||
|
"force_high_labels": ["security", "do-not-automerge", "high-risk", "gatekeep:high"],
|
||||||
|
"force_medium_labels": ["needs-human", "medium-risk", "gatekeep:medium"],
|
||||||
|
"do_not_automerge_labels": ["do-not-automerge", "gatekeep:no-automerge"],
|
||||||
|
"security_agreement": "require_agreement_unless_scan_hard_fail"
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"_deprecated": "Prefer providers.yaml in this folder.",
|
||||||
|
"agreement_min": 2,
|
||||||
|
"providers": {
|
||||||
|
"openrouter": {
|
||||||
|
"key": "OPENROUTER_API_KEY",
|
||||||
|
"protocol": "openai",
|
||||||
|
"models": [
|
||||||
|
"google/gemma-4-31b-it:free",
|
||||||
|
"google/gemma-4-26b-a4b-it:free",
|
||||||
|
"cohere/north-mini-code:free",
|
||||||
|
"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# Example / owner consumer override — OpenRouter only.
|
||||||
|
# `key` is free-form; map it in the workflow step env from secrets.
|
||||||
|
|
||||||
|
agreement_min: 2
|
||||||
|
temperature: 0
|
||||||
|
max_tokens: 4096
|
||||||
|
|
||||||
|
providers:
|
||||||
|
openrouter:
|
||||||
|
key: OPENROUTER_API_KEY
|
||||||
|
protocol: openai
|
||||||
|
models:
|
||||||
|
- google/gemma-4-31b-it:free
|
||||||
|
- google/gemma-4-26b-a4b-it:free
|
||||||
|
- cohere/north-mini-code:free
|
||||||
|
- nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"enabled": true,
|
||||||
|
"thresholds": {
|
||||||
|
"startup_ms_max": 15000,
|
||||||
|
"memory_mb_max": 1024,
|
||||||
|
"fps_min": 30
|
||||||
|
},
|
||||||
|
"notes": "Caller-configured verify-layer thresholds. Agent discovers how to run benches from README/package.json/Makefile; architecture-level perf debt is out of scope. Set a metric to null to skip it.",
|
||||||
|
"commands": {
|
||||||
|
"hint": "Optional explicit overrides. Empty = discover only.",
|
||||||
|
"benchmark": "",
|
||||||
|
"startup": "",
|
||||||
|
"memory": ""
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"_deprecated": "Use providers.json. Kept only so old config-dir copies still resolve; resolve-providers.py migrates this shape.",
|
||||||
|
"models": [
|
||||||
|
"meta-llama/llama-3.3-70b-instruct:free",
|
||||||
|
"google/gemma-3-27b-it:free"
|
||||||
|
],
|
||||||
|
"openrouter_base_url": "https://openrouter.ai/api/v1",
|
||||||
|
"agreement_min": 2,
|
||||||
|
"temperature": 0,
|
||||||
|
"max_tokens": 4096
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"medium_path_globs": [
|
||||||
|
"**/package-lock.json",
|
||||||
|
"**/pnpm-lock.yaml",
|
||||||
|
"**/yarn.lock",
|
||||||
|
"src/**",
|
||||||
|
"lib/**",
|
||||||
|
"server/**",
|
||||||
|
"packages/**"
|
||||||
|
],
|
||||||
|
"high_path_globs": [
|
||||||
|
".github/workflows/**",
|
||||||
|
".github/actions/**",
|
||||||
|
"**/Dockerfile*",
|
||||||
|
"**/*secret*",
|
||||||
|
"**/*credential*",
|
||||||
|
"**/auth/**",
|
||||||
|
"**/billing/**",
|
||||||
|
"**/payment/**",
|
||||||
|
"**/.env*",
|
||||||
|
"**/security/**",
|
||||||
|
"scripts/release/**"
|
||||||
|
],
|
||||||
|
"low_max_files": 8,
|
||||||
|
"low_max_diff_lines": 200,
|
||||||
|
"force_high_labels": ["security", "do-not-automerge", "high-risk", "gatekeep:high"],
|
||||||
|
"force_medium_labels": ["needs-human", "medium-risk", "gatekeep:medium"],
|
||||||
|
"do_not_automerge_labels": ["do-not-automerge", "gatekeep:no-automerge"],
|
||||||
|
"security_agreement": "require_agreement_unless_scan_hard_fail"
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"_deprecated": "Use providers.yaml.",
|
||||||
|
"agreement_min": 2,
|
||||||
|
"providers": {
|
||||||
|
"openrouter": {
|
||||||
|
"key": "OPENROUTER_API_KEY",
|
||||||
|
"protocol": "openai",
|
||||||
|
"models": [
|
||||||
|
"google/gemma-4-31b-it:free",
|
||||||
|
"google/gemma-4-26b-a4b-it:free",
|
||||||
|
"cohere/north-mini-code:free",
|
||||||
|
"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
# Bundled defaults — only providers you actually use.
|
||||||
|
# `key` is a FREE-FORM secret / env var name (no required gatekeep_ prefix).
|
||||||
|
# Put that name in the workflow step env, e.g. env: { OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} }
|
||||||
|
|
||||||
|
agreement_min: 2
|
||||||
|
temperature: 0
|
||||||
|
max_tokens: 4096
|
||||||
|
|
||||||
|
providers:
|
||||||
|
openrouter:
|
||||||
|
key: OPENROUTER_API_KEY
|
||||||
|
protocol: openai
|
||||||
|
models:
|
||||||
|
- google/gemma-4-31b-it:free
|
||||||
|
- google/gemma-4-26b-a4b-it:free
|
||||||
|
- cohere/north-mini-code:free
|
||||||
|
- nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
# Deploy slots (stub — later)
|
||||||
|
|
||||||
|
Gatekeep’s PR Action does **not** perform production rollback. Ops keeps two slots:
|
||||||
|
|
||||||
|
| Slot | Role |
|
||||||
|
| --- | --- |
|
||||||
|
| `current` / `active` | What players/users hit now |
|
||||||
|
| `previous` | Last known-good artifact for one-command swap |
|
||||||
|
|
||||||
|
**v1 note only:** wire symlink or compose services outside this Action; record `previous_good_sha` on every deploy; PR template “How to revert” points operators here. Auto-rollback stays human-confirm by default except crash-loop burn-in (see Project rollback design).
|
||||||
|
|
||||||
|
Artifact retention for review/verify jobs uses `actions/upload-artifact` (`gatekeep-last-good-<run_id>`, 14 days) — that is **CI evidence**, not the production slot.
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
# Dogfood — Gatekeep reviews Gatekeep
|
||||||
|
|
||||||
|
The Action repo itself runs Gatekeep on its own pull requests.
|
||||||
|
|
||||||
|
| Piece | Path / value |
|
||||||
|
| --- | --- |
|
||||||
|
| Live workflow | `.github/workflows/gatekeep-pr.yml` |
|
||||||
|
| Action | `uses: ./` (repo-root `action.yml`) |
|
||||||
|
| Secret | `OPENROUTER_API_KEY` → step `env` |
|
||||||
|
| Trigger | `pull_request` (non-draft) + `workflow_dispatch` |
|
||||||
|
|
||||||
|
## How to see a review
|
||||||
|
|
||||||
|
1. Branch off `main`, make a tiny change, open a PR on https://git.potatolab.uk/layue13/gatekeep
|
||||||
|
2. Wait for the **Gatekeep** workflow (same Actions runner pool as other `layue13` repos)
|
||||||
|
3. Look for the summary comment `<!-- gatekeep-review -->` on the PR
|
||||||
|
|
||||||
|
Consumer sample workflows remain under top-level `workflows/` for copy-paste into *other* repos’ `.github/workflows/`.
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
# Remote `uses:` — limitations & workarounds
|
||||||
|
|
||||||
|
Gatekeep is a **self-contained composite Action** (`action.yml` at repo root + `scripts/` + `defaults/`). Consumers should prefer:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
uses: https://git.example.com/OWNER/gatekeep@v1
|
||||||
|
# Project example:
|
||||||
|
uses: https://git.potatolab.uk/layue13/gatekeep@main
|
||||||
|
```
|
||||||
|
|
||||||
|
No copy of Action sources into the consumer repo is required. Optional `.github/gatekeep/` overrides still work via `config-dir`.
|
||||||
|
|
||||||
|
## What must be true
|
||||||
|
|
||||||
|
| Requirement | Notes |
|
||||||
|
| --- | --- |
|
||||||
|
| Runner can fetch the Action git repo | Network + credentials if the Action repo is private |
|
||||||
|
| Runner can reach OpenRouter | `OPENROUTER_API_KEY` secret |
|
||||||
|
| `GITHUB_TOKEN` permissions | `pull-requests: write`, `checks: write` (see sample workflow) |
|
||||||
|
| Org / repo Actions policy | External Actions allowed |
|
||||||
|
|
||||||
|
## GitHub.com-hosted consumers + Gitea-hosted Action
|
||||||
|
|
||||||
|
**Honest status:** GitHub Actions’ first-class `uses: owner/repo@ref` syntax is for **github.com**. Full URL `uses: https://…@ref` is documented for public Actions, but **enterprise / org allowlists** often restrict non-`github.com` Action sources. Behavior can also differ by plan and policy version.
|
||||||
|
|
||||||
|
If `uses: https://git.potatolab.uk/layue13/gatekeep@main` fails on github.com runners with an allowlist / “not permitted” / fetch error:
|
||||||
|
|
||||||
|
### Workaround A — GitHub mirror (recommended)
|
||||||
|
|
||||||
|
1. Mirror `layue13/gatekeep` to a github.com repo you control (e.g. `your-org/gatekeep`).
|
||||||
|
2. Point consumers at:
|
||||||
|
```yaml
|
||||||
|
uses: your-org/gatekeep@main
|
||||||
|
# or: uses: your-org/gatekeep@v1
|
||||||
|
```
|
||||||
|
3. Keep the Gitea repo as source of truth; sync tags on release.
|
||||||
|
|
||||||
|
### Workaround B — Vendor checkout (still no Marketplace)
|
||||||
|
|
||||||
|
If remotes are fully blocked, checkout Gatekeep into the job and run the composite via local path:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
repository: layue13/gatekeep # or mirror
|
||||||
|
# For Gitea, use a custom remote + token:
|
||||||
|
# (or actions/checkout with sparse / git mirror step)
|
||||||
|
path: .gatekeep-action
|
||||||
|
ref: main
|
||||||
|
- uses: ./.gatekeep-action
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
openrouter-api-key: ${{ secrets.OPENROUTER_API_KEY }}
|
||||||
|
```
|
||||||
|
|
||||||
|
For a pure Gitea URL when `actions/checkout` only speaks github.com, use an explicit `git clone` step with a deploy token, then `uses: ./.gatekeep-action`.
|
||||||
|
|
||||||
|
### Workaround C — Copy-in (appendix)
|
||||||
|
|
||||||
|
Copy Action files into `.github/actions/gatekeep` (see README appendix). Same runtime; more drift risk.
|
||||||
|
|
||||||
|
## Gitea / Forgejo Actions runners
|
||||||
|
|
||||||
|
If the **consumer** also runs on Gitea Actions (not GitHub-hosted), use that forge’s documented `uses:` / local action syntax. Paths remain `github.action_path`-relative so the same Action tree works once the runner can resolve it.
|
||||||
|
|
||||||
|
## Secrets never travel with the Action
|
||||||
|
|
||||||
|
Remote fetch does **not** include consumer secrets. Always set `OPENROUTER_API_KEY` (and any forge token for private Action clones) on the **consumer** repo/org.
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# Gatekeep security seed checklist
|
||||||
|
|
||||||
|
Used in model prompts **and** the deterministic scanner (`action/scripts/security-scan.sh`).
|
||||||
|
|
||||||
|
| Item | Deterministic scan | Model review |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Hardcoded secrets/tokens | Added-line regex | Yes |
|
||||||
|
| Prompt injection via repo files | Instruction-override phrases in diff | Yes |
|
||||||
|
| Auth bypass | Path globs (`**/auth/**`) floor high | Yes |
|
||||||
|
| Unauthorized API access | — | Yes |
|
||||||
|
| Billing logic bypass | Path globs (`**/billing/**`, `**/payment/**`) | Yes |
|
||||||
|
| CI workflow tampering | `.github/workflows/**` → high + scan hit | Yes |
|
||||||
|
| Unsafe deserialization | pickle/yaml.load/unserialize patterns | Yes |
|
||||||
|
| SQL/command injection | Concat / `shell=True` patterns | Yes |
|
||||||
|
|
||||||
|
## Blocking policy (default)
|
||||||
|
|
||||||
|
`policy.json` → `security_agreement`: **`require_agreement_unless_scan_hard_fail`**
|
||||||
|
|
||||||
|
- **Inline + block:** security / **Critical** findings where **≥2 models agree**, or any **deterministic scan hard-fail**.
|
||||||
|
- **Summary only:** lone-model security flags (disputed) — do not inline; humans can still escalate.
|
||||||
|
- Scan hard-fail always floors PR tier to **high** and fails check `gatekeep / security`.
|
||||||
|
- Every finding (including security) still shows **severity + where + why it matters**.
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# Gatekeep floor evals
|
||||||
|
|
||||||
|
Pins the **deterministic policy floor** (fail-closed). Multi-model LLM regression is optional later; MVP evals do not call OpenRouter.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash scripts/eval-floor.sh evals/cases defaults/policy.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Add a case after every incident (`expect_min_tier` required). Seed set: ≥3 must-stay-low, ≥2 must-stay-high.
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"id": "unique-kebab-id",
|
||||||
|
"description": "What this pins",
|
||||||
|
"expect_min_tier": "high",
|
||||||
|
"expect_tier": "high",
|
||||||
|
"incident_ref": null,
|
||||||
|
"fixture": {
|
||||||
|
"title": "PR title",
|
||||||
|
"body": "",
|
||||||
|
"labels": [],
|
||||||
|
"files": ["path/to/file"],
|
||||||
|
"additions": 10,
|
||||||
|
"deletions": 2,
|
||||||
|
"ci_status": "success",
|
||||||
|
"diff": "diff --git a/path/to/file b/path/to/file\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"id": "must-stay-high-secrets",
|
||||||
|
"description": "auth paths never low",
|
||||||
|
"expect_min_tier": "high",
|
||||||
|
"expect_tier": "high",
|
||||||
|
"fixture": {
|
||||||
|
"title": "chore: comment in auth module",
|
||||||
|
"files": ["src/auth/session.ts"],
|
||||||
|
"additions": 3,
|
||||||
|
"deletions": 0,
|
||||||
|
"ci_status": "success",
|
||||||
|
"diff": "diff --git a/src/auth/session.ts b/src/auth/session.ts\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"id": "must-stay-high-workflow",
|
||||||
|
"description": ".github/workflows always high",
|
||||||
|
"expect_min_tier": "high",
|
||||||
|
"expect_tier": "high",
|
||||||
|
"incident_ref": "INC-EXAMPLE-001",
|
||||||
|
"fixture": {
|
||||||
|
"title": "ci: speed up unit job",
|
||||||
|
"files": [".github/workflows/ci.yml"],
|
||||||
|
"additions": 5,
|
||||||
|
"deletions": 1,
|
||||||
|
"ci_status": "success",
|
||||||
|
"diff": "diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"id": "must-stay-low-docs",
|
||||||
|
"description": "Docs-only stays low",
|
||||||
|
"expect_min_tier": "low",
|
||||||
|
"expect_tier": "low",
|
||||||
|
"fixture": {
|
||||||
|
"title": "docs: clarify setup",
|
||||||
|
"files": ["README.md", "docs/setup.md"],
|
||||||
|
"additions": 12,
|
||||||
|
"deletions": 4,
|
||||||
|
"ci_status": "success",
|
||||||
|
"diff": "diff --git a/README.md b/README.md\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"id": "must-stay-low-test-helper",
|
||||||
|
"description": "Isolated test fixture stays low",
|
||||||
|
"expect_min_tier": "low",
|
||||||
|
"expect_tier": "low",
|
||||||
|
"fixture": {
|
||||||
|
"title": "test: rename fixture id",
|
||||||
|
"files": ["tests/fixtures/world-seed.json"],
|
||||||
|
"additions": 2,
|
||||||
|
"deletions": 2,
|
||||||
|
"ci_status": "success",
|
||||||
|
"diff": "diff --git a/tests/fixtures/world-seed.json b/tests/fixtures/world-seed.json\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"id": "must-stay-low-typo",
|
||||||
|
"description": "Tiny non-sensitive typo stays low",
|
||||||
|
"expect_min_tier": "low",
|
||||||
|
"expect_tier": "low",
|
||||||
|
"fixture": {
|
||||||
|
"title": "fix: typo",
|
||||||
|
"files": ["tools/format-notes.txt"],
|
||||||
|
"additions": 1,
|
||||||
|
"deletions": 1,
|
||||||
|
"ci_status": "success",
|
||||||
|
"diff": "diff --git a/tools/format-notes.txt b/tools/format-notes.txt\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
# Incidents → evals
|
||||||
|
|
||||||
|
1. Restore service (revert / slot swap). Flip `GATEKEEP_MERGE_ENABLED=false` if needed.
|
||||||
|
2. File **gatekeep-incident** issue (template in `templates/gatekeep-incident.yml`).
|
||||||
|
3. Add `incidents/seed/<id>.json` + `evals/cases/<id>.json` with `expect_min_tier`.
|
||||||
|
4. Tighten `defaults/policy.json` (or consumer override) / security checklist; never delete a failing case to go green.
|
||||||
|
5. `bash scripts/eval-floor.sh evals/cases defaults/policy.json`
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# Incident record schema
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "INC-2026-001",
|
||||||
|
"date": "2026-10-26",
|
||||||
|
"severity": "low|medium|high|critical",
|
||||||
|
"summary": "One-line what went wrong",
|
||||||
|
"pr_url": "https://github.com/org/repo/pull/123",
|
||||||
|
"merge_sha": "abc123",
|
||||||
|
"ai_tier_at_merge": "low|medium|high|null",
|
||||||
|
"should_have_been_tier": "low|medium|high",
|
||||||
|
"root_cause": "classifier_under_tier|missed_agreed_finding|agreement_split|missing_oracle|human_skip|prompt_miss|other",
|
||||||
|
"models_at_review": ["model-a", "model-b"],
|
||||||
|
"revert_pr_url": "https://github.com/org/repo/pull/124",
|
||||||
|
"revert_minutes": 15,
|
||||||
|
"eval_case_id": "must-stay-high-from-inc-2026-001",
|
||||||
|
"policy_change": "What glob/label/checklist changed",
|
||||||
|
"notes": "Freeform"
|
||||||
|
}
|
||||||
|
```
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"id": "INC-EXAMPLE-001",
|
||||||
|
"date": "2026-10-01",
|
||||||
|
"severity": "high",
|
||||||
|
"summary": "Workflow change under-tiered; unpinned action merged.",
|
||||||
|
"pr_url": "https://github.com/example/repo/pull/0",
|
||||||
|
"merge_sha": "0000000000000000000000000000000000000000",
|
||||||
|
"ai_tier_at_merge": "low",
|
||||||
|
"should_have_been_tier": "high",
|
||||||
|
"root_cause": "classifier_under_tier",
|
||||||
|
"models_at_review": [
|
||||||
|
"meta-llama/llama-3.3-70b-instruct:free",
|
||||||
|
"google/gemma-3-27b-it:free"
|
||||||
|
],
|
||||||
|
"revert_pr_url": "https://github.com/example/repo/pull/1",
|
||||||
|
"revert_minutes": 12,
|
||||||
|
"eval_case_id": "must-stay-high-workflow",
|
||||||
|
"policy_change": "Keep .github/workflows/** in high_path_globs (seed policy).",
|
||||||
|
"notes": "Seed example only."
|
||||||
|
}
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Call one model via provider protocol.
|
||||||
|
# Protocols: openai | openai-completions | openai-responses | anthropic
|
||||||
|
# (openai / openai-compatible → chat completions)
|
||||||
|
# Args: <provider> <model> <protocol> <base_url> <prompt.json> <out.json> <secrets.json>
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PROVIDER="${1:?provider}"
|
||||||
|
MODEL="${2:?model}"
|
||||||
|
PROTOCOL="${3:?protocol}"
|
||||||
|
BASE_URL="${4:?base_url}"
|
||||||
|
PROMPT_FILE="${5:?prompt}"
|
||||||
|
OUT_FILE="${6:?out}"
|
||||||
|
SECRETS_FILE="${7:?secrets}"
|
||||||
|
|
||||||
|
KEY=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get(sys.argv[2],""))' "$SECRETS_FILE" "$PROVIDER")
|
||||||
|
if [[ -z "$KEY" ]]; then
|
||||||
|
echo "missing key for $PROVIDER" >&2
|
||||||
|
echo "{\"error\":\"missing_key\",\"model\":\"$MODEL\",\"provider\":\"$PROVIDER\"}" > "$OUT_FILE"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
MAX_TOKENS="${GATEKEEP_MAX_TOKENS:-4096}"
|
||||||
|
SYSTEM=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["system"])' "$PROMPT_FILE")
|
||||||
|
USER=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["user"])' "$PROMPT_FILE")
|
||||||
|
|
||||||
|
LABEL="${PROVIDER}:${MODEL}"
|
||||||
|
|
||||||
|
# Normalize protocol aliases
|
||||||
|
WIRE=$(python3 -c 'import sys
|
||||||
|
p=sys.argv[1].lower()
|
||||||
|
m={"openai":"openai-completions","openai-compatible":"openai-completions",
|
||||||
|
"openai-completions":"openai-completions","openai-responses":"openai-responses",
|
||||||
|
"anthropic":"anthropic"}
|
||||||
|
print(m.get(p,""))' "$PROTOCOL")
|
||||||
|
|
||||||
|
if [[ -z "$WIRE" ]]; then
|
||||||
|
echo "{\"error\":\"bad_protocol\",\"protocol\":\"$PROTOCOL\",\"model\":\"$MODEL\"}" > "$OUT_FILE"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$WIRE" == "openai-completions" ]]; then
|
||||||
|
BODY=$(python3 - "$MODEL" "$SYSTEM" "$USER" "$MAX_TOKENS" <<'PY'
|
||||||
|
import json,sys
|
||||||
|
model, system, user, max_tokens = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
|
||||||
|
print(json.dumps({
|
||||||
|
"model": model,
|
||||||
|
"temperature": 0,
|
||||||
|
"max_tokens": max_tokens,
|
||||||
|
"messages": [
|
||||||
|
{"role": "system", "content": system},
|
||||||
|
{"role": "user", "content": user},
|
||||||
|
],
|
||||||
|
}))
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
HTTP_CODE=$(curl -sS -o "$OUT_FILE.raw" -w "%{http_code}" \
|
||||||
|
"${BASE_URL%/}/chat/completions" \
|
||||||
|
-H "Authorization: Bearer $KEY" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "HTTP-Referer: https://github.com/${GITHUB_REPOSITORY:-gatekeep}" \
|
||||||
|
-H "X-Title: Gatekeep" \
|
||||||
|
-d "$BODY")
|
||||||
|
EXTRACT=openai
|
||||||
|
elif [[ "$WIRE" == "openai-responses" ]]; then
|
||||||
|
BODY=$(python3 - "$MODEL" "$SYSTEM" "$USER" "$MAX_TOKENS" <<'PY'
|
||||||
|
import json,sys
|
||||||
|
model, system, user, max_tokens = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
|
||||||
|
print(json.dumps({
|
||||||
|
"model": model,
|
||||||
|
"temperature": 0,
|
||||||
|
"max_output_tokens": max_tokens,
|
||||||
|
"input": [
|
||||||
|
{"role": "system", "content": system},
|
||||||
|
{"role": "user", "content": user},
|
||||||
|
],
|
||||||
|
}))
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
HTTP_CODE=$(curl -sS -o "$OUT_FILE.raw" -w "%{http_code}" \
|
||||||
|
"${BASE_URL%/}/responses" \
|
||||||
|
-H "Authorization: Bearer $KEY" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "$BODY")
|
||||||
|
EXTRACT=openai-responses
|
||||||
|
elif [[ "$WIRE" == "anthropic" ]]; then
|
||||||
|
BODY=$(python3 - "$MODEL" "$SYSTEM" "$USER" "$MAX_TOKENS" <<'PY'
|
||||||
|
import json,sys
|
||||||
|
model, system, user, max_tokens = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
|
||||||
|
print(json.dumps({
|
||||||
|
"model": model,
|
||||||
|
"max_tokens": max_tokens,
|
||||||
|
"temperature": 0,
|
||||||
|
"system": system,
|
||||||
|
"messages": [{"role": "user", "content": user}],
|
||||||
|
}))
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
HTTP_CODE=$(curl -sS -o "$OUT_FILE.raw" -w "%{http_code}" \
|
||||||
|
"${BASE_URL%/}/v1/messages" \
|
||||||
|
-H "x-api-key: $KEY" \
|
||||||
|
-H "anthropic-version: 2023-06-01" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "$BODY")
|
||||||
|
EXTRACT=anthropic
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$HTTP_CODE" != "200" ]]; then
|
||||||
|
echo "$LABEL failed HTTP $HTTP_CODE" >&2
|
||||||
|
head -c 800 "$OUT_FILE.raw" >&2 || true
|
||||||
|
echo "{\"error\":\"http\",\"model\":\"$MODEL\",\"provider\":\"$PROVIDER\",\"http\":$HTTP_CODE}" > "$OUT_FILE"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
python3 - "$OUT_FILE.raw" "$OUT_FILE" "$MODEL" "$PROVIDER" "$EXTRACT" <<'PY'
|
||||||
|
import json,sys
|
||||||
|
raw=json.load(open(sys.argv[1]))
|
||||||
|
out, model, provider, kind = sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5]
|
||||||
|
if kind == "anthropic":
|
||||||
|
text = "".join(
|
||||||
|
(c.get("text") or "")
|
||||||
|
for c in (raw.get("content") or [])
|
||||||
|
if c.get("type") == "text"
|
||||||
|
)
|
||||||
|
elif kind == "openai-responses":
|
||||||
|
# Best-effort extract from Responses API
|
||||||
|
text = raw.get("output_text") or ""
|
||||||
|
if not text:
|
||||||
|
chunks = []
|
||||||
|
for item in raw.get("output") or []:
|
||||||
|
for c in item.get("content") or []:
|
||||||
|
if c.get("type") in ("output_text", "text"):
|
||||||
|
chunks.append(c.get("text") or "")
|
||||||
|
text = "".join(chunks)
|
||||||
|
else:
|
||||||
|
text = (((raw.get("choices") or [{}])[0].get("message") or {}).get("content")) or ""
|
||||||
|
start, end = text.find("{"), text.rfind("}")
|
||||||
|
payload = {}
|
||||||
|
if start >= 0 and end > start:
|
||||||
|
try:
|
||||||
|
payload = json.loads(text[start:end+1])
|
||||||
|
except Exception as e:
|
||||||
|
payload = {"parse_error": str(e), "raw_text": text[:2000]}
|
||||||
|
else:
|
||||||
|
payload = {"parse_error": "no_json", "raw_text": text[:2000]}
|
||||||
|
payload["_model"] = f"{provider}:{model}"
|
||||||
|
payload["_provider"] = provider
|
||||||
|
json.dump(payload, open(out,"w"), indent=2)
|
||||||
|
PY
|
||||||
|
|
||||||
|
echo "$LABEL → $OUT_FILE" >&2
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Discover build/test/benchmark entry points from common manifests.
|
||||||
|
# No Minecraft-hardcoded commands — agent/caller repos vary.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="${GATEKEEP_ROOT:-.}"
|
||||||
|
OUT_DIR="$ROOT/.gatekeep/out"
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
|
||||||
|
json_escape() {
|
||||||
|
python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))' <<<"$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
entries=()
|
||||||
|
|
||||||
|
add() {
|
||||||
|
local kind="$1" cmd="$2" src="$3"
|
||||||
|
entries+=("{\"kind\":$(json_escape "$kind"),\"command\":$(json_escape "$cmd"),\"source\":$(json_escape "$src")}")
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ -f "$ROOT/package.json" ]]; then
|
||||||
|
# Prefer known script names without requiring jq
|
||||||
|
for s in test check lint build verify bench benchmark start; do
|
||||||
|
if python3 - "$ROOT/package.json" "$s" <<'PY'
|
||||||
|
import json,sys
|
||||||
|
p=json.load(open(sys.argv[1]))
|
||||||
|
scripts=p.get("scripts") or {}
|
||||||
|
sys.exit(0 if sys.argv[2] in scripts else 1)
|
||||||
|
PY
|
||||||
|
then
|
||||||
|
add "npm:$s" "npm run $s" "package.json"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f "$ROOT/Makefile" ]]; then
|
||||||
|
for t in test check verify bench benchmark build; do
|
||||||
|
if grep -qE "^${t}[[:space:]]*:" "$ROOT/Makefile"; then
|
||||||
|
add "make:$t" "make $t" "Makefile"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f "$ROOT/gradlew" ]]; then
|
||||||
|
add "gradle:test" "./gradlew test" "gradlew"
|
||||||
|
add "gradle:build" "./gradlew build" "gradlew"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f "$ROOT/pom.xml" ]]; then
|
||||||
|
add "mvn:test" "mvn -B test" "pom.xml"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f "$ROOT/Cargo.toml" ]]; then
|
||||||
|
add "cargo:test" "cargo test" "Cargo.toml"
|
||||||
|
add "cargo:bench" "cargo bench" "Cargo.toml"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f "$ROOT/go.mod" ]]; then
|
||||||
|
add "go:test" "go test ./..." "go.mod"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f "$ROOT/pyproject.toml" ]] || [[ -f "$ROOT/pytest.ini" ]]; then
|
||||||
|
add "pytest" "pytest -q" "python"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# README hints (non-executing notes for the model/verify layer)
|
||||||
|
readme_hints=""
|
||||||
|
for f in README.md README.rst docs/README.md; do
|
||||||
|
if [[ -f "$ROOT/$f" ]]; then
|
||||||
|
readme_hints+=$(grep -iE '^(#+ ).*(test|build|bench|verify|ci)' "$ROOT/$f" | head -20 || true)
|
||||||
|
readme_hints+=$'\n'
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
joined=$(IFS=,; echo "${entries[*]}")
|
||||||
|
discover_json="{\"entries\":[${joined}],\"readme_hints\":$(json_escape "$readme_hints")}"
|
||||||
|
printf '%s\n' "$discover_json" > "$OUT_DIR/discover.json"
|
||||||
|
|
||||||
|
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||||
|
{
|
||||||
|
echo "discover_json<<GATEKEEP_EOF"
|
||||||
|
echo "$discover_json"
|
||||||
|
echo "GATEKEEP_EOF"
|
||||||
|
} >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Discovered ${#entries[@]} entry point(s)" >&2
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Offline eval of deterministic policy floor (fail-closed). No OpenRouter calls.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
CASES_DIR="${1:?evals/cases dir}"
|
||||||
|
POLICY="${2:-}"
|
||||||
|
ACTION_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
export GATEKEEP_ACTION="$ACTION_ROOT"
|
||||||
|
export GATEKEEP_ROOT="${GATEKEEP_ROOT:-$ACTION_ROOT}"
|
||||||
|
export CONFIG_DIR="${CONFIG_DIR:-}"
|
||||||
|
# shellcheck source=lib-config.sh
|
||||||
|
source "$ACTION_ROOT/scripts/lib-config.sh"
|
||||||
|
POLICY="${POLICY:-$(gatekeep_config_file policy.json)}"
|
||||||
|
|
||||||
|
failed=0
|
||||||
|
total=0
|
||||||
|
|
||||||
|
for case in "$CASES_DIR"/*.json; do
|
||||||
|
[[ -f "$case" ]] || continue
|
||||||
|
[[ "$(basename "$case")" == _template.json ]] && continue
|
||||||
|
total=$((total + 1))
|
||||||
|
python3 - "$case" "$POLICY" <<'PY' || failed=$((failed + 1))
|
||||||
|
import json, fnmatch, sys
|
||||||
|
|
||||||
|
case = json.load(open(sys.argv[1]))
|
||||||
|
policy = json.load(open(sys.argv[2]))
|
||||||
|
fx = case["fixture"]
|
||||||
|
files = fx.get("files") or []
|
||||||
|
labels = [x.lower() for x in (fx.get("labels") or [])]
|
||||||
|
add = int(fx.get("additions") or 10)
|
||||||
|
delete = int(fx.get("deletions") or 2)
|
||||||
|
ci = fx.get("ci_status") or "success"
|
||||||
|
|
||||||
|
rank = {"low": 0, "medium": 1, "high": 2}
|
||||||
|
floor = "low"
|
||||||
|
reasons = []
|
||||||
|
|
||||||
|
def bump(t, why):
|
||||||
|
global floor
|
||||||
|
if rank[t] > rank[floor]:
|
||||||
|
floor = t
|
||||||
|
reasons.append(why)
|
||||||
|
|
||||||
|
def match(path, glob):
|
||||||
|
g = glob.replace("**/", "*").replace("**", "*")
|
||||||
|
return fnmatch.fnmatch(path, glob) or fnmatch.fnmatch(path, g)
|
||||||
|
|
||||||
|
for lab in labels:
|
||||||
|
if lab in [x.lower() for x in policy.get("force_high_labels", [])]:
|
||||||
|
bump("high", f"label {lab}")
|
||||||
|
if lab in [x.lower() for x in policy.get("force_medium_labels", [])]:
|
||||||
|
bump("medium", f"label {lab}")
|
||||||
|
|
||||||
|
for f in files:
|
||||||
|
for g in policy.get("high_path_globs", []):
|
||||||
|
if match(f, g):
|
||||||
|
bump("high", f"{f}~{g}")
|
||||||
|
for g in policy.get("medium_path_globs", []):
|
||||||
|
if match(f, g):
|
||||||
|
bump("medium", f"{f}~{g}")
|
||||||
|
|
||||||
|
if len(files) > int(policy.get("low_max_files", 8)):
|
||||||
|
bump("medium", "file count")
|
||||||
|
if add + delete > int(policy.get("low_max_diff_lines", 200)):
|
||||||
|
bump("medium", "diff lines")
|
||||||
|
if ci in ("failure", "error"):
|
||||||
|
bump("high", f"ci {ci}")
|
||||||
|
|
||||||
|
min_t = case["expect_min_tier"]
|
||||||
|
exact = case.get("expect_tier")
|
||||||
|
ok = rank[floor] >= rank[min_t] and (exact is None or floor == exact)
|
||||||
|
status = "PASS" if ok else "FAIL"
|
||||||
|
print(f"{status} {case['id']}: floor={floor} min={min_t}" + (f" expect={exact}" if exact else ""))
|
||||||
|
if not ok:
|
||||||
|
print(" " + "; ".join(reasons) or "(none)")
|
||||||
|
print(" " + case.get("description", ""))
|
||||||
|
sys.exit(1)
|
||||||
|
PY
|
||||||
|
done
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "$((total - failed))/$total passed"
|
||||||
|
exit "$failed"
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Minimal GitHub / Gitea API helper for Gatekeep (no gh CLI required).
|
||||||
|
Uses GITHUB_API_URL + GITHUB_TOKEN (Gitea Actions sets these to the forge).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
|
||||||
|
def api_base() -> str:
|
||||||
|
base = (os.environ.get("GITHUB_API_URL") or "https://api.github.com").rstrip("/")
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def is_gitea() -> bool:
|
||||||
|
b = api_base().lower()
|
||||||
|
return "api.github.com" not in b
|
||||||
|
|
||||||
|
|
||||||
|
def token() -> str:
|
||||||
|
return (
|
||||||
|
os.environ.get("GITHUB_TOKEN")
|
||||||
|
or os.environ.get("GITEA_TOKEN")
|
||||||
|
or os.environ.get("GH_TOKEN")
|
||||||
|
or ""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def request(method: str, path: str, body: dict | None = None, accept: str | None = None) -> tuple[int, str]:
|
||||||
|
path = path if path.startswith("/") else f"/{path}"
|
||||||
|
url = f"{api_base()}{path}"
|
||||||
|
data = None if body is None else json.dumps(body).encode("utf-8")
|
||||||
|
headers = {
|
||||||
|
"Authorization": f"token {token()}" if is_gitea() else f"Bearer {token()}",
|
||||||
|
"Accept": accept or ("application/json" if is_gitea() else "application/vnd.github+json"),
|
||||||
|
"User-Agent": "gatekeep-action",
|
||||||
|
}
|
||||||
|
if body is not None:
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
# GitHub also accepts token prefix
|
||||||
|
if not is_gitea() and token():
|
||||||
|
headers["Authorization"] = f"Bearer {token()}"
|
||||||
|
req = urllib.request.Request(url, data=data, headers=headers, method=method.upper())
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||||
|
return resp.status, resp.read().decode("utf-8", errors="replace")
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
return e.code, e.read().decode("utf-8", errors="replace")
|
||||||
|
|
||||||
|
|
||||||
|
def request_json(method: str, path: str, body: dict | None = None):
|
||||||
|
code, text = request(method, path, body)
|
||||||
|
if code >= 400:
|
||||||
|
raise RuntimeError(f"{method} {path} → HTTP {code}: {text[:500]}")
|
||||||
|
if not text.strip():
|
||||||
|
return None
|
||||||
|
return json.loads(text)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str]) -> int:
|
||||||
|
"""CLI: forge_api.py METHOD PATH [json-body] → prints response body"""
|
||||||
|
if len(argv) < 3:
|
||||||
|
print("usage: forge_api.py METHOD /path [json-body]", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
method, path = argv[1], argv[2]
|
||||||
|
body = json.loads(argv[3]) if len(argv) > 3 else None
|
||||||
|
code, text = request(method, path, body)
|
||||||
|
sys.stdout.write(text)
|
||||||
|
return 0 if code < 400 else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main(sys.argv))
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Resolve Gatekeep config files for remote-referenceable Action installs.
|
||||||
|
# Prefer optional consumer override dir; else bundled defaults under action_path.
|
||||||
|
#
|
||||||
|
# Expects: GATEKEEP_ROOT, GATEKEEP_ACTION, CONFIG_DIR (may be empty)
|
||||||
|
# Usage: source "$GATEKEEP_ACTION/scripts/lib-config.sh"
|
||||||
|
# cfg=$(gatekeep_config_file models.json)
|
||||||
|
|
||||||
|
gatekeep_config_file() {
|
||||||
|
local name="$1"
|
||||||
|
local root="${GATEKEEP_ROOT:-.}"
|
||||||
|
local action="${GATEKEEP_ACTION:-.}"
|
||||||
|
local cdir="${CONFIG_DIR:-}"
|
||||||
|
|
||||||
|
# Optional consumer override (only if config-dir set and file exists)
|
||||||
|
if [[ -n "$cdir" && -f "$root/$cdir/$name" ]]; then
|
||||||
|
printf '%s\n' "$root/$cdir/$name"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Bundled defaults shipped inside the Action (remote uses: / copy-in)
|
||||||
|
if [[ -f "$action/defaults/$name" ]]; then
|
||||||
|
printf '%s\n' "$action/defaults/$name"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Legacy pack layouts (dev / old copy-in)
|
||||||
|
if [[ -f "$action/../config/$name" ]]; then
|
||||||
|
printf '%s\n' "$action/../config/$name"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [[ -f "$action/../defaults/$name" ]]; then
|
||||||
|
printf '%s\n' "$action/../defaults/$name"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "gatekeep: missing config $name (no consumer override, no Action defaults)" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# Prefer providers.yaml, then providers.yml, then legacy providers.json / models.json
|
||||||
|
gatekeep_providers_config() {
|
||||||
|
local f
|
||||||
|
for f in providers.yaml providers.yml providers.json models.json; do
|
||||||
|
if fpath="$(gatekeep_config_file "$f" 2>/dev/null)"; then
|
||||||
|
printf '%s\n' "$fpath"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo "gatekeep: missing providers.yaml (or legacy providers.json / models.json)" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
@@ -0,0 +1,230 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Merge parallel model outputs into agreed findings + disputed + fail-closed PR risk tier.
|
||||||
|
* Zero npm dependencies (Node 20 stdlib only).
|
||||||
|
*
|
||||||
|
* Finding-level severity (Critical/High/Medium/Low) is distinct from PR risk tier.
|
||||||
|
* Every finding carries: severity, exact location, why_it_matters.
|
||||||
|
*
|
||||||
|
* Agreement: fingerprint must appear in >= agreementMin distinct models (default 2).
|
||||||
|
* Security: agreed security findings block; scan hard-fail always blocks.
|
||||||
|
*/
|
||||||
|
import { readFileSync, writeFileSync, readdirSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
const args = parseArgs(process.argv.slice(2));
|
||||||
|
const dir = args.dir;
|
||||||
|
const floor = args.floor || "low";
|
||||||
|
const agreementMin = Number(args["agreement-min"] || 2);
|
||||||
|
const scanHardFail = args["scan-hard-fail"] === "true";
|
||||||
|
const outPath = args.out;
|
||||||
|
|
||||||
|
const tierRank = { low: 0, medium: 1, high: 2 };
|
||||||
|
const sevRank = { Critical: 0, High: 1, Medium: 2, Low: 3 };
|
||||||
|
const maxTier = (a, b) => (tierRank[a] >= tierRank[b] ? a : b);
|
||||||
|
const normTier = (t) => {
|
||||||
|
const s = String(t || "").toLowerCase();
|
||||||
|
if (s === "low" || s === "medium" || s === "high") return s;
|
||||||
|
if (s === "easy") return "low";
|
||||||
|
if (s === "hard") return "high";
|
||||||
|
return "high";
|
||||||
|
};
|
||||||
|
|
||||||
|
const files = readdirSync(dir).filter((f) => f.startsWith("model-") && f.endsWith(".json"));
|
||||||
|
const models = [];
|
||||||
|
for (const f of files) {
|
||||||
|
try {
|
||||||
|
const j = JSON.parse(readFileSync(join(dir, f), "utf8"));
|
||||||
|
if (j.error || j.parse_error) {
|
||||||
|
models.push({
|
||||||
|
model: j._model || f,
|
||||||
|
error: j.error || j.parse_error,
|
||||||
|
findings: [],
|
||||||
|
risk_tier: "high",
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
models.push({
|
||||||
|
model: j._model || f,
|
||||||
|
risk_tier: normTier(j.risk_tier),
|
||||||
|
summary: j.summary || "",
|
||||||
|
rationale: j.rationale || "",
|
||||||
|
findings: Array.isArray(j.findings) ? j.findings : [],
|
||||||
|
human_focus: Array.isArray(j.human_focus) ? j.human_focus : [],
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
models.push({ model: f, error: String(e), findings: [], risk_tier: "high" });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function fingerprint(f) {
|
||||||
|
const path = String(f.path || "").replace(/\\/g, "/").toLowerCase();
|
||||||
|
const lineBucket = f.line ? String(Math.floor(Number(f.line) / 5) * 5) : "file";
|
||||||
|
const kind = String(f.kind || "other").toLowerCase();
|
||||||
|
const title = String(f.title || "")
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9]+/g, " ")
|
||||||
|
.trim()
|
||||||
|
.split(/\s+/)
|
||||||
|
.slice(0, 8)
|
||||||
|
.join(" ");
|
||||||
|
return `${kind}|${path}|${lineBucket}|${title}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normSeverity(raw, kind, tag) {
|
||||||
|
const s = String(raw || "").trim();
|
||||||
|
const map = {
|
||||||
|
critical: "Critical",
|
||||||
|
crit: "Critical",
|
||||||
|
high: "High",
|
||||||
|
medium: "Medium",
|
||||||
|
med: "Medium",
|
||||||
|
low: "Low",
|
||||||
|
important: "High",
|
||||||
|
nit: "Low",
|
||||||
|
};
|
||||||
|
if (map[s.toLowerCase()]) return map[s.toLowerCase()];
|
||||||
|
// Infer from kind/tag when model omitted severity
|
||||||
|
if (String(kind).toLowerCase() === "security") return "Critical";
|
||||||
|
if (tag === "Important") return "High";
|
||||||
|
if (tag === "Nit") return "Low";
|
||||||
|
if (tag === "Pre-existing") return "Medium";
|
||||||
|
return "Medium";
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatLocation(path, line, lineEnd) {
|
||||||
|
const p = String(path || "").trim() || "(unknown file)";
|
||||||
|
if (line == null || line === "") return p;
|
||||||
|
if (lineEnd != null && Number(lineEnd) > Number(line)) return `${p}:${line}-${lineEnd}`;
|
||||||
|
return `${p}:${line}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeFinding(f, model) {
|
||||||
|
const kind = ["security", "correctness", "behavior", "other"].includes(String(f.kind).toLowerCase())
|
||||||
|
? String(f.kind).toLowerCase()
|
||||||
|
: "other";
|
||||||
|
let tag = ["Important", "Nit", "Pre-existing"].includes(f.tag) ? f.tag : "Important";
|
||||||
|
if (f.pre_existing === true) tag = "Pre-existing";
|
||||||
|
const severity = normSeverity(f.severity, kind, tag);
|
||||||
|
const path = String(f.path || "");
|
||||||
|
const line = typeof f.line === "number" ? f.line : undefined;
|
||||||
|
const line_end = typeof f.line_end === "number" ? f.line_end : undefined;
|
||||||
|
const why =
|
||||||
|
String(f.why_it_matters || f.why || f.impact || "").trim() ||
|
||||||
|
// Fall back: last sentence-ish of body, else body itself
|
||||||
|
(() => {
|
||||||
|
const body = String(f.body || "").trim();
|
||||||
|
if (!body) return "Impact not spelled out — treat as unresolved until clarified.";
|
||||||
|
const parts = body.split(/\n+/).map((x) => x.trim()).filter(Boolean);
|
||||||
|
return parts[parts.length - 1] || body;
|
||||||
|
})();
|
||||||
|
const title = String(f.title || "Finding");
|
||||||
|
const body = String(f.body || "");
|
||||||
|
return {
|
||||||
|
path,
|
||||||
|
line,
|
||||||
|
line_end,
|
||||||
|
location: formatLocation(path, line, line_end),
|
||||||
|
severity,
|
||||||
|
tag,
|
||||||
|
kind,
|
||||||
|
title,
|
||||||
|
body,
|
||||||
|
why_it_matters: why,
|
||||||
|
blocking: Boolean(f.blocking) || kind === "security" || severity === "Critical",
|
||||||
|
models: [model],
|
||||||
|
fingerprint: fingerprint({ ...f, kind, title }),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function preferSeverity(a, b) {
|
||||||
|
return (sevRank[a] ?? 9) <= (sevRank[b] ?? 9) ? a : b;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @type {Map<string, any>} */
|
||||||
|
const bucket = new Map();
|
||||||
|
for (const m of models) {
|
||||||
|
for (const raw of m.findings) {
|
||||||
|
const n = normalizeFinding(raw, m.model);
|
||||||
|
const prev = bucket.get(n.fingerprint);
|
||||||
|
if (!prev) {
|
||||||
|
bucket.set(n.fingerprint, n);
|
||||||
|
} else {
|
||||||
|
if (!prev.models.includes(m.model)) prev.models.push(m.model);
|
||||||
|
prev.severity = preferSeverity(prev.severity, n.severity);
|
||||||
|
if (n.tag === "Important") prev.tag = "Important";
|
||||||
|
if (n.tag === "Pre-existing" && prev.tag !== "Important") prev.tag = "Pre-existing";
|
||||||
|
if ((n.why_it_matters || "").length > (prev.why_it_matters || "").length) {
|
||||||
|
prev.why_it_matters = n.why_it_matters;
|
||||||
|
}
|
||||||
|
if ((n.body || "").length > (prev.body || "").length) prev.body = n.body;
|
||||||
|
if (n.blocking) prev.blocking = true;
|
||||||
|
if (!prev.line && n.line) prev.line = n.line;
|
||||||
|
if (!prev.line_end && n.line_end) prev.line_end = n.line_end;
|
||||||
|
prev.location = formatLocation(prev.path, prev.line, prev.line_end);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const agreed = [];
|
||||||
|
const disputed = [];
|
||||||
|
for (const f of bucket.values()) {
|
||||||
|
if (f.models.length >= agreementMin) agreed.push(f);
|
||||||
|
else disputed.push(f);
|
||||||
|
}
|
||||||
|
|
||||||
|
const bySeverity = (a, b) => (sevRank[a.severity] ?? 9) - (sevRank[b.severity] ?? 9);
|
||||||
|
agreed.sort(bySeverity);
|
||||||
|
disputed.sort(bySeverity);
|
||||||
|
|
||||||
|
let risk_tier = floor;
|
||||||
|
for (const m of models) risk_tier = maxTier(risk_tier, m.risk_tier);
|
||||||
|
for (const f of agreed) {
|
||||||
|
if (f.kind === "security" || f.severity === "Critical") risk_tier = maxTier(risk_tier, "high");
|
||||||
|
else if (f.severity === "High" || f.tag === "Important") risk_tier = maxTier(risk_tier, "medium");
|
||||||
|
}
|
||||||
|
if (scanHardFail) risk_tier = maxTier(risk_tier, "high");
|
||||||
|
|
||||||
|
const openSecurityBlocking =
|
||||||
|
scanHardFail || agreed.some((f) => (f.kind === "security" || f.severity === "Critical") && f.blocking);
|
||||||
|
|
||||||
|
const result = {
|
||||||
|
risk_tier,
|
||||||
|
policy_floor: floor,
|
||||||
|
agreement_min: agreementMin,
|
||||||
|
models: models.map((m) => ({
|
||||||
|
id: m.model,
|
||||||
|
risk_tier: m.risk_tier,
|
||||||
|
finding_count: m.findings.length,
|
||||||
|
error: m.error || null,
|
||||||
|
})),
|
||||||
|
agreed_findings: agreed,
|
||||||
|
disputed_findings: disputed,
|
||||||
|
open_security_blocking: openSecurityBlocking,
|
||||||
|
scan_hard_fail: scanHardFail,
|
||||||
|
human_focus: [...new Set(models.flatMap((m) => m.human_focus || []))].slice(0, 12),
|
||||||
|
summaries: models.map((m) => ({ model: m.model, summary: m.summary, rationale: m.rationale })),
|
||||||
|
};
|
||||||
|
|
||||||
|
writeFileSync(outPath, JSON.stringify(result, null, 2));
|
||||||
|
process.stdout.write(
|
||||||
|
JSON.stringify({
|
||||||
|
risk_tier: result.risk_tier,
|
||||||
|
security_blocking: openSecurityBlocking,
|
||||||
|
agreed_count: agreed.length,
|
||||||
|
disputed_count: disputed.length,
|
||||||
|
}) + "\n",
|
||||||
|
);
|
||||||
|
|
||||||
|
function parseArgs(argv) {
|
||||||
|
const out = {};
|
||||||
|
for (let i = 0; i < argv.length; i++) {
|
||||||
|
const a = argv[i];
|
||||||
|
if (a.startsWith("--")) {
|
||||||
|
const key = a.slice(2);
|
||||||
|
const val = argv[i + 1] && !argv[i + 1].startsWith("--") ? argv[++i] : "true";
|
||||||
|
out[key] = val;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
@@ -0,0 +1,178 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Deterministic fail-closed policy floor + lightweight security scan.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="${GATEKEEP_ROOT:-.}"
|
||||||
|
ACTION="${GATEKEEP_ACTION:-.}"
|
||||||
|
OUT="$ROOT/.gatekeep/out"
|
||||||
|
mkdir -p "$OUT"
|
||||||
|
|
||||||
|
# shellcheck source=lib-config.sh
|
||||||
|
source "$ACTION/scripts/lib-config.sh"
|
||||||
|
POLICY="$(gatekeep_config_file policy.json)"
|
||||||
|
|
||||||
|
EVENT="${GITHUB_EVENT_PATH:-}"
|
||||||
|
if [[ -z "$EVENT" || ! -f "$EVENT" ]]; then
|
||||||
|
echo "GITHUB_EVENT_PATH missing" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
PR_NUMBER=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["pull_request"]["number"])' "$EVENT")
|
||||||
|
REPO="${GITHUB_REPOSITORY:?}"
|
||||||
|
OWNER="${REPO%%/*}"
|
||||||
|
NAME="${REPO##*/}"
|
||||||
|
FORGE="$ACTION/scripts/forge_api.py"
|
||||||
|
|
||||||
|
python3 - "$FORGE" "$OWNER" "$NAME" "$PR_NUMBER" "$OUT" <<'PY'
|
||||||
|
import json, pathlib, sys, subprocess
|
||||||
|
|
||||||
|
forge, owner, name, pr, out = sys.argv[1:6]
|
||||||
|
out = pathlib.Path(out)
|
||||||
|
|
||||||
|
def api(method, path):
|
||||||
|
p = subprocess.run(
|
||||||
|
[sys.executable, forge, method, path],
|
||||||
|
capture_output=True, text=True,
|
||||||
|
)
|
||||||
|
if p.returncode != 0:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return json.loads(p.stdout) if p.stdout.strip() else None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
files_data = api("GET", f"/repos/{owner}/{name}/pulls/{pr}/files") or []
|
||||||
|
# Gitea may paginate poorly; accept list
|
||||||
|
if isinstance(files_data, dict):
|
||||||
|
files_data = files_data.get("files") or []
|
||||||
|
filenames = []
|
||||||
|
add = delete = 0
|
||||||
|
for f in files_data:
|
||||||
|
filenames.append(f.get("filename") or f.get("name") or "")
|
||||||
|
add += int(f.get("additions") or 0)
|
||||||
|
delete += int(f.get("deletions") or 0)
|
||||||
|
filenames = [f for f in filenames if f]
|
||||||
|
(out / "changed-files.txt").write_text("\n".join(filenames) + ("\n" if filenames else ""))
|
||||||
|
|
||||||
|
pr_data = api("GET", f"/repos/{owner}/{name}/pulls/{pr}") or {}
|
||||||
|
if not add and not delete:
|
||||||
|
add = int(pr_data.get("additions") or 0)
|
||||||
|
delete = int(pr_data.get("deletions") or 0)
|
||||||
|
labels = []
|
||||||
|
for l in pr_data.get("labels") or []:
|
||||||
|
if isinstance(l, dict):
|
||||||
|
labels.append(l.get("name") or "")
|
||||||
|
else:
|
||||||
|
labels.append(str(l))
|
||||||
|
head_sha = ((pr_data.get("head") or {}).get("sha")) or ""
|
||||||
|
ci = "unknown"
|
||||||
|
if head_sha:
|
||||||
|
st = api("GET", f"/repos/{owner}/{name}/commits/{head_sha}/statuses") or []
|
||||||
|
# Gitea returns list; derive combined
|
||||||
|
if isinstance(st, list) and st:
|
||||||
|
states = [s.get("state") for s in st]
|
||||||
|
if any(s == "failure" or s == "error" for s in states):
|
||||||
|
ci = "failure"
|
||||||
|
elif any(s == "pending" for s in states):
|
||||||
|
ci = "pending"
|
||||||
|
elif all(s == "success" for s in states):
|
||||||
|
ci = "success"
|
||||||
|
else:
|
||||||
|
ci = states[0] or "unknown"
|
||||||
|
elif isinstance(st, dict):
|
||||||
|
ci = st.get("state") or "unknown"
|
||||||
|
|
||||||
|
meta = {"additions": add, "deletions": delete, "labels": labels, "ci": ci, "head_sha": head_sha}
|
||||||
|
(out / "pr-meta.json").write_text(json.dumps(meta, indent=2))
|
||||||
|
print(json.dumps({"files": len(filenames), "add": add, "del": delete, "ci": ci}))
|
||||||
|
PY
|
||||||
|
|
||||||
|
ADD=$(python3 -c 'import json; print(json.load(open("'"$OUT"'/pr-meta.json")).get("additions",0))')
|
||||||
|
DEL=$(python3 -c 'import json; print(json.load(open("'"$OUT"'/pr-meta.json")).get("deletions",0))')
|
||||||
|
LABELS=$(python3 -c 'import json; print(",".join(json.load(open("'"$OUT"'/pr-meta.json")).get("labels") or []))')
|
||||||
|
CI_STATE=$(python3 -c 'import json; print(json.load(open("'"$OUT"'/pr-meta.json")).get("ci","unknown"))')
|
||||||
|
|
||||||
|
python3 - "$POLICY" "$OUT/changed-files.txt" "$ADD" "$DEL" "$LABELS" "$CI_STATE" "$OUT" <<'PY'
|
||||||
|
import json, fnmatch, sys, pathlib
|
||||||
|
|
||||||
|
policy = json.load(open(sys.argv[1]))
|
||||||
|
files = [l.strip() for l in open(sys.argv[2]) if l.strip()]
|
||||||
|
add, delete = int(sys.argv[3] or 0), int(sys.argv[4] or 0)
|
||||||
|
labels = [x.strip().lower() for x in (sys.argv[5] or "").split(",") if x.strip()]
|
||||||
|
ci = sys.argv[6]
|
||||||
|
out = pathlib.Path(sys.argv[7])
|
||||||
|
|
||||||
|
rank = {"low": 0, "medium": 1, "high": 2}
|
||||||
|
floor = "low"
|
||||||
|
reasons = []
|
||||||
|
|
||||||
|
def bump(t, why):
|
||||||
|
global floor
|
||||||
|
if rank[t] > rank[floor]:
|
||||||
|
floor = t
|
||||||
|
reasons.append(why)
|
||||||
|
|
||||||
|
for lab in labels:
|
||||||
|
if lab in [x.lower() for x in policy.get("force_high_labels", [])]:
|
||||||
|
bump("high", f"label forces high: {lab}")
|
||||||
|
if lab in [x.lower() for x in policy.get("force_medium_labels", [])]:
|
||||||
|
bump("medium", f"label forces medium: {lab}")
|
||||||
|
|
||||||
|
def match(path, glob):
|
||||||
|
g = glob.replace("**/", "*").replace("**", "*")
|
||||||
|
return fnmatch.fnmatch(path, glob) or fnmatch.fnmatch(path, g) or fnmatch.fnmatch(path.split("/")[-1], glob)
|
||||||
|
|
||||||
|
for f in files:
|
||||||
|
for g in policy.get("high_path_globs", []):
|
||||||
|
if match(f, g):
|
||||||
|
bump("high", f"high-path {f} ~ {g}")
|
||||||
|
for g in policy.get("medium_path_globs", []):
|
||||||
|
if match(f, g):
|
||||||
|
bump("medium", f"medium-path {f} ~ {g}")
|
||||||
|
|
||||||
|
if len(files) > int(policy.get("low_max_files", 8)):
|
||||||
|
bump("medium", f"file count {len(files)} > low cap")
|
||||||
|
if add + delete > int(policy.get("low_max_diff_lines", 200)):
|
||||||
|
bump("medium", f"diff lines {add+delete} > low cap")
|
||||||
|
if ci in ("failure", "error"):
|
||||||
|
bump("high", f"CI status {ci}")
|
||||||
|
|
||||||
|
seen=set(); uniq=[]
|
||||||
|
for r in reasons:
|
||||||
|
if r not in seen:
|
||||||
|
seen.add(r); uniq.append(r)
|
||||||
|
|
||||||
|
result = {"floor": floor, "reasons": uniq, "files": files, "additions": add, "deletions": delete, "labels": labels, "ci": ci}
|
||||||
|
(out / "floor.json").write_text(json.dumps(result, indent=2))
|
||||||
|
print(floor)
|
||||||
|
print("---")
|
||||||
|
print("\n".join(uniq))
|
||||||
|
PY
|
||||||
|
|
||||||
|
FLOOR_JSON="$OUT/floor.json"
|
||||||
|
FLOOR=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["floor"])' "$FLOOR_JSON")
|
||||||
|
REASONS=$(python3 -c 'import json,sys; print(" | ".join(json.load(open(sys.argv[1]))["reasons"]))' "$FLOOR_JSON")
|
||||||
|
|
||||||
|
SCAN_OUT="$OUT/security-scan.json"
|
||||||
|
bash "$ACTION/scripts/security-scan.sh" "$OUT/changed-files.txt" "$SCAN_OUT" || true
|
||||||
|
SCAN_HARD=$(python3 -c 'import json,sys; print("true" if json.load(open(sys.argv[1])).get("hard_fail") else "false")' "$SCAN_OUT")
|
||||||
|
|
||||||
|
if [[ "$SCAN_HARD" == "true" && "$FLOOR" != "high" ]]; then
|
||||||
|
FLOOR=high
|
||||||
|
REASONS="${REASONS} | deterministic security scan hard-fail"
|
||||||
|
python3 - "$FLOOR_JSON" <<'PY'
|
||||||
|
import json,sys
|
||||||
|
p=json.load(open(sys.argv[1]))
|
||||||
|
p["floor"]="high"
|
||||||
|
p["reasons"].append("deterministic security scan hard-fail")
|
||||||
|
json.dump(p, open(sys.argv[1],"w"), indent=2)
|
||||||
|
PY
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||||
|
echo "floor=$FLOOR" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "reasons=$REASONS" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "scan_hard_fail=$SCAN_HARD" >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Policy floor: $FLOOR (scan_hard_fail=$SCAN_HARD)" >&2
|
||||||
@@ -0,0 +1,293 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Publish Gatekeep summary + agreed inline comments.
|
||||||
|
* Forge-agnostic (GitHub + Gitea) via fetch — no gh CLI required.
|
||||||
|
*/
|
||||||
|
import { readFileSync, writeFileSync } from "node:fs";
|
||||||
|
|
||||||
|
const mergedPath = process.argv[2];
|
||||||
|
const floorPath = process.argv[3];
|
||||||
|
const scanPath = process.argv[4];
|
||||||
|
const merged = JSON.parse(readFileSync(mergedPath, "utf8"));
|
||||||
|
const floor = JSON.parse(readFileSync(floorPath, "utf8"));
|
||||||
|
const scan = scanPath ? JSON.parse(readFileSync(scanPath, "utf8")) : { hits: [] };
|
||||||
|
|
||||||
|
const event = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, "utf8"));
|
||||||
|
const pr = event.pull_request;
|
||||||
|
const repo = process.env.GITHUB_REPOSITORY;
|
||||||
|
const [owner, name] = repo.split("/");
|
||||||
|
const token =
|
||||||
|
process.env.GITHUB_TOKEN || process.env.GITEA_TOKEN || process.env.GH_TOKEN || "";
|
||||||
|
const marker = "<!-- gatekeep-review -->";
|
||||||
|
const checkName = "gatekeep / security";
|
||||||
|
|
||||||
|
const apiBase = (process.env.GITHUB_API_URL || "https://api.github.com").replace(/\/$/, "");
|
||||||
|
const gitea = !apiBase.includes("api.github.com");
|
||||||
|
|
||||||
|
const SEV_EMOJI = { Critical: "🔴", High: "🟠", Medium: "🟡", Low: "🟢" };
|
||||||
|
|
||||||
|
async function forgeApi(method, path, body) {
|
||||||
|
const p = path.startsWith("/") ? path : `/${path}`;
|
||||||
|
const url = `${apiBase}${p}`;
|
||||||
|
const headers = {
|
||||||
|
Accept: gitea ? "application/json" : "application/vnd.github+json",
|
||||||
|
"User-Agent": "gatekeep-action",
|
||||||
|
Authorization: gitea ? `token ${token}` : `Bearer ${token}`,
|
||||||
|
};
|
||||||
|
if (body !== undefined) headers["Content-Type"] = "application/json";
|
||||||
|
const res = await fetch(url, {
|
||||||
|
method,
|
||||||
|
headers,
|
||||||
|
body: body !== undefined ? JSON.stringify(body) : undefined,
|
||||||
|
});
|
||||||
|
const text = await res.text();
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(`${method} ${p} → HTTP ${res.status}: ${text.slice(0, 400)}`);
|
||||||
|
}
|
||||||
|
return text.trim() ? JSON.parse(text) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function approvals(tier) {
|
||||||
|
if (tier === "low")
|
||||||
|
return "Looks low-risk — no human needed once Gatekeep + security are green (auto-merge only if the kill switch is on).";
|
||||||
|
if (tier === "medium") return "Needs **1 human** look. Bot thumbs-up don't count as that look.";
|
||||||
|
return "Needs **2 human** approvals. Bots never fill those seats.";
|
||||||
|
}
|
||||||
|
|
||||||
|
function loc(f) {
|
||||||
|
return f.location || (f.path ? `${f.path}${f.line ? `:${f.line}` : ""}` : "(no location)");
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatFindingBlock(f, { disputed = false } = {}) {
|
||||||
|
const sev = f.severity || "Medium";
|
||||||
|
const emoji = SEV_EMOJI[sev] || "⚪";
|
||||||
|
const pre = f.tag === "Pre-existing" ? " · pre-existing" : "";
|
||||||
|
const block = f.blocking ? " · blocking" : "";
|
||||||
|
const dispute = disputed ? " · disputed (not inlined)" : "";
|
||||||
|
const tagNote = f.tag && f.tag !== "Important" ? ` · ${f.tag}` : "";
|
||||||
|
const why = f.why_it_matters || "Impact not spelled out — please clarify before dismissing.";
|
||||||
|
const detail = (f.body || "").trim();
|
||||||
|
const detailBit = detail && detail !== why ? `\n\n${detail}` : "";
|
||||||
|
return (
|
||||||
|
`#### ${emoji} ${sev}${pre}${block}${dispute}${tagNote}: ${f.title}\n` +
|
||||||
|
`- **Where:** \`${loc(f)}\`\n` +
|
||||||
|
`- **Why it matters:** ${why}` +
|
||||||
|
detailBit +
|
||||||
|
`\n- _Models:_ ${f.models.join(", ")}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatInline(f) {
|
||||||
|
const sev = f.severity || "Medium";
|
||||||
|
const emoji = SEV_EMOJI[sev] || "⚪";
|
||||||
|
const pre = f.tag === "Pre-existing" ? " (pre-existing)" : "";
|
||||||
|
const why = f.why_it_matters || "Impact not spelled out — please clarify before dismissing.";
|
||||||
|
const detail = (f.body || "").trim();
|
||||||
|
let out =
|
||||||
|
`### ${emoji} ${sev}${pre} — ${f.title}\n\n` +
|
||||||
|
`**Where:** \`${loc(f)}\`\n\n` +
|
||||||
|
`**Why it matters:** ${why}\n`;
|
||||||
|
if (detail && detail !== why) out += `\n${detail}\n`;
|
||||||
|
out +=
|
||||||
|
`\n---\n` +
|
||||||
|
`_Agreed by ${f.models.join(", ")}. Fix it, or reply \`dismissed: <reason>\` and resolve the thread._\n` +
|
||||||
|
`_No chat commands — Gatekeep doesn't take @-mentions._`;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
const sortedAgreed = [...(merged.agreed_findings || [])];
|
||||||
|
const sortedDisputed = [...(merged.disputed_findings || [])];
|
||||||
|
|
||||||
|
const agreedList =
|
||||||
|
sortedAgreed.length === 0
|
||||||
|
? "_Nothing the models agreed on — nice, or spooky. Glance at disputed below._"
|
||||||
|
: sortedAgreed.map((f) => formatFindingBlock(f)).join("\n\n");
|
||||||
|
|
||||||
|
const disputedList =
|
||||||
|
sortedDisputed.length === 0
|
||||||
|
? "_No disagreements worth parking._"
|
||||||
|
: sortedDisputed.map((f) => formatFindingBlock(f, { disputed: true })).join("\n\n");
|
||||||
|
|
||||||
|
const modelRows = merged.models
|
||||||
|
.map(
|
||||||
|
(m) =>
|
||||||
|
`| \`${m.id}\` | \`${m.risk_tier}\` | ${m.finding_count}${m.error ? ` (err: ${m.error})` : ""} |`,
|
||||||
|
)
|
||||||
|
.join("\n");
|
||||||
|
|
||||||
|
const critCount = sortedAgreed.filter((f) => f.severity === "Critical").length;
|
||||||
|
const highCount = sortedAgreed.filter((f) => f.severity === "High").length;
|
||||||
|
|
||||||
|
const body = `${marker}
|
||||||
|
## Gatekeep review
|
||||||
|
|
||||||
|
Hey — here's the multi-model pass. **Severity first** so you can skim Critical → Low in seconds.
|
||||||
|
|
||||||
|
| | |
|
||||||
|
| --- | --- |
|
||||||
|
| **PR risk tier** | \`${merged.risk_tier}\` |
|
||||||
|
| Policy floor | \`${merged.policy_floor}\` |
|
||||||
|
| Agreement bar | ≥${merged.agreement_min} models |
|
||||||
|
| Agreed / disputed | ${sortedAgreed.length} / ${sortedDisputed.length} |
|
||||||
|
| Critical / High (agreed) | ${critCount} / ${highCount} |
|
||||||
|
|
||||||
|
${approvals(merged.risk_tier)}
|
||||||
|
|
||||||
|
### Models (parallel)
|
||||||
|
| Model | Their PR tier | Findings |
|
||||||
|
| --- | --- | --- |
|
||||||
|
${modelRows}
|
||||||
|
|
||||||
|
### Agreed findings
|
||||||
|
Sorted by severity. Each one spells out **severity**, **where**, and **why it matters**.
|
||||||
|
Fix or dismiss with \`dismissed: <reason>\` (then resolve the thread). Gatekeep has **no** chat / @-command surface.
|
||||||
|
|
||||||
|
${agreedList}
|
||||||
|
|
||||||
|
### Disputed (summary only — not posted inline)
|
||||||
|
One model saw it; others didn't. Useful signal, not a merge blocker by itself.
|
||||||
|
|
||||||
|
${disputedList}
|
||||||
|
|
||||||
|
### Deterministic security scan
|
||||||
|
${(scan.hits || []).length ? (scan.hits || []).map((h) => `- ${h}`).join("\n") : "_No hard-fail hits._"}
|
||||||
|
|
||||||
|
### Policy floor signals
|
||||||
|
${(floor.reasons || []).map((r) => `- ${r}`).join("\n") || "- none"}
|
||||||
|
|
||||||
|
### Still on humans
|
||||||
|
${(merged.human_focus || []).map((h) => `- ${h}`).join("\n") || "- taste, architecture, billing/CI/auth, reading the evidence"}
|
||||||
|
|
||||||
|
---
|
||||||
|
Missed something real? Open a **gatekeep-incident** issue — that's how it becomes an eval.
|
||||||
|
_Inline comments only when models agree. PR tier is fail-closed. Findings lead with severity + location + why._
|
||||||
|
`;
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const comments =
|
||||||
|
(await forgeApi(
|
||||||
|
"GET",
|
||||||
|
`/repos/${owner}/${name}/issues/${pr.number}/comments?limit=100&per_page=100`,
|
||||||
|
)) || [];
|
||||||
|
const list = Array.isArray(comments) ? comments : [];
|
||||||
|
const existing = list.find((c) => (c.body || "").includes(marker));
|
||||||
|
if (existing) {
|
||||||
|
await forgeApi("PATCH", `/repos/${owner}/${name}/issues/comments/${existing.id}`, { body });
|
||||||
|
} else {
|
||||||
|
await forgeApi("POST", `/repos/${owner}/${name}/issues/${pr.number}/comments`, { body });
|
||||||
|
}
|
||||||
|
|
||||||
|
const inline = sortedAgreed
|
||||||
|
.filter((f) => f.path && f.line && f.severity !== "Low" && f.tag !== "Nit")
|
||||||
|
.slice(0, 30)
|
||||||
|
.map((f) => ({
|
||||||
|
path: f.path,
|
||||||
|
// Gitea uses old_position/new_position in some versions; line works on GitHub
|
||||||
|
line: f.line,
|
||||||
|
body: formatInline(f),
|
||||||
|
}));
|
||||||
|
|
||||||
|
if (inline.length) {
|
||||||
|
try {
|
||||||
|
await forgeApi("POST", `/repos/${owner}/${name}/pulls/${pr.number}/reviews`, {
|
||||||
|
commit_id: pr.head.sha,
|
||||||
|
event: "COMMENT",
|
||||||
|
body: "Gatekeep agreed findings (severity · where · why). Fix or dismiss — no @-commands.",
|
||||||
|
comments: inline,
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
console.error("Inline review failed; falling back to issue comments:", e.message);
|
||||||
|
for (const f of inline) {
|
||||||
|
await forgeApi("POST", `/repos/${owner}/${name}/issues/${pr.number}/comments`, {
|
||||||
|
body: f.body + `\n\n_\`${f.path}:${f.line}\`_`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const f of sortedAgreed) {
|
||||||
|
if (
|
||||||
|
(f.severity === "Critical" || f.severity === "High" || f.tag === "Important") &&
|
||||||
|
!f.line &&
|
||||||
|
f.path
|
||||||
|
) {
|
||||||
|
await forgeApi("POST", `/repos/${owner}/${name}/issues/${pr.number}/comments`, {
|
||||||
|
body: formatInline(f) + `\n\n_File-level (no single line) on \`${f.path}\`._`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const want = `risk:${merged.risk_tier}`;
|
||||||
|
for (const lab of ["risk:low", "risk:medium", "risk:high"]) {
|
||||||
|
if (lab !== want) {
|
||||||
|
try {
|
||||||
|
await forgeApi(
|
||||||
|
"DELETE",
|
||||||
|
`/repos/${owner}/${name}/issues/${pr.number}/labels/${encodeURIComponent(lab)}`,
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
/* label may not exist */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
// Gitea expects { labels: ["name"] }; GitHub same for issues API
|
||||||
|
await forgeApi("POST", `/repos/${owner}/${name}/issues/${pr.number}/labels`, {
|
||||||
|
labels: [want],
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
console.error("label:", e.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
const conclusion = merged.open_security_blocking ? "failure" : "success";
|
||||||
|
const statusState = merged.open_security_blocking ? "failure" : "success";
|
||||||
|
const desc = merged.open_security_blocking
|
||||||
|
? "Open security / Critical findings — merge blocked"
|
||||||
|
: `PR risk ${merged.risk_tier} — no open security blockers`;
|
||||||
|
|
||||||
|
// Prefer Checks API (GitHub); fall back to commit status (Gitea-friendly)
|
||||||
|
try {
|
||||||
|
await forgeApi("POST", `/repos/${owner}/${name}/check-runs`, {
|
||||||
|
name: checkName,
|
||||||
|
head_sha: pr.head.sha,
|
||||||
|
status: "completed",
|
||||||
|
conclusion,
|
||||||
|
output: {
|
||||||
|
title: desc,
|
||||||
|
summary: desc,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
try {
|
||||||
|
await forgeApi("POST", `/repos/${owner}/${name}/statuses/${pr.head.sha}`, {
|
||||||
|
state: statusState,
|
||||||
|
context: checkName,
|
||||||
|
description: desc.slice(0, 140),
|
||||||
|
});
|
||||||
|
} catch (e2) {
|
||||||
|
console.error("status/check publish failed:", e2.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (process.env.GITHUB_OUTPUT) {
|
||||||
|
writeFileSync(
|
||||||
|
process.env.GITHUB_OUTPUT,
|
||||||
|
[
|
||||||
|
`risk_tier=${merged.risk_tier}`,
|
||||||
|
`security_blocking=${merged.open_security_blocking}`,
|
||||||
|
`agreed_count=${sortedAgreed.length}`,
|
||||||
|
].join("\n") + "\n",
|
||||||
|
{ flag: "a" },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (merged.open_security_blocking) {
|
||||||
|
console.error("Security blocking — exiting non-zero for job red X");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -0,0 +1,321 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Resolve Gatekeep providers from YAML (or legacy JSON).
|
||||||
|
|
||||||
|
Each provider entry needs:
|
||||||
|
key: FREE-FORM secret/env var name (NO required gatekeep_ prefix)
|
||||||
|
protocol: openai | openai-completions | openai-responses | anthropic
|
||||||
|
Optional:
|
||||||
|
base_url, models (defaults applied for known provider ids)
|
||||||
|
|
||||||
|
Discovers model lists from the provider API; validates requested names; writes
|
||||||
|
provider-plan.json for mixed parallel review.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
PROTOCOLS = ("openai", "openai-completions", "openai-responses", "anthropic")
|
||||||
|
# Wire aliases used by call-model.sh
|
||||||
|
PROTOCOL_WIRE = {
|
||||||
|
"openai": "openai-completions",
|
||||||
|
"openai-completions": "openai-completions",
|
||||||
|
"openai-compatible": "openai-completions", # legacy
|
||||||
|
"openai-responses": "openai-responses",
|
||||||
|
"anthropic": "anthropic",
|
||||||
|
}
|
||||||
|
|
||||||
|
KNOWN_DEFAULTS = {
|
||||||
|
"openrouter": {
|
||||||
|
"base_url": "https://openrouter.ai/api/v1",
|
||||||
|
"protocol": "openai",
|
||||||
|
},
|
||||||
|
"anthropic": {
|
||||||
|
"base_url": "https://api.anthropic.com",
|
||||||
|
"protocol": "anthropic",
|
||||||
|
},
|
||||||
|
"openai": {
|
||||||
|
"base_url": "https://api.openai.com/v1",
|
||||||
|
"protocol": "openai",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def die(msg: str, code: int = 1) -> None:
|
||||||
|
print(f"gatekeep: {msg}", file=sys.stderr)
|
||||||
|
sys.exit(code)
|
||||||
|
|
||||||
|
|
||||||
|
def load_raw(path: Path) -> dict:
|
||||||
|
text = path.read_text()
|
||||||
|
if path.suffix.lower() in (".yaml", ".yml"):
|
||||||
|
try:
|
||||||
|
import yaml # type: ignore
|
||||||
|
except ImportError:
|
||||||
|
die("PyYAML required to load providers.yaml (pip install pyyaml)")
|
||||||
|
raw = yaml.safe_load(text) or {}
|
||||||
|
else:
|
||||||
|
raw = json.loads(text)
|
||||||
|
|
||||||
|
# Legacy models.json → single openrouter provider
|
||||||
|
if "providers" not in raw and "models" in raw:
|
||||||
|
base = raw.get("openrouter_base_url") or KNOWN_DEFAULTS["openrouter"]["base_url"]
|
||||||
|
raw = {
|
||||||
|
"agreement_min": raw.get("agreement_min", 2),
|
||||||
|
"temperature": raw.get("temperature", 0),
|
||||||
|
"max_tokens": raw.get("max_tokens", 4096),
|
||||||
|
"providers": {
|
||||||
|
"openrouter": {
|
||||||
|
"key": "OPENROUTER_API_KEY",
|
||||||
|
"protocol": "openai",
|
||||||
|
"base_url": base,
|
||||||
|
"models": raw["models"],
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Legacy JSON providers without `key`
|
||||||
|
providers = raw.get("providers") or {}
|
||||||
|
for pid, pconf in list(providers.items()):
|
||||||
|
if not isinstance(pconf, dict):
|
||||||
|
continue
|
||||||
|
if "key" not in pconf:
|
||||||
|
# migrate old shape: invent free-form env name suggestion from provider id
|
||||||
|
# (not forcing gatekeep_ prefix)
|
||||||
|
pconf = {**pconf, "key": f"{pid.upper()}_API_KEY"}
|
||||||
|
if pconf.get("protocol") == "openai-compatible":
|
||||||
|
pconf["protocol"] = "openai"
|
||||||
|
providers[pid] = pconf
|
||||||
|
raw["providers"] = providers
|
||||||
|
|
||||||
|
if not providers:
|
||||||
|
die(f"no providers in {path}")
|
||||||
|
return raw
|
||||||
|
|
||||||
|
|
||||||
|
def secret_value(key_name: str) -> str:
|
||||||
|
"""Look up FREE-FORM secret name in the process environment. No prefix rules."""
|
||||||
|
name = (key_name or "").strip()
|
||||||
|
if not name:
|
||||||
|
return ""
|
||||||
|
val = os.environ.get(name, "")
|
||||||
|
if val and val.strip():
|
||||||
|
return val.strip()
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def http_json(url: str, headers: dict) -> dict | list:
|
||||||
|
req = urllib.request.Request(url, headers=headers, method="GET")
|
||||||
|
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||||
|
return json.loads(resp.read().decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_protocol(p: str) -> str:
|
||||||
|
p = (p or "").strip().lower()
|
||||||
|
if p not in PROTOCOL_WIRE:
|
||||||
|
die(f"unknown protocol {p!r} (want {' | '.join(PROTOCOLS)})")
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
def wire_protocol(p: str) -> str:
|
||||||
|
return PROTOCOL_WIRE[normalize_protocol(p)]
|
||||||
|
|
||||||
|
|
||||||
|
def discover_model_ids(base_url: str, protocol: str, api_key: str) -> set[str]:
|
||||||
|
base = base_url.rstrip("/")
|
||||||
|
wire = wire_protocol(protocol)
|
||||||
|
ids: set[str] = set()
|
||||||
|
try:
|
||||||
|
if wire in ("openai-completions", "openai-responses"):
|
||||||
|
data = http_json(
|
||||||
|
f"{base}/models",
|
||||||
|
{"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
for m in data.get("data") or []:
|
||||||
|
mid = m.get("id")
|
||||||
|
if mid:
|
||||||
|
ids.add(mid)
|
||||||
|
elif wire == "anthropic":
|
||||||
|
data = http_json(
|
||||||
|
f"{base}/v1/models",
|
||||||
|
{
|
||||||
|
"x-api-key": api_key,
|
||||||
|
"anthropic-version": "2023-06-01",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
for m in data.get("data") or []:
|
||||||
|
mid = m.get("id")
|
||||||
|
if mid:
|
||||||
|
ids.add(mid)
|
||||||
|
else:
|
||||||
|
die(f"unhandled protocol wire {wire!r}")
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
body = e.read().decode("utf-8", errors="replace")[:400]
|
||||||
|
die(f"model discovery failed for {base}: HTTP {e.code} {body}")
|
||||||
|
except Exception as e:
|
||||||
|
die(f"model discovery failed for {base}: {e}")
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
|
def enrich_provider(pid: str, pconf: dict) -> dict:
|
||||||
|
known = KNOWN_DEFAULTS.get(pid, {})
|
||||||
|
out = {**pconf}
|
||||||
|
if not out.get("base_url"):
|
||||||
|
if known.get("base_url"):
|
||||||
|
out["base_url"] = known["base_url"]
|
||||||
|
else:
|
||||||
|
die(f"provider {pid}: base_url required (no default for this id)")
|
||||||
|
if not out.get("protocol"):
|
||||||
|
out["protocol"] = known.get("protocol") or "openai"
|
||||||
|
out["protocol"] = normalize_protocol(out["protocol"])
|
||||||
|
if not out.get("key"):
|
||||||
|
die(f"provider {pid}: `key` (free-form secret/env name) is required")
|
||||||
|
if not isinstance(out.get("models"), list) or not out["models"]:
|
||||||
|
die(f"provider {pid}: `models` list required (names validated against provider API)")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
if len(sys.argv) < 3:
|
||||||
|
die("usage: resolve-providers.py <providers.yaml|json> <out-plan.json> [override-models-csv]")
|
||||||
|
cfg_path = Path(sys.argv[1])
|
||||||
|
out_path = Path(sys.argv[2])
|
||||||
|
override_csv = sys.argv[3] if len(sys.argv) > 3 else ""
|
||||||
|
|
||||||
|
cfg = load_raw(cfg_path)
|
||||||
|
providers_in = cfg["providers"]
|
||||||
|
providers: dict[str, dict] = {
|
||||||
|
pid: enrich_provider(pid, pconf) for pid, pconf in providers_in.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
override_items: list[tuple[str | None, str]] = []
|
||||||
|
if override_csv.strip():
|
||||||
|
for part in override_csv.split(","):
|
||||||
|
part = part.strip()
|
||||||
|
if not part:
|
||||||
|
continue
|
||||||
|
head, _, rest = part.partition(":")
|
||||||
|
if rest and head in providers:
|
||||||
|
override_items.append((head, rest))
|
||||||
|
else:
|
||||||
|
override_items.append((None, part))
|
||||||
|
|
||||||
|
requested: list[dict] = []
|
||||||
|
if override_items:
|
||||||
|
catalog: dict[str, list[str]] = {}
|
||||||
|
for pid, pconf in providers.items():
|
||||||
|
for m in pconf["models"]:
|
||||||
|
catalog.setdefault(m, []).append(pid)
|
||||||
|
for prov, model in override_items:
|
||||||
|
if prov:
|
||||||
|
requested.append({"provider": prov, "model": model})
|
||||||
|
else:
|
||||||
|
hits = catalog.get(model) or []
|
||||||
|
if len(hits) == 1:
|
||||||
|
requested.append({"provider": hits[0], "model": model})
|
||||||
|
elif len(hits) > 1:
|
||||||
|
die(f"model {model!r} under multiple providers {hits}; use provider:model")
|
||||||
|
elif len(providers) == 1:
|
||||||
|
requested.append({"provider": next(iter(providers)), "model": model})
|
||||||
|
else:
|
||||||
|
die(f"model {model!r} not in config; use provider:model")
|
||||||
|
else:
|
||||||
|
for pid, pconf in providers.items():
|
||||||
|
for m in pconf["models"]:
|
||||||
|
requested.append({"provider": pid, "model": m})
|
||||||
|
|
||||||
|
seen: set[tuple[str, str]] = set()
|
||||||
|
uniq: list[dict] = []
|
||||||
|
for r in requested:
|
||||||
|
key = (r["provider"], r["model"])
|
||||||
|
if key not in seen:
|
||||||
|
seen.add(key)
|
||||||
|
uniq.append(r)
|
||||||
|
requested = uniq
|
||||||
|
|
||||||
|
if len(requested) < 2:
|
||||||
|
die(f"need 2–3 models for parallel review; got {len(requested)}")
|
||||||
|
if len(requested) > 3:
|
||||||
|
print(f"gatekeep: truncating to first 3 of {len(requested)} models", file=sys.stderr)
|
||||||
|
requested = requested[:3]
|
||||||
|
|
||||||
|
provider_meta: dict[str, dict] = {}
|
||||||
|
for pid in {r["provider"] for r in requested}:
|
||||||
|
pconf = providers[pid]
|
||||||
|
key_name = str(pconf["key"])
|
||||||
|
api_key = secret_value(key_name)
|
||||||
|
if not api_key:
|
||||||
|
die(
|
||||||
|
f"missing secret/env for provider {pid!r}: expected env var named {key_name!r} "
|
||||||
|
f"(from providers.*.key — free-form; set it on the workflow step env from secrets)."
|
||||||
|
)
|
||||||
|
available = discover_model_ids(pconf["base_url"], pconf["protocol"], api_key)
|
||||||
|
if not available:
|
||||||
|
die(f"provider {pid}: model list empty — check base_url/protocol/key")
|
||||||
|
provider_meta[pid] = {
|
||||||
|
"base_url": pconf["base_url"].rstrip("/"),
|
||||||
|
"protocol": pconf["protocol"],
|
||||||
|
"wire_protocol": wire_protocol(pconf["protocol"]),
|
||||||
|
"key_name": key_name,
|
||||||
|
"api_key": api_key,
|
||||||
|
"available_count": len(available),
|
||||||
|
"_available": available,
|
||||||
|
}
|
||||||
|
|
||||||
|
plan_models = []
|
||||||
|
for r in requested:
|
||||||
|
pid, model = r["provider"], r["model"]
|
||||||
|
avail = provider_meta[pid]["_available"]
|
||||||
|
if model not in avail:
|
||||||
|
sample = sorted(avail)[:8]
|
||||||
|
die(
|
||||||
|
f"model {model!r} not available on provider {pid} "
|
||||||
|
f"({provider_meta[pid]['available_count']} listed). Examples: {sample}"
|
||||||
|
)
|
||||||
|
plan_models.append(
|
||||||
|
{
|
||||||
|
"provider": pid,
|
||||||
|
"model": model,
|
||||||
|
"base_url": provider_meta[pid]["base_url"],
|
||||||
|
"protocol": provider_meta[pid]["protocol"],
|
||||||
|
"wire_protocol": provider_meta[pid]["wire_protocol"],
|
||||||
|
"key_name": provider_meta[pid]["key_name"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
secrets_path = out_path.with_suffix(".secrets.json")
|
||||||
|
secrets_path.write_text(
|
||||||
|
json.dumps({pid: provider_meta[pid]["api_key"] for pid in provider_meta}, indent=2)
|
||||||
|
)
|
||||||
|
os.chmod(secrets_path, 0o600)
|
||||||
|
|
||||||
|
plan = {
|
||||||
|
"agreement_min": int(cfg.get("agreement_min", 2)),
|
||||||
|
"temperature": cfg.get("temperature", 0),
|
||||||
|
"max_tokens": int(cfg.get("max_tokens", 4096)),
|
||||||
|
"models": plan_models,
|
||||||
|
"secrets_file": str(secrets_path),
|
||||||
|
"providers_used": sorted(provider_meta.keys()),
|
||||||
|
}
|
||||||
|
out_path.write_text(json.dumps(plan, indent=2))
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"providers": plan["providers_used"],
|
||||||
|
"models": [f"{m['provider']}:{m['model']}" for m in plan_models],
|
||||||
|
"keys": {m["provider"]: m["key_name"] for m in plan_models},
|
||||||
|
"agreement_min": plan["agreement_min"],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,186 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Orchestrate parallel multi-provider model calls → merge → publish.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="${GATEKEEP_ROOT:-.}"
|
||||||
|
ACTION="${GATEKEEP_ACTION:-.}"
|
||||||
|
OUT="$ROOT/.gatekeep/out"
|
||||||
|
MODELS_DIR="$OUT/models"
|
||||||
|
mkdir -p "$MODELS_DIR"
|
||||||
|
|
||||||
|
EVENT="${GITHUB_EVENT_PATH:?}"
|
||||||
|
export GATEKEEP_ROOT ACTION CONFIG_DIR OUT
|
||||||
|
|
||||||
|
# shellcheck source=lib-config.sh
|
||||||
|
source "$ACTION/scripts/lib-config.sh"
|
||||||
|
|
||||||
|
if [[ "${SKIP_DRAFT:-true}" == "true" ]]; then
|
||||||
|
DRAFT=$(python3 -c 'import json,sys; print(str(json.load(open(sys.argv[1]))["pull_request"].get("draft", False)).lower())' "$EVENT")
|
||||||
|
if [[ "$DRAFT" == "true" ]]; then
|
||||||
|
echo "Draft PR — skip"
|
||||||
|
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||||
|
echo "risk_tier=skipped" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "security_blocking=false" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "agreed_count=0" >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
REPO="${GITHUB_REPOSITORY:?}"
|
||||||
|
PR=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["pull_request"]["number"])' "$EVENT")
|
||||||
|
|
||||||
|
PROVIDERS_CFG="$(gatekeep_providers_config)"
|
||||||
|
|
||||||
|
PLAN="$OUT/provider-plan.json"
|
||||||
|
python3 "$ACTION/scripts/resolve-providers.py" \
|
||||||
|
"$PROVIDERS_CFG" \
|
||||||
|
"$PLAN" \
|
||||||
|
"${GATEKEEP_MODELS:-}"
|
||||||
|
|
||||||
|
AGREEMENT_MIN=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["agreement_min"])' "$PLAN")
|
||||||
|
export GATEKEEP_MAX_TOKENS
|
||||||
|
GATEKEEP_MAX_TOKENS=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("max_tokens",4096))' "$PLAN")
|
||||||
|
SECRETS_FILE=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["secrets_file"])' "$PLAN")
|
||||||
|
cleanup_secrets() { rm -f "$SECRETS_FILE"; }
|
||||||
|
trap cleanup_secrets EXIT
|
||||||
|
|
||||||
|
# Diff (truncated) — forge-agnostic (.diff URL)
|
||||||
|
: > "$OUT/diff.full"
|
||||||
|
SERVER="${GITHUB_SERVER_URL:-}"
|
||||||
|
TOK="${GITHUB_TOKEN:-${GITEA_TOKEN:-}}"
|
||||||
|
if [[ -n "$SERVER" ]]; then
|
||||||
|
curl -sS -H "Authorization: token ${TOK}" \
|
||||||
|
"${SERVER%/}/${REPO}/pulls/${PR}.diff" -o "$OUT/diff.full" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [[ ! -s "$OUT/diff.full" ]]; then
|
||||||
|
python3 "$ACTION/scripts/forge_api.py" GET "/repos/${REPO}/pulls/${PR}.diff" >"$OUT/diff.full" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
python3 -c 'import pathlib; p=pathlib.Path("'"$OUT"'/diff.full"); t=p.read_text(errors="replace"); pathlib.Path("'"$OUT"'/diff.txt").write_text(t if len(t)<=200000 else t[:200000]+"\n\n… [truncated]")'
|
||||||
|
|
||||||
|
FLOOR="${POLICY_FLOOR:-$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["floor"])' "$OUT/floor.json")}"
|
||||||
|
SCAN_HARD="${SCAN_HARD_FAIL:-false}"
|
||||||
|
export POLICY_FLOOR="$FLOOR" POLICY_REASONS="${POLICY_REASONS:-}"
|
||||||
|
|
||||||
|
python3 <<'PY'
|
||||||
|
import json, os, pathlib
|
||||||
|
out = pathlib.Path(os.environ["OUT"])
|
||||||
|
floor = os.environ.get("POLICY_FLOOR") or json.loads((out / "floor.json").read_text())["floor"]
|
||||||
|
reasons = os.environ.get("POLICY_REASONS", "")
|
||||||
|
event = json.loads(pathlib.Path(os.environ["GITHUB_EVENT_PATH"]).read_text())
|
||||||
|
pr = event["pull_request"]
|
||||||
|
files = [l.strip() for l in (out / "changed-files.txt").read_text().splitlines() if l.strip()]
|
||||||
|
floor_j = json.loads((out / "floor.json").read_text())
|
||||||
|
diff = (out / "diff.txt").read_text() if (out / "diff.txt").exists() else ""
|
||||||
|
discover = json.loads((out / "discover.json").read_text()) if (out / "discover.json").exists() else {}
|
||||||
|
plan = json.loads((out / "provider-plan.json").read_text())
|
||||||
|
|
||||||
|
system = """You are one reviewer in Gatekeep, a multi-model PR review system for the repo owner's own projects.
|
||||||
|
Your job is useful review content that makes human triage cheap — not chat UX.
|
||||||
|
|
||||||
|
Return ONLY valid JSON (no markdown fences) with:
|
||||||
|
{
|
||||||
|
"risk_tier": "low|medium|high",
|
||||||
|
"summary": "short chatty overview",
|
||||||
|
"rationale": "why this PR risk tier",
|
||||||
|
"findings": [{
|
||||||
|
"severity": "Critical|High|Medium|Low",
|
||||||
|
"path": "file path",
|
||||||
|
"line": 1,
|
||||||
|
"line_end": null,
|
||||||
|
"tag": "Important|Nit|Pre-existing",
|
||||||
|
"kind": "security|correctness|behavior|other",
|
||||||
|
"title": "short headline",
|
||||||
|
"why_it_matters": "concrete impact if ignored",
|
||||||
|
"body": "optional extra detail in a friendly tone",
|
||||||
|
"blocking": false
|
||||||
|
}],
|
||||||
|
"human_focus": ["string"]
|
||||||
|
}
|
||||||
|
|
||||||
|
Finding rules (required on EVERY finding):
|
||||||
|
1. severity — Critical/High/Medium/Low so humans see critical vs minor instantly. (Distinct from PR risk_tier.)
|
||||||
|
2. exact location — path + line (use line_end for a range when needed).
|
||||||
|
3. why_it_matters — the impact/reason, not just \"please fix\".
|
||||||
|
Tone: clear and human; structured fields, not robotic checklist-speak.
|
||||||
|
tag is secondary (Important/Nit/Pre-existing); severity dominates.
|
||||||
|
Prefer real harm over style nits. Drop pure style. kind=security or severity=Critical ⇒ blocking true.
|
||||||
|
Pre-existing issues: still include severity + location + why if the PR touches or worsens them.
|
||||||
|
|
||||||
|
PR risk_tier rules:
|
||||||
|
- From the change (paths, blast radius, security, CI), not author trust.
|
||||||
|
- MUST be >= the provided policy_floor.
|
||||||
|
- Security checklist seed: hardcoded secrets/tokens; prompt injection via repo files; auth bypass; unauthorized API access; billing logic bypass; CI workflow tampering; unsafe deserialization; SQL/command injection.
|
||||||
|
- You never approve as a human. Bots never fill human slots.
|
||||||
|
- Do not invent chat commands or ask humans to @-mention you.
|
||||||
|
"""
|
||||||
|
user = {
|
||||||
|
"policy_floor": floor,
|
||||||
|
"policy_floor_reasons": [r.strip() for r in reasons.split("|") if r.strip()],
|
||||||
|
"discover": discover,
|
||||||
|
"reviewers": [f"{m['provider']}:{m['model']}" for m in plan.get("models", [])],
|
||||||
|
"pull_request": {
|
||||||
|
"title": pr.get("title"),
|
||||||
|
"body": pr.get("body") or "",
|
||||||
|
"author": (pr.get("user") or {}).get("login"),
|
||||||
|
"base": (pr.get("base") or {}).get("ref"),
|
||||||
|
"head": (pr.get("head") or {}).get("ref"),
|
||||||
|
"labels": [l.get("name") if isinstance(l, dict) else l for l in (pr.get("labels") or [])],
|
||||||
|
"ci_status": floor_j.get("ci"),
|
||||||
|
"changed_files": files,
|
||||||
|
"additions": floor_j.get("additions"),
|
||||||
|
"deletions": floor_j.get("deletions"),
|
||||||
|
},
|
||||||
|
"diff": diff,
|
||||||
|
}
|
||||||
|
json.dump({"system": system, "user": json.dumps(user, indent=2)}, open(out / "prompt.json", "w"))
|
||||||
|
print(f"Prompt ready; floor={floor}; plan={len(plan.get('models',[]))} models", flush=True)
|
||||||
|
PY
|
||||||
|
|
||||||
|
# Fire models in parallel (may be mixed providers)
|
||||||
|
pids=()
|
||||||
|
i=0
|
||||||
|
while IFS= read -r line; do
|
||||||
|
i=$((i + 1))
|
||||||
|
provider=$(printf '%s' "$line" | python3 -c 'import json,sys; print(json.load(sys.stdin)["provider"])')
|
||||||
|
model=$(printf '%s' "$line" | python3 -c 'import json,sys; print(json.load(sys.stdin)["model"])')
|
||||||
|
protocol=$(printf '%s' "$line" | python3 -c 'import json,sys; print(json.load(sys.stdin)["protocol"])')
|
||||||
|
base_url=$(printf '%s' "$line" | python3 -c 'import json,sys; print(json.load(sys.stdin)["base_url"])')
|
||||||
|
safe=$(echo "${provider}_${model}" | tr '/:' '__')
|
||||||
|
bash "$ACTION/scripts/call-model.sh" \
|
||||||
|
"$provider" "$model" "$protocol" "$base_url" \
|
||||||
|
"$OUT/prompt.json" "$MODELS_DIR/model-$i-$safe.json" "$SECRETS_FILE" &
|
||||||
|
pids+=($!)
|
||||||
|
done < <(python3 -c 'import json,sys; [print(json.dumps(m)) for m in json.load(open(sys.argv[1]))["models"]]' "$PLAN")
|
||||||
|
|
||||||
|
ec=0
|
||||||
|
for pid in "${pids[@]}"; do
|
||||||
|
wait "$pid" || ec=1
|
||||||
|
done
|
||||||
|
echo "Parallel model calls finished (wait_ec=$ec)" >&2
|
||||||
|
|
||||||
|
# Scrub secrets file from workspace artifact path if present
|
||||||
|
rm -f "$SECRETS_FILE"
|
||||||
|
|
||||||
|
node "$ACTION/scripts/merge-findings.js" \
|
||||||
|
--dir "$MODELS_DIR" \
|
||||||
|
--floor "$FLOOR" \
|
||||||
|
--agreement-min "$AGREEMENT_MIN" \
|
||||||
|
--scan-hard-fail "$SCAN_HARD" \
|
||||||
|
--out "$OUT/merged.json" | tee "$OUT/merge-meta.json"
|
||||||
|
|
||||||
|
set +e
|
||||||
|
node "$ACTION/scripts/publish.js" "$OUT/merged.json" "$OUT/floor.json" "$OUT/security-scan.json"
|
||||||
|
pub_ec=$?
|
||||||
|
set -e
|
||||||
|
|
||||||
|
META=$(cat "$OUT/merge-meta.json")
|
||||||
|
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
|
||||||
|
python3 -c 'import json,os,sys
|
||||||
|
m=json.loads(sys.argv[1])
|
||||||
|
open(os.environ["GITHUB_OUTPUT"],"a").write(
|
||||||
|
f"risk_tier={m[\"risk_tier\"]}\nsecurity_blocking={str(m[\"security_blocking\"]).lower()}\nagreed_count={m[\"agreed_count\"]}\n"
|
||||||
|
)' "$META"
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit "$pub_ec"
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Deterministic security checklist seed scanner (paths + diff hunks).
|
||||||
|
# Hits here are hard-fail regardless of model agreement (see policy.security_agreement).
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
FILES_LIST="${1:?changed-files list}"
|
||||||
|
OUT="${2:?out json}"
|
||||||
|
ROOT="${GATEKEEP_ROOT:-.}"
|
||||||
|
REPO="${GITHUB_REPOSITORY:-}"
|
||||||
|
EVENT="${GITHUB_EVENT_PATH:-}"
|
||||||
|
|
||||||
|
hits=()
|
||||||
|
|
||||||
|
while IFS= read -r f; do
|
||||||
|
[[ -z "$f" ]] && continue
|
||||||
|
case "$f" in
|
||||||
|
.github/workflows/*|.github/actions/*)
|
||||||
|
hits+=("CI workflow tampering surface: $f")
|
||||||
|
;;
|
||||||
|
*secret*|*credential*|.env*|.env)
|
||||||
|
hits+=("Secrets/credential path touched: $f")
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done < "$FILES_LIST"
|
||||||
|
|
||||||
|
# Diff scan for high-signal patterns (best-effort; forge-agnostic)
|
||||||
|
if [[ -n "$EVENT" && -f "$EVENT" && -n "$REPO" ]]; then
|
||||||
|
PR=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["pull_request"]["number"])' "$EVENT")
|
||||||
|
ACTION_DIR="${GATEKEEP_ACTION:-$(cd "$(dirname "$0")/.." && pwd)}"
|
||||||
|
TMP=$(mktemp)
|
||||||
|
# Prefer .diff endpoint (works on Gitea + GitHub)
|
||||||
|
SERVER="${GITHUB_SERVER_URL:-}"
|
||||||
|
TOKEN="${GITHUB_TOKEN:-${GITEA_TOKEN:-}}"
|
||||||
|
if [[ -n "$SERVER" ]]; then
|
||||||
|
curl -sS -H "Authorization: token ${TOKEN}" \
|
||||||
|
"${SERVER%/}/${REPO}/pulls/${PR}.diff" -o "$TMP" 2>/dev/null \
|
||||||
|
|| curl -sS -H "Authorization: token ${TOKEN}" \
|
||||||
|
"${GITHUB_API_URL%/}/repos/${REPO}/pulls/${PR}.diff" -o "$TMP" 2>/dev/null \
|
||||||
|
|| true
|
||||||
|
fi
|
||||||
|
if [[ ! -s "$TMP" ]]; then
|
||||||
|
python3 "$ACTION_DIR/scripts/forge_api.py" GET "/repos/${REPO}/pulls/${PR}.diff" >"$TMP" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
DIFF=$(cat "$TMP" 2>/dev/null || true)
|
||||||
|
# Only look at added lines
|
||||||
|
ADDED=$(grep -E '^\+' "$TMP" | grep -vE '^\+\+\+' || true)
|
||||||
|
check() {
|
||||||
|
local name="$1" pat="$2"
|
||||||
|
if printf '%s\n' "$ADDED" | grep -Eiq -- "$pat"; then
|
||||||
|
hits+=("$name")
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
check "Hardcoded secrets/tokens pattern" '(api[_-]?key|secret|token|password|private[_-]?key)\s*[:=]\s*['\''\"]?[A-Za-z0-9_\-]{16,}'
|
||||||
|
check "Prompt injection via repo instruction files" '(ignore (all )?(previous|above) instructions|you are now|system prompt)'
|
||||||
|
check "Possible SQL injection concat" '(execute\(|query\(|raw\().*\+|(SELECT|DROP|UNION).*\+'
|
||||||
|
check "Possible command injection" '(exec\(|spawn\(|os\.system\(|subprocess\.[a-z]+\([^)]*shell\s*=\s*True)'
|
||||||
|
check "Unsafe deserialization" '(pickle\.loads|yaml\.load\(|unserialize\(|ObjectInputStream)'
|
||||||
|
rm -f "$TMP"
|
||||||
|
fi
|
||||||
|
|
||||||
|
HARD=false
|
||||||
|
if ((${#hits[@]} > 0)); then
|
||||||
|
HARD=true
|
||||||
|
fi
|
||||||
|
|
||||||
|
python3 - "$OUT" "$HARD" "${hits[@]}" <<'PY'
|
||||||
|
import json,sys
|
||||||
|
out=sys.argv[1]
|
||||||
|
hard=sys.argv[2]=="true"
|
||||||
|
hits=sys.argv[3:]
|
||||||
|
json.dump({"hard_fail": hard, "hits": hits, "checklist": [
|
||||||
|
"hardcoded secrets/tokens",
|
||||||
|
"prompt injection via repo files",
|
||||||
|
"auth bypass",
|
||||||
|
"unauthorized API access",
|
||||||
|
"billing logic bypass",
|
||||||
|
"CI workflow tampering",
|
||||||
|
"unsafe deserialization",
|
||||||
|
"SQL/command injection"
|
||||||
|
]}, open(out,"w"), indent=2)
|
||||||
|
print(f"security-scan hard_fail={hard} hits={len(hits)}", file=sys.stderr)
|
||||||
|
PY
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Verify layer: discover or run caller-configured benchmarks vs thresholds.
|
||||||
|
# Architecture-level performance debt is out of scope — metrics only.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="${GATEKEEP_ROOT:-.}"
|
||||||
|
ACTION="${GATEKEEP_ACTION:-.}"
|
||||||
|
OUT="$ROOT/.gatekeep/verify"
|
||||||
|
mkdir -p "$OUT"
|
||||||
|
|
||||||
|
# shellcheck source=lib-config.sh
|
||||||
|
source "$ACTION/scripts/lib-config.sh"
|
||||||
|
BENCH_CFG="$(gatekeep_config_file benchmarks.json)"
|
||||||
|
|
||||||
|
ENABLED=$(python3 -c 'import json,sys; print(str(json.load(open(sys.argv[1])).get("enabled", True)).lower())' "$BENCH_CFG")
|
||||||
|
if [[ "$ENABLED" != "true" ]]; then
|
||||||
|
echo "Benchmarks disabled" | tee "$OUT/result.txt"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
python3 - "$BENCH_CFG" "$OUT" "$ROOT" <<'PY'
|
||||||
|
import json, os, pathlib, subprocess, time, re, sys
|
||||||
|
|
||||||
|
cfg = json.load(open(sys.argv[1]))
|
||||||
|
out = pathlib.Path(sys.argv[2])
|
||||||
|
root = pathlib.Path(sys.argv[3])
|
||||||
|
thr = cfg.get("thresholds") or {}
|
||||||
|
cmds = cfg.get("commands") or {}
|
||||||
|
discover = {}
|
||||||
|
dpath = root / ".gatekeep/out/discover.json"
|
||||||
|
if dpath.exists():
|
||||||
|
discover = json.loads(dpath.read_text())
|
||||||
|
|
||||||
|
def pick_command(kind_prefixes):
|
||||||
|
explicit = cmds.get("benchmark") or cmds.get(kind_prefixes[0].split(":")[0]) or ""
|
||||||
|
if explicit:
|
||||||
|
return explicit, "config"
|
||||||
|
for e in discover.get("entries") or []:
|
||||||
|
k = e.get("kind") or ""
|
||||||
|
if any(k.startswith(p) or p in k for p in kind_prefixes):
|
||||||
|
return e.get("command"), e.get("source")
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
results = {"metrics": {}, "thresholds": thr, "ran": [], "skipped": [], "pass": True}
|
||||||
|
|
||||||
|
# Optional explicit startup / memory / fps harnesses — callers wire real collectors.
|
||||||
|
# Default: run discovered bench/test if present; parse common patterns when printed.
|
||||||
|
|
||||||
|
bench_cmd, src = pick_command(["npm:bench", "npm:benchmark", "make:bench", "make:benchmark", "cargo:bench"])
|
||||||
|
test_cmd, tsrc = pick_command(["npm:test", "make:test", "gradle:test", "cargo:test", "go:test", "pytest", "mvn:test"])
|
||||||
|
|
||||||
|
def run(label, cmd):
|
||||||
|
if not cmd:
|
||||||
|
results["skipped"].append(label)
|
||||||
|
return ""
|
||||||
|
results["ran"].append({"label": label, "command": cmd})
|
||||||
|
t0 = time.time()
|
||||||
|
p = subprocess.run(cmd, shell=True, cwd=root, capture_output=True, text=True)
|
||||||
|
dt = int((time.time() - t0) * 1000)
|
||||||
|
(out / f"{label}.stdout.txt").write_text(p.stdout[-50000:])
|
||||||
|
(out / f"{label}.stderr.txt").write_text(p.stderr[-20000:])
|
||||||
|
results["metrics"][f"{label}_exit"] = p.returncode
|
||||||
|
results["metrics"][f"{label}_wall_ms"] = dt
|
||||||
|
if p.returncode != 0:
|
||||||
|
results["pass"] = False
|
||||||
|
return p.stdout + "\n" + p.stderr
|
||||||
|
|
||||||
|
log = ""
|
||||||
|
if bench_cmd:
|
||||||
|
log += run("benchmark", bench_cmd)
|
||||||
|
elif test_cmd:
|
||||||
|
# Fall back: treat test suite wall time as coarse startup/regression signal only when no bench
|
||||||
|
log += run("verify_tests", test_cmd)
|
||||||
|
else:
|
||||||
|
results["skipped"].append("no discoverable benchmark/test command")
|
||||||
|
|
||||||
|
# Parse metrics if present in output
|
||||||
|
def grab(name, patterns):
|
||||||
|
for pat in patterns:
|
||||||
|
m = re.search(pat, log, re.I | re.M)
|
||||||
|
if m:
|
||||||
|
try:
|
||||||
|
results["metrics"][name] = float(m.group(1))
|
||||||
|
return
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
grab("fps", [r"fps[=:\s]+([0-9.]+)", r"frames?/s[=:\s]+([0-9.]+)"])
|
||||||
|
grab("startup_ms", [r"startup[^0-9]*([0-9.]+)\s*ms", r"started in ([0-9.]+)\s*ms"])
|
||||||
|
grab("memory_mb", [r"memory[^0-9]*([0-9.]+)\s*mb", r"rss[=:\s]+([0-9.]+)\s*mb"])
|
||||||
|
|
||||||
|
# Threshold checks (null skips)
|
||||||
|
def check(metric, op, key):
|
||||||
|
lim = thr.get(key)
|
||||||
|
if lim is None:
|
||||||
|
return
|
||||||
|
if metric not in results["metrics"]:
|
||||||
|
results["skipped"].append(f"missing metric for {key}")
|
||||||
|
return
|
||||||
|
val = results["metrics"][metric]
|
||||||
|
ok = val <= lim if op == "max" else val >= lim
|
||||||
|
results["metrics"][f"{key}_ok"] = ok
|
||||||
|
if not ok:
|
||||||
|
results["pass"] = False
|
||||||
|
|
||||||
|
check("startup_ms", "max", "startup_ms_max")
|
||||||
|
check("memory_mb", "max", "memory_mb_max")
|
||||||
|
check("fps", "min", "fps_min")
|
||||||
|
|
||||||
|
(out / "benchmarks.json").write_text(json.dumps(results, indent=2))
|
||||||
|
print(json.dumps({"pass": results["pass"], "metrics": results["metrics"]}, indent=2))
|
||||||
|
if not results["pass"]:
|
||||||
|
sys.exit(1)
|
||||||
|
PY
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
## Summary
|
||||||
|
|
||||||
|
<!-- What and why. Risk is discovered by Gatekeep — do not rubber-stamp "low risk" here. -->
|
||||||
|
|
||||||
|
## Testing notes
|
||||||
|
|
||||||
|
- [ ] Discovered build/test entry points run locally or in CI (paste command / job link)
|
||||||
|
- [ ] Benchmarks vs thresholds (fps / startup / memory) if behavior-sensitive — or N/A
|
||||||
|
- [ ] Client-visible paths: noted if **not** verified on GitHub-hosted runners
|
||||||
|
|
||||||
|
Commands / evidence:
|
||||||
|
|
||||||
|
```
|
||||||
|
<!-- e.g. npm test, make verify, artifact URL from gatekeep-last-good-* -->
|
||||||
|
```
|
||||||
|
|
||||||
|
## Risk (author hint — Gatekeep may raise the PR tier)
|
||||||
|
|
||||||
|
Author self-assessment (optional): low / medium / high
|
||||||
|
|
||||||
|
Sensitive paths touched? (auth, billing, `.github/workflows`, secrets, release): yes / no
|
||||||
|
|
||||||
|
<!-- Gatekeep findings lead with severity (Critical→Low) + exact location + why it matters.
|
||||||
|
No @-mention chat commands — fix or dismiss: dismissed: <reason> -->
|
||||||
|
|
||||||
|
## How to revert
|
||||||
|
|
||||||
|
Rollback speed is the load-bearing wall. Fill this in before merge.
|
||||||
|
|
||||||
|
1. **One-step revert:** `git revert <merge-sha>` (or GitHub "Revert") then merge the revert PR.
|
||||||
|
2. **Artifact / slot rollback:** point `active` at previous known-good artifact (see Gatekeep deploy-slots note); restart.
|
||||||
|
3. **Feature flag / kill switch:** <!-- name → GATEKEEP_MERGE_ENABLED=false also stops AI auto-merge -->
|
||||||
|
4. **Data / world migrations:** <!-- none | forward-only | script path -->
|
||||||
|
5. **Owner if this causes an incident:** <!-- @owner → file gatekeep-incident -->
|
||||||
|
|
||||||
|
If this change is hard to revert, treat it as **high** risk and get the required human approvals.
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
name: Gatekeep incident
|
||||||
|
description: Record a review/merge miss so it becomes a fail-closed eval case
|
||||||
|
title: "[gatekeep-incident] "
|
||||||
|
labels: ["gatekeep-incident"]
|
||||||
|
body:
|
||||||
|
- type: markdown
|
||||||
|
attributes:
|
||||||
|
value: |
|
||||||
|
Service restore first. Then file this so Gatekeep learns (incident → eval).
|
||||||
|
Set `GATEKEEP_MERGE_ENABLED=false` while firefighting if auto-merge was involved.
|
||||||
|
- type: input
|
||||||
|
id: pr_url
|
||||||
|
attributes:
|
||||||
|
label: PR URL
|
||||||
|
placeholder: https://github.com/org/repo/pull/123
|
||||||
|
validations:
|
||||||
|
required: true
|
||||||
|
- type: dropdown
|
||||||
|
id: miss_type
|
||||||
|
attributes:
|
||||||
|
label: What did Gatekeep get wrong?
|
||||||
|
options:
|
||||||
|
- Missed finding (security / Important not raised or not agreed)
|
||||||
|
- Wrong tier (under-tiered)
|
||||||
|
- Verify/benchmark miss
|
||||||
|
- Auto-merge latch error
|
||||||
|
- Other
|
||||||
|
validations:
|
||||||
|
required: true
|
||||||
|
- type: dropdown
|
||||||
|
id: ai_tier
|
||||||
|
attributes:
|
||||||
|
label: Tier at merge
|
||||||
|
options: [low, medium, high, unknown]
|
||||||
|
validations:
|
||||||
|
required: true
|
||||||
|
- type: dropdown
|
||||||
|
id: should_tier
|
||||||
|
attributes:
|
||||||
|
label: Should have been tier
|
||||||
|
options: [low, medium, high]
|
||||||
|
validations:
|
||||||
|
required: true
|
||||||
|
- type: textarea
|
||||||
|
id: failure
|
||||||
|
attributes:
|
||||||
|
label: Actual failure / incident summary
|
||||||
|
description: What broke, how you detected it, revert minutes if known.
|
||||||
|
validations:
|
||||||
|
required: true
|
||||||
|
- type: textarea
|
||||||
|
id: review_snapshot
|
||||||
|
attributes:
|
||||||
|
label: Gatekeep summary snapshot
|
||||||
|
description: Paste the Gatekeep review comment (agreed/disputed/models) at the time.
|
||||||
|
validations:
|
||||||
|
required: false
|
||||||
|
- type: input
|
||||||
|
id: revert_pr
|
||||||
|
attributes:
|
||||||
|
label: Revert PR / rollback command
|
||||||
|
validations:
|
||||||
|
required: false
|
||||||
|
- type: textarea
|
||||||
|
id: proposed_eval
|
||||||
|
attributes:
|
||||||
|
label: Proposed eval case / policy change
|
||||||
|
description: Paths, truncated diff, expect_min_tier, checklist item to pin.
|
||||||
|
validations:
|
||||||
|
required: false
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
# SAMPLE for the Action repo or consumers — prefer .github/workflows/gatekeep-eval.yml on Gitea.
|
||||||
|
# Live dogfood eval workflow: .github/workflows/gatekeep-eval.yml
|
||||||
|
|
||||||
|
name: Gatekeep evals
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
eval-floor:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09
|
||||||
|
- run: bash scripts/eval-floor.sh evals/cases defaults/policy.json
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# SAMPLE for *other* consumer repos — copy to .github/workflows/gatekeep-pr.yml
|
||||||
|
#
|
||||||
|
# On the Gatekeep dogfood repo itself, the live workflow is:
|
||||||
|
# .github/workflows/gatekeep-pr.yml (uses: ./)
|
||||||
|
#
|
||||||
|
# Secrets: free-form names from providers.yaml `key` (e.g. OPENROUTER_API_KEY)
|
||||||
|
|
||||||
|
name: Gatekeep
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
types: [opened, synchronize, reopened, ready_for_review, labeled]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
gatekeep:
|
||||||
|
name: gatekeep
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: github.event.pull_request.draft == false
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
|
||||||
|
- name: Gatekeep review + verify
|
||||||
|
id: gk
|
||||||
|
# Consumer remote reference (not used for dogfood — dogfood uses ./ )
|
||||||
|
uses: https://git.potatolab.uk/layue13/gatekeep@main
|
||||||
|
env:
|
||||||
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
run-verify: "true"
|
||||||
|
retain-artifacts: "true"
|
||||||
Reference in new issue
Block a user