diff options
Diffstat (limited to 'pkgs/vet/vet.sh')
| -rw-r--r-- | pkgs/vet/vet.sh | 460 |
1 files changed, 460 insertions, 0 deletions
diff --git a/pkgs/vet/vet.sh b/pkgs/vet/vet.sh new file mode 100644 index 0000000..04d9345 --- /dev/null +++ b/pkgs/vet/vet.sh @@ -0,0 +1,460 @@ +usage() { + cat <<'EOF' +vet holds flake inputs and agent skills back by a cooldown, and has Claude +review what changed in third-party sources before they are installed. + +Usage: + vet lock + Move flake inputs and skills to the newest revisions that are past the + cooldown. Own sources move to their newest revision. + vet check [--base REV] [--report FILE] + Compare flake.lock and skills.lock with REV (default HEAD). Changes + inside the cooldown are blocked; third-party changes are reviewed by + Claude. Exits 1 when blocked, 2 when a review needs a human and stdin + is not a terminal. + vet cves OLD NEW [--report FILE] + List the CVEs that switching from system OLD to NEW adds and fixes, + matched by vulnix against the NVD. Known false positives go in + vulnix.toml at the root of the repository. Only reports, never blocks. + +Sources are sorted into own, trusted and third-party in vet.json at the root +of the repository. VET_COOLDOWN_DAYS and VET_MODEL override the defaults. +EOF +} + +log() { echo "vet: $*" >&2; } +die() { + log "$*" + exit 1 +} + +# Lower-cases a URL and strips scheme, user and .git, so that +# ssh://git@host/repo, git@host:repo and https://host/repo.git all compare equal. +norm() { + local u=${1,,} + u=${u#*://} + u=${u#*@} + u=${u%.git} + u=${u/://} + printf '%s\n' "$u" +} + +tier() { + local p + for p in "${own[@]}"; do + if [[ $1 == "$p"* ]]; then + echo own + return + fi + done + for p in "${trusted[@]}"; do + if [[ $1 == "$p" ]]; then + echo trusted + return + fi + done + echo third-party +} + +# Prints {name, node} for each direct input of a flake.lock. +inputs() { + jq -c '.nodes as $n | .nodes.root.inputs | to_entries[] + | select(.value | type == "string") | {name: .key, node: $n[.value]}' "$1" +} + +base_node() { + jq -c --arg n "$1" '.nodes.root.inputs[$n] as $k + | if ($k | type) == "string" then .nodes[$k] else null end' "$tmp/base-flake.lock" +} + +node_source() { + norm "$(jq -r '.original | if .type == "github" then "github.com/\(.owner)/\(.repo)" else .url // "" end' <<<"$1")" +} + +gh_api() { + local auth=() + if [[ -n $token ]]; then auth=(-H "Authorization: Bearer $token"); fi + curl -fsSL "${auth[@]}" -H 'Accept: application/vnd.github+json' "https://api.github.com/$1" +} + +short() { + if [[ -n $1 ]]; then echo "${1:0:12}"; else echo new; fi +} + +iso() { date -u -d "@$1" +%Y-%m-%dT%H:%M:%SZ; } + +# Prints "<rev> <commit date>" for the newest commit on ref that is older than +# the cutoff. +github_rev() { + local q until + until=$(iso "$cutoff") + q="repos/$1/$2/commits?per_page=1&until=$until" + if [[ -n $3 ]]; then q+="&sha=$3"; fi + gh_api "$q" | jq -r '.[0] // empty | "\(.sha) \(.commit.committer.date)"' +} + +# The newest nixpkgs-unstable channel release older than the cutoff. Channel +# releases are commits Hydra has built, so they are in the binary cache; an +# arbitrary older commit on the branch might not be. +nixpkgs_channel_rev() { + local current key + current=$(curl -fsSI https://channels.nixos.org/nixpkgs-unstable | tr -d '\r' | sed -n 's|^location: .*/||Ip') + [[ -n $current ]] || return 1 + key=$(curl -fsSL "https://nix-releases.s3.amazonaws.com/?delimiter=/&prefix=nixpkgs/${current%%pre*}pre" | + grep -o '<Key>[^<]*</Key><LastModified>[^<]*' | + sed 's|<Key>||; s|</Key><LastModified>| |' | + awk -v c="$(iso "$cutoff")" '$2 < c { print $2, $1 }' | sort | tail -n 1 | cut -d ' ' -f 2) + [[ -n $key ]] || return 1 + curl -fsSL "https://releases.nixos.org/$key/git-revision" +} + +# Fetches revs from url into a cached bare repository and prints its path. +fetch() { + local url=$1 d rev + shift + d=$cache/repos/$(printf '%s' "$url" | sha256sum | cut -c 1-16) + [[ -d $d ]] || git init -q --bare "$d" + for rev; do + git -C "$d" cat-file -e "$rev^{commit}" 2>/dev/null && continue + git -C "$d" fetch -q "$url" "$rev" 2>/dev/null || + git -C "$d" fetch -q "$url" '+refs/heads/*:refs/heads/*' '+refs/tags/*:refs/tags/*' + git -C "$d" cat-file -e "$rev^{commit}" 2>/dev/null || die "$url has no commit $rev" + done + printf '%s\n' "$d" +} + +load_base() { + git show "$1:flake.lock" >"$tmp/base-flake.lock" 2>/dev/null || + echo '{"nodes":{"root":{"inputs":{}}}}' >"$tmp/base-flake.lock" + git show "$1:skills.lock" >"$tmp/base-skills.lock" 2>/dev/null || + echo '{"sources":{}}' >"$tmp/base-skills.lock" +} + +cmd_lock() { + local overrides=() updates=() line name node src owner repo ref rev date base base_mod + load_base HEAD + + while IFS= read -r line; do + name=$(jq -r .name <<<"$line") + node=$(jq -c .node <<<"$line") + if [[ -n $(jq -r '.original.rev // empty' <<<"$node") ]]; then + log "$name: pinned to a commit in flake.nix, left alone" + continue + fi + src=$(node_source "$node") + if [[ $(tier "$src") == own ]]; then + updates+=("$name") + continue + fi + if [[ $(jq -r .original.type <<<"$node") != github ]]; then + log "$name: only GitHub inputs can be held back, left alone" + continue + fi + owner=$(jq -r .original.owner <<<"$node") + repo=$(jq -r .original.repo <<<"$node") + ref=$(jq -r '.original.ref // empty' <<<"$node") + + if [[ $src == github.com/nixos/nixpkgs && $ref == nixpkgs-unstable ]]; then + if ! rev=$(nixpkgs_channel_rev); then + log "$name: no channel release is $days days old yet, left alone" + continue + fi + date=$(gh_api "repos/$owner/$repo/commits/$rev" | jq -r .commit.committer.date) + else + read -r rev date < <(github_rev "$owner" "$repo" "$ref") || true + if [[ -z $rev ]]; then + log "$name: no commit is $days days old yet, left alone" + continue + fi + fi + + # Never move behind what HEAD already locks. + base=$(base_node "$name") + base_mod=$(jq -r '.locked.lastModified // 0' <<<"$base") + if (($(date -d "$date" +%s) < base_mod)); then + rev=$(jq -r .locked.rev <<<"$base") + date=$(iso "$base_mod") + fi + overrides+=(--override-input "$name" "github:$owner/$repo/$rev") + log "$name: ${rev:0:12} ($date)" + done < <(inputs flake.lock) + + if ((${#updates[@]})); then + log "own inputs to newest: ${updates[*]}" + nix flake update "${updates[@]}" + fi + if ((${#overrides[@]})); then nix flake lock "${overrides[@]}"; fi + + [[ -f skills.lock ]] || return 0 + SKILLS_MANAGED_BY=home-manager SKILLS_MANIFEST=$repo/skills.json skills update -q + + local url d committed entry rows row + mapfile -t rows < <(jq -r '.sources | to_entries[] | [.key, .value.url, .value.rev] | @tsv' skills.lock) + for row in "${rows[@]}"; do + IFS=$'\t' read -r name url rev <<<"$row" + [[ $(tier "$(norm "$url")") == own ]] && continue + entry=$(jq -c --arg n "$name" '.sources[$n] // empty' "$tmp/base-skills.lock") + [[ -n $entry && $(jq -r .rev <<<"$entry") == "$rev" ]] && continue + d=$(fetch "$url" "$rev") + committed=$(git -C "$d" show -s --format=%ct "$rev") + ((committed <= cutoff)) && continue + if [[ -z $entry ]]; then + log "skill source $name is new and inside the cooldown; vet check will block it" + continue + fi + log "skill source $name: ${rev:0:12} is inside the cooldown, kept at $(jq -r '.rev[:12]' <<<"$entry")" + jq --arg n "$name" --argjson e "$entry" '.sources[$n] = $e' skills.lock >"$tmp/skills.lock" + cat "$tmp/skills.lock" >skills.lock + done +} + +worst=pass +report=() + +record() { + case $1 in + block) worst=block ;; + review) [[ $worst == block ]] || worst=review ;; + esac + report+=("$2") +} + +schema='{ + "type": "object", + "properties": { + "verdict": {"type": "string", "enum": ["pass", "review", "block"]}, + "summary": {"type": "string", "description": "One sentence on what changed and why it is or is not a concern."}, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "severity": {"type": "string", "enum": ["low", "medium", "high"]}, + "file": {"type": "string"}, + "description": {"type": "string"} + }, + "required": ["severity", "description"] + } + } + }, + "required": ["verdict", "summary", "findings"] +}' + +# review <name> <kind> <url> <old rev or ""> <new rev> [paths...] +review() { + local name=$1 kind=$2 url=$3 old=$4 new=$5 d from in size truncated=false out result v + shift 5 + if [[ -n $old ]]; then d=$(fetch "$url" "$old" "$new"); else d=$(fetch "$url" "$new"); fi + from=${old:-$(git -C "$d" hash-object -t tree /dev/null)} + in=$tmp/review-$name.txt + + { + echo "Dependency: $name ($kind)" + echo "Source: $url" + if [[ -n $old ]]; then echo "Change: $old..$new"; else echo "Change: new dependency, full contents at $new"; fi + if (($#)); then echo "Paths that get installed: $*"; fi + echo + echo "## Commits" + if [[ -n $old ]]; then + git -C "$d" log -n 200 --format='%h %an <%ae> %cI %s' "$old..$new" -- "$@" + else + git -C "$d" log -n 20 --format='%h %an <%ae> %cI %s' "$new" -- "$@" + fi + echo + echo "## Diffstat" + git -C "$d" diff --stat "$from" "$new" -- "$@" + echo + echo "## Diff" + } >"$in" + git -C "$d" diff --no-color --no-ext-diff "$from" "$new" -- "$@" >"$tmp/diff" + size=$(wc -c <"$tmp/diff") + if ((size > max_bytes)); then + truncated=true + head -c "$max_bytes" "$tmp/diff" >>"$in" + echo "[diff truncated at $max_bytes of $size bytes]" >>"$in" + else + cat "$tmp/diff" >>"$in" + fi + + log "reviewing $name with Claude ($((size / 1024)) KiB of diff)" + # Run from an empty directory without tools, MCP servers, settings or + # slash commands: the diff is attacker-controlled input. + if ! out=$(cd "$tmp/empty" && claude -p --model "$model" \ + --system-prompt "$(cat "$VET_PROMPT")" \ + --tools "" --strict-mcp-config --setting-sources "" --disable-slash-commands \ + --no-session-persistence --output-format json --json-schema "$schema" \ + "Review the dependency change on stdin." <"$in"); then + record review "- $name: Claude review failed, review by hand" + return + fi + result=$(jq -c '.structured_output // empty' <<<"$out") + if [[ -z $result ]]; then + record review "- $name: Claude gave no verdict, review by hand" + return + fi + v=$(jq -r .verdict <<<"$result") + if [[ $truncated == true && $v == pass ]]; then v=review; fi + record "$v" "$(jq -r --arg n "$name" --arg v "$v" --argjson t "$truncated" \ + '"- \($n): \($v)\(if $t then " (diff truncated)" else "" end): \(.summary)" + + ([.findings[] | "\n - [\(.severity)] \(if .file then .file + ": " else "" end)\(.description)"] | join(""))' \ + <<<"$result")" +} + +cooldown() { # name lastModified + if (($2 > cutoff)); then + record block "- $1: blocked, committed $(((now - $2) / 3600)) hours ago (cooldown is $days days)" + return 1 + fi +} + +cmd_check() { + local base=HEAD out="" line name node old_node src t new_id old_id old new kind url + while (($#)); do + case $1 in + --base) base=$2; shift 2 ;; + --report) out=$2; shift 2 ;; + *) usage >&2; exit 64 ;; + esac + done + load_base "$base" + + while IFS= read -r line; do + name=$(jq -r .name <<<"$line") + node=$(jq -c .node <<<"$line") + old_node=$(base_node "$name") + new_id=$(jq -r '.locked | .rev // .narHash' <<<"$node") + old_id=$(jq -r '.locked | .rev // .narHash // empty' <<<"$old_node") + [[ $new_id == "$old_id" ]] && continue + + src=$(node_source "$node") + t=$(tier "$src") + new=$(jq -r '.locked.rev // empty' <<<"$node") + old=$(jq -r '.locked.rev // empty' <<<"$old_node") + if [[ $t == own ]]; then + report+=("- $name: own source, $(short "$old") -> $(short "$new")") + continue + fi + cooldown "$name" "$(jq -r '.locked.lastModified // 0' <<<"$node")" || continue + if [[ $t == trusted ]]; then + report+=("- $name: trusted, $(short "$old") -> $(short "$new")") + continue + fi + [[ -n $new ]] || { + record review "- $name: no git revision to review, review by hand" + continue + } + case $src in + github.com/*/homebrew-*) kind="homebrew tap" ;; + *) kind="nix flake input" ;; + esac + url=$(jq -r '.locked | if .type == "github" then "https://github.com/\(.owner)/\(.repo)" else .url end' <<<"$node") + review "$name" "$kind" "$url" "$old" "$new" + done < <(inputs flake.lock) + + if [[ -f skills.lock ]]; then + local paths=() d + while IFS= read -r line; do + name=$(jq -r .key <<<"$line") + url=$(jq -r .value.url <<<"$line") + new=$(jq -r .value.rev <<<"$line") + old=$(jq -r --arg n "$name" '.sources[$n].rev // empty' "$tmp/base-skills.lock") + [[ $new == "$old" ]] && continue + if [[ $(tier "$(norm "$url")") == own ]]; then + report+=("- skills/$name: own source, $(short "$old") -> $(short "$new")") + continue + fi + d=$(fetch "$url" "$new") + cooldown "skills/$name" "$(git -C "$d" show -s --format=%ct "$new")" || continue + mapfile -t paths < <(jq -r '.value.skills[]' <<<"$line") + review "skills/$name" "agent skill" "$url" "$old" "$new" "${paths[@]}" + done < <(jq -c '.sources | to_entries[]' skills.lock) + fi + + if ((${#report[@]} == 0)); then + log "nothing changed since $base" + : >"${out:-/dev/null}" + return 0 + fi + printf '%s\n' "${report[@]}" | tee "${out:-/dev/null}" + + case $worst in + block) + log "blocked" + exit 1 + ;; + review) + [[ -t 0 ]] || exit 2 + local answer + read -r -p "vet: accept these changes? [y/N] " answer + [[ $answer == [yY]* ]] || exit 1 + ;; + esac +} + +# vulnix exits 2 when it finds vulnerabilities, which is not a failure here. +vulnix_scan() { + local args=(-C --json) + if [[ -f $repo/vulnix.toml ]]; then args+=(-w "$repo/vulnix.toml"); fi + vulnix "${args[@]}" "$1" || (($? == 2)) +} + +cmd_cves() { + local old=$1 new=$2 out=/dev/null + shift 2 + if [[ ${1:-} == --report ]]; then out=$2; fi + log "scanning $old and $new with vulnix" + vulnix_scan "$old" >"$tmp/old.json" + vulnix_scan "$new" >"$tmp/new.json" + jq -nr --slurpfile old "$tmp/old.json" --slurpfile new "$tmp/new.json" ' + def cves: [.[] | .pname as $p | .version as $v | .cvssv3_basescore as $s + | .affected_by[] | {key: "\($p) \(.)", pkg: "\($p) \($v)", cve: ., score: ($s[.] // null)}]; + ($old[0] | cves | map(.key)) as $before + | ($new[0] | cves) as $after + | ($after | map(.key)) as $now + | ($after | map(select(.key as $k | $before | index($k) | not)) + | sort_by(-(.score // 0))) as $added + | ($before | map(select(. as $k | $now | index($k) | not))) as $fixed + | "CVEs added: \($added | length), fixed: \($fixed | length), open: \($now | length)", + ($added[] | "- \(.pkg): \(.cve)\(if .score then " (CVSS \(.score))" else "" end)") + ' | tee "$out" +} + +repo=$(git rev-parse --show-toplevel) +cd "$repo" +[[ -f vet.json ]] || die "no vet.json in $repo" + +cache=${XDG_CACHE_HOME:-$HOME/.cache}/vet +mkdir -p "$cache/repos" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +mkdir "$tmp/empty" + +days=${VET_COOLDOWN_DAYS:-$(jq -r .cooldownDays vet.json)} +model=${VET_MODEL:-opus} +max_bytes=400000 +now=$(date +%s) +cutoff=$((now - days * 86400)) +mapfile -t own < <(jq -r '.own[] | ascii_downcase' vet.json) +mapfile -t trusted < <(jq -r '.trusted[] | ascii_downcase' vet.json) +token=${GITHUB_TOKEN:-$(gh auth token 2>/dev/null || true)} + +case ${1:-} in + lock) cmd_lock ;; + cves) + shift + (($# >= 2)) || { + usage >&2 + exit 64 + } + cmd_cves "$@" + ;; + check) + shift + cmd_check "$@" + ;; + -h | --help | help) usage ;; + *) + usage >&2 + exit 64 + ;; +esac |