tickmarkr 2.8.0 → 2.8.2

package/skills/tickmarkr-overseer/scripts/grade-ci.sh

@@ -1,45 +1,59 @@
 #!/bin/bash
-# grade-ci.sh <run-id> <expected-count> [tag] — grade EVERY job of a public CI run's closed inventory from
-# its JOB LOG. Grade only at run-end. A missing/in-progress/empty log is UNREADABLE, never evidence of green.
-# Exit 0 GREEN, 1 RED, 2 UNREADABLE, 3 INFRA_KILLED: a job the GitHub runner shut down or cancelled with
-# no failed or timed-out test (the classifier's class, D-897) — never green, never a pass, only unproven.
+# grade-ci.sh <run-id> <expected-sha> <repo-dir> [tag] — grade EVERY job of a public CI run of <expected-sha>
+# from GitHub's structured run record and each job's uploaded proof group (v2.8.2 T3, ANL-022 §7.2b; queue rows
+# 165 and 148). NO LOG TEXT IS READ: no job log, no step label, no `##[error]` annotation decides anything.
+# Grade only at run-end. Exit 0 GREEN, 1 RED, 2 UNREADABLE, 3 INFRA_KILLED: a cancelled run, job or declared
+# step with no failed declared step — never green, never a pass, only unproven.
 # UNREADABLE dominates a mixed result, then RED: a failure anywhere dominates a kill.
 #
+# INPUT: <expected-sha> is 40 lowercase hex naming a commit in <repo-dir> — the mirror clone (RELEASING.md step
+# 4) or the Release workflow's own checkout — else exit 2 before any gh call. No count argument: the manifest of
+# tracked test files is recomputed from that commit.
+# ONE READ: `gh run view <run-id> --json headSha,status,conclusion,jobs`. IDENTITY: UNREADABLE, before any
+# download, unless the run's headSha is <expected-sha> and its status is completed.
+#
 # Two CLOSED inventories (v2.7.8 T4; the workflow renders these names): serial {test, test-macos}; sharded,
 # for os in ubuntu-latest and macos-latest, `suite <os> k/4` for k 1..4, `single-fork <os>`, `aggregate <os>`.
 # A run holding any sharded name is graded against the sharded inventory, else the serial one. The first
 # line is `INVENTORY form=<serial|sharded> declared=<n> observed=<n>`; each missing, duplicated or unknown
 # name prints `INVENTORY_DEFECT kind=<missing|duplicate|unknown> job=<json-quoted name>` and makes the run
-# UNREADABLE: a job the inventory does not name is never skipped.
+# UNREADABLE: a job the inventory does not name is never skipped. The sharded proof-group selection is ANL-022
+# §7.4's, so until it ships a sharded run reads UNREADABLE after its INVENTORY line.
 #
-# A verdict line is read only from the log lines whose gh step column equals its declared step, exactly
-# once, else UNREADABLE — a line a test or any other step prints, before or after the real one, never decides:
-#   COUNT_ORACLE <GREEN|RED> expected=<n|UNREADABLE> actual=<n|UNREADABLE>   step `Assert collected test-file count`
-#   COVERAGE_FLOORS verdict=… floors=… failing=…  (T1's final floors line)   step `Recheck coverage floors`
-#   SET_ORACLE verdict=… expected=… actual=… defects=…  (T3's set oracle)     step `Assert collected test-file set`
-# ROLES — the shared classifier (classify-vitest-log.sh) stays the one log verdict:
-#   serial       the base rule (count oracle GREEN at the expected count, the classifier given that count
-#                reading CLEAN or RPC_ONLY, passed+skipped equal to it) AND exactly one COVERAGE_FLOORS
-#                verdict=GREEN, so a dropped floors step reads UNREADABLE.
-#   shard, single-fork
-#                GREEN only on conclusion success whose whole log the classifier, given no count, reads CLEAN
-#                or RPC_ONLY: their wrapper already applied the shard contract in-job.
-#   aggregate    GREEN only on conclusion success with exactly one COVERAGE_FLOORS verdict=GREEN, exactly one
-#                `SET_ORACLE verdict=GREEN expected=N actual=N defects=0`, and the lines of its step
-#                `Merge coverage (one threshold check per OS)` classified CLEAN or RPC_ONLY.
-# Exit annotations: SERIAL keeps the one exception run-ci-vitest.sh grants (a failure explained ONLY by steps
-# that exited exactly 1 on a log of their own the classifier reads RPC_ONLY); any other failing annotation is
-# never GREEN. Shard, single-fork and aggregate admit none — their wrapper applied the exception in-job.
-# INFRA_KILLED is the base rule for every role.
+# Per serial job, every part from the same read or the job's own artifact:
+#   steps        each DECLARED step below present exactly once in .jobs[].steps[] (missing or duplicated ->
+#                UNREADABLE); GREEN needs every one `success`. A cancelled job or declared step is INFRA_KILLED
+#                unless a declared step concluded `failure` (then RED); the steps a kill skipped do not count.
+#   conclusions  GREEN also needs the job's and the run's conclusion `success`, so a failing undeclared step (an
+#                action's post-step) reds the grade; a cancelled run is INFRA_KILLED, and any other run conclusion
+#                no job's grade accounts for is RED.
+#   proof        `gh run download <run-id> -n proof-<job>` into a fresh directory (missing -> UNREADABLE), then
+#                the grader's own checkout's `tsx scripts/ci/cli.ts proof --head <expected-sha> --repo <repo-dir>
+#                <dir>` (that checkout is the git toplevel holding this script; without its cli.ts, exit 2):
+#                the group's original records read CLEAN or RPC_ONLY at <expected-sha>, the tracked set recomputed
+#                from the commit holds every file once and finished, and the floors sidecar reads GREEN.
+# Output: the INVENTORY lines, `RUN head=… expected=… status=… conclusion=…`, per job `<job>: steps=[…]
+# proof=[<PROOF line>]` then `<job>: <GRADE>`, and a final `VERDICT rc=<n> …`.
 set -u
 
-run=${1:?run id required}
-expected=${2:?expected tracked test-file count required}
-tag=${3:-$run}
+[ "$#" -ge 3 ] && [ "$#" -le 4 ] \
+  || { echo "UNREADABLE: usage: grade-ci.sh <run-id> <expected-sha> <repo-dir> [tag]"; exit 2; }
+run=$1
+sha=$2
+repo_dir=$3
+tag=${4:-$run}
 here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-repo=${TKR_GRADE_CI_REPO:-alzahrani-khalid/tickmarkr}
+# the grader's own checkout — the same for the canonical copy and the installed .claude/skills one
+checkout=$(git -C "$here" rev-parse --show-toplevel 2>/dev/null)
+gh_repo=${TKR_GRADE_CI_REPO:-alzahrani-khalid/tickmarkr}
 out_dir=${TKR_GRADE_CI_DIR:-${TKR_STATE_DIR:-.tickmarkr}/overseer/diag}
-mkdir -p "$out_dir" || { echo "UNREADABLE: cannot create log directory $out_dir"; exit 2; }
+
+[[ $sha =~ ^[0-9a-f]{40}$ ]] || { echo "UNREADABLE: the expected head is not 40 lowercase hex"; exit 2; }
+git -C "$repo_dir" cat-file -e "$sha^{commit}" 2>/dev/null \
+  || { echo "UNREADABLE: $sha is not a commit in $repo_dir"; exit 2; }
+[ -n "$checkout" ] && [ -f "$checkout/scripts/ci/cli.ts" ] \
+  || { echo "UNREADABLE: the grader's own checkout holds no scripts/ci/cli.ts to recompute the proof"; exit 2; }
+mkdir -p "$out_dir" || { echo "UNREADABLE: cannot create the proof directory $out_dir"; exit 2; }
 
 serial_inventory=$'test\ntest-macos'
 sharded_inventory=$(for os in ubuntu-latest macos-latest; do
@@ -47,21 +61,24 @@ sharded_inventory=$(for os in ubuntu-latest macos-latest; do
   echo "single-fork $os"
   echo "aggregate $os"
 done)
-COUNT_STEP="Assert collected test-file count"
-FLOORS_STEP="Recheck coverage floors"
-SET_STEP="Assert collected test-file set"
-MERGE_STEP="Merge coverage (one threshold check per OS)"
-COUNT_FORMAT='COUNT_ORACLE (GREEN|RED) expected=([0-9]+|UNREADABLE) actual=([0-9]+|UNREADABLE)'
-FLOORS_FORMAT='COVERAGE_FLOORS verdict=(GREEN|RED|UNREADABLE) floors=([0-9]+|UNREADABLE) failing=([0-9]+|UNREADABLE)'
-SET_FORMAT='SET_ORACLE verdict=(GREEN|RED|UNREADABLE) expected=([0-9]+|UNREADABLE) actual=([0-9]+|UNREADABLE) defects=[0-9]+'
+# the serial job's declared steps as GitHub names them (an unnamed `run:` step is `Run <command>`)
+serial_steps='Run npm ci
+Run npm run build
+Run npm run lint
+Run coverage proof (parallel suite project)
+Recheck coverage floors
+Run single-fork projects (independent of the step above)
+Assert collected test-file set
+Upload proof group'
+cli=("$checkout/node_modules/.bin/tsx" "$checkout/scripts/ci/cli.ts")
 
 verdict=0
 mark_unreadable() { verdict=2; }
 mark_red() { [ "$verdict" -eq 2 ] || verdict=1; }
 mark_killed() { [ "$verdict" -ne 0 ] || verdict=3; }
-field() { printf '%s\n' "$classified" | sed -n "s/^VITEST_LOG.* $1=\([^ ]*\).*/\1/p"; }
+finish() { echo "VERDICT rc=$verdict (0=all GREEN 1=RED 2=UNREADABLE 3=INFRA_KILLED)"; exit "$verdict"; }
 count() { printf '%s' "$1" | grep -c ''; }
-# json <name> — one JSON string; awk -v decodes gh's @tsv escapes (\t \n \r \\) first
+# json <name> — one JSON string; awk -v decodes the @tsv escapes (\t \n \r \\) first
 json() {
   LC_ALL=C awk -v s="$1" 'BEGIN {
     for (i = 1; i < 32; i++) esc[sprintf("%c", i)] = sprintf("\\u%04x", i)
@@ -71,28 +88,37 @@ json() {
   }'
 }
 defect() { echo "INVENTORY_DEFECT kind=$1 job=$(json "$2")"; mark_unreadable; }
-# step_log <log> <step> — the job-log lines gh's step column attributes to <step>, as gh printed them
-step_log() { awk -F'\t' -v s="$2" 'NF >= 3 && $2 == s' "$1"; }
-# verdict_line <log> <step> <family> <format> — the ONE <family> line <step> printed, in <format>;
-# else MISSING, MULTIPLE(n) or MALFORMED, each UNREADABLE
-verdict_line() {
-  local lines
-  lines=$(step_log "$1" "$2" | awk -v esc="$(printf '\033')" '{
-    gsub(/\^\[\[[0-9;]*m/, ""); gsub(esc "\\[[0-9;]*m", "")
-    sub(/^[^\t]*\t[^\t]*\t[0-9T:.-]+Z ?/, ""); sub(/[ \t\r]+$/, ""); print
-  }' | grep -E "^$3( |\$)")
-  case $(count "$lines") in
-    0) echo MISSING ;;
-    1) if printf '%s\n' "$lines" | grep -qxE "$4"; then printf '%s\n' "$lines"; else echo MALFORMED; fi ;;
-    *) echo "MULTIPLE($(count "$lines"))" ;;
-  esac
-}
-readable() { case $1 in MISSING|MULTIPLE*|MALFORMED) return 1 ;; esac; }
-floors_green() { printf '%s\n' "$1" | grep -qxE 'COVERAGE_FLOORS verdict=GREEN floors=[0-9]+ failing=0'; }
 
-jobs=$(gh run view "$run" --repo "$repo" --json jobs \
-  --jq '.jobs[] | [.databaseId, .name, .status, (.conclusion // "")] | @tsv') \
-  || { echo "UNREADABLE: job list"; exit 2; }
+run_json=$(gh run view "$run" --repo "$gh_repo" --json headSha,status,conclusion,jobs) \
+  || { echo "UNREADABLE: gh run view $run"; mark_unreadable; finish; }
+# the read as @tsv-escaped rows — RUN <head> <status> <conclusion>, JOB <index> <name> <status> <conclusion>,
+# STEP <job index> <name> <conclusion>; a null or empty value prints `null`; any other shape fails closed
+rows=$(printf '%s' "$run_json" | node -e '
+const field = (v) => v === null || v === undefined || v === "" ? "null"
+  : String(v).replace(/\\/g, "\\\\").replace(/\t/g, "\\t").replace(/\n/g, "\\n").replace(/\r/g, "\\r");
+const scalar = (v) => v === null || v === undefined || typeof v === "string" || typeof v === "number";
+let text = "";
+process.stdin.setEncoding("utf8").on("data", (d) => { text += d; }).on("end", () => {
+  try {
+    const run = JSON.parse(text);
+    const object = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
+    if (!object(run) || !Array.isArray(run.jobs) || ![run.headSha, run.status, run.conclusion].every(scalar)) throw 0;
+    const out = [["RUN", run.headSha, run.status, run.conclusion]];
+    run.jobs.forEach((job, i) => {
+      if (!object(job) || ![job.name, job.status, job.conclusion].every(scalar)) throw 0;
+      if (job.steps !== undefined && job.steps !== null && !Array.isArray(job.steps)) throw 0;
+      out.push(["JOB", i, job.name, job.status, job.conclusion]);
+      for (const step of job.steps ?? []) {
+        if (!object(step) || ![step.name, step.conclusion].every(scalar)) throw 0;
+        out.push(["STEP", i, step.name, step.conclusion]);
+      }
+    });
+    process.stdout.write(out.map((row) => row.map(field).join("\t") + "\n").join(""));
+  } catch { process.exit(2); }
+});') || { echo "UNREADABLE: gh run view $run returned no readable run record"; mark_unreadable; finish; }
+IFS=$'\t' read -r _ head status conclusion <<< "$(printf '%s\n' "$rows" | sed -n $'s/^RUN\t/-\t/p')"
+jobs=$(printf '%s\n' "$rows" | sed -n $'s/^JOB\t//p')
+steps=$(printf '%s\n' "$rows" | sed -n $'s/^STEP\t//p')
 # an empty job list is an inventory like any other: serial, observed=0, every declared job missing
 
 names=$(printf '%s\n' "$jobs" | cut -f2)
@@ -113,126 +139,96 @@ while IFS= read -r name; do
   printf '%s\n' "$declared" | grep -qxF -- "$name" || defect unknown "$name"
 done < <([ -z "$jobs" ] || printf '%s\n' "$names" | awk '!seen[$0]++')
 
-[ -z "$jobs" ] || while IFS=$'\t' read -r id name status conclusion; do
-  # an unknown job has no role to grade: its INVENTORY_DEFECT already made the run UNREADABLE
-  printf '%s\n' "$declared" | grep -qxF -- "$name" || continue
-  case "$name" in
-    test|test-macos) role=serial ;;
-    suite\ *) role=shard ;;
-    single-fork\ *) role=single-fork ;;
-    *) role=aggregate ;;
-  esac
+if [ "$form" = sharded ]; then
+  echo "SHARDED: UNREADABLE (the sharded proof-group selection is ANL-022 §7.4's and has not shipped)"
+  mark_unreadable
+  finish
+fi
 
-  echo "$name: role=$role status=$status conclusion=${conclusion:-none} job=$id"
-  if [ "$status" != "completed" ]; then
-    echo "$name: UNREADABLE (job has not reached run-end)"
-    mark_unreadable
-    continue
-  fi
+echo "RUN head=$head expected=$sha status=$status conclusion=$conclusion"
+if [ "$head" != "$sha" ]; then
+  echo "RUN: UNREADABLE (the run's head is not the expected head; nothing downloaded)"
+  mark_unreadable
+  finish
+fi
+if [ "$status" != completed ]; then
+  echo "RUN: UNREADABLE (the run has not reached run-end; nothing downloaded)"
+  mark_unreadable
+  finish
+fi
 
-  log="$out_dir/CI-$tag-$(printf '%s' "$name" | tr -c 'A-Za-z0-9._-' '-').log"
-  gh run view --repo "$repo" --job "$id" --log > "$log" 2>/dev/null
-  if [ ! -s "$log" ]; then
-    echo "$name: UNREADABLE (empty log)"
+[ -z "$jobs" ] || while IFS=$'\t' read -r index name job_status job_conclusion; do
+  # an unknown or duplicated job has no role to grade: its INVENTORY_DEFECT already made the run UNREADABLE
+  [ "$(printf '%s\n' "$names" | grep -cxF -- "$name")" -eq 1 ] && printf '%s\n' "$declared" | grep -qxF -- "$name" || continue
+
+  echo "$name: status=$job_status conclusion=$job_conclusion"
+  if [ "$job_status" != completed ]; then
+    echo "$name: UNREADABLE (job has not reached run-end)"
     mark_unreadable
     continue
   fi
 
-  files=$(grep -oE 'Test Files .*' "$log" | sed 's/[[:space:]]*$//')
-  # OBS-1184: the log verdict comes from the ONE classifier the public CI wrapper also uses
-  # (scripts/run-ci-vitest.sh), so the badge and this grade cannot disagree. Its rules — complete
-  # summaries, failed/timed-out tests, coverage-threshold misses, and the RPC-timeout-only exception
-  # to "any unhandled error is RED" (OBS-1058) — live in classify-vitest-log.sh, stated once there.
-  # A serial job's expected count is its declared file contract: both steps' summaries must collect it all.
-  # A shard or single-fork job carries no count: its wrapper held it to its plan in-job.
-  if [ "$role" = serial ]; then
-    classified=$(bash "$here/classify-vitest-log.sh" "$log" "$expected" 2>&1)
-  else
-    classified=$(bash "$here/classify-vitest-log.sh" "$log" 2>&1)
-  fi
-  log_verdict=$(field verdict)
-  passed=$(field passed); skipped=$(field skipped); failed=$(field failed); timedout=$(field timedout)
-  unhandled=$(field unhandled); rpc=$(field runner_rpc_timeouts); coverage=$(field coverage_misses)
-  kills=$(field runner_kills)
-  errors=$(grep -oE '##\[error\].*' "$log" | sort | uniq -c | sed 's/^ *//' | tr '\n' ';')
-  # The commands' own outcomes stand, as in the wrapper: GREEN needs a success conclusion with no step
-  # exit annotation, or — SERIAL only — a failure explained ONLY by steps that exited exactly 1 on a log
-  # of their own (gh's step column) that the classifier calls RPC_ONLY. Any other code (2, a signal's
-  # 137), any annotation on another role, an unexplained failure, or a cancelled, timed-out or missing
-  # conclusion is RED.
-  exits=$(grep -E '##\[error\]Process completed with exit code [0-9]+' "$log")
-  case "$conclusion" in
-    success) outcome=ok ;;
-    failure) if [ -n "$exits" ]; then outcome=ok; else outcome=unexplained-failure; fi ;;
-    *) outcome="conclusion-${conclusion:-none}" ;;
-  esac
-  if [ -n "$exits" ]; then
-    while IFS= read -r exit_line; do
-      code=$(printf '%s\n' "$exit_line" | sed -E 's/.*Process completed with exit code ([0-9]+).*/\1/')
-      if [ "$code" != 1 ] || [ "$role" != serial ]; then outcome="exit-$code"; break; fi
-      step=$(printf '%s\n' "$exit_line" | awk -F'\t' 'NF >= 3 { print $2 }')
-      step_log "$log" "$step" > "$log.step"
-      case $(bash "$here/classify-vitest-log.sh" "$log.step" 2>&1) in
-        "VITEST_LOG verdict=RPC_ONLY "*) ;;
-        *) outcome="exit-1-not-rpc-only"; break ;;
-      esac
-    done <<< "$exits"
-  fi
-
-  oracles=
-  case $role in
-    serial)
-      oracle=$(verdict_line "$log" "$COUNT_STEP" COUNT_ORACLE "$COUNT_FORMAT")
-      floors=$(verdict_line "$log" "$FLOORS_STEP" COVERAGE_FLOORS "$FLOORS_FORMAT")
-      oracles="oracle=[$oracle] floors=[$floors] " ;;
-    aggregate)
-      floors=$(verdict_line "$log" "$FLOORS_STEP" COVERAGE_FLOORS "$FLOORS_FORMAT")
-      set_oracle=$(verdict_line "$log" "$SET_STEP" SET_ORACLE "$SET_FORMAT")
-      step_log "$log" "$MERGE_STEP" > "$log.merge"
-      merge=$(bash "$here/classify-vitest-log.sh" "$log.merge" 2>&1 | sed -n 's/^VITEST_LOG verdict=\([^ ]*\) reason=\([^ ]*\) .*/\1 \2/p')
-      oracles="floors=[$floors] set=[$set_oracle] merge=[$merge] " ;;
-  esac
+  own=$(printf '%s\n' "$steps" | awk -F'\t' -v i="$index" '$1 == i')
+  listed=; unread=; failed=; unproven=; killed=
+  [ "$job_conclusion" != cancelled ] || killed=1
+  while IFS= read -r step; do
+    matches=$(printf '%s\n' "$own" | awk -F'\t' -v s="$step" '$2 == s { print $3 }')
+    case $(count "$matches") in
+      0) outcome=MISSING; unread=1 ;;
+      1) outcome=$matches ;;
+      *) outcome="MULTIPLE($(count "$matches"))"; unread=1 ;;
+    esac
+    case $outcome in
+      success|MISSING|MULTIPLE*) ;;
+      cancelled) killed=1 ;;
+      failure) failed=1 ;;
+      *) unproven=1 ;; # skipped, or no conclusion: never success
+    esac
+    listed="$listed${listed:+ | }$step: $outcome"
+  done <<< "$serial_steps"
 
-  echo "$name: ${oracles}files=[$(printf '%s' "$files" | tr '\n' '|')] passed=$passed skipped=$skipped failed=$failed timedout=$timedout unhandled=$unhandled runner_rpc_timeouts=$rpc coverage_misses=$coverage runner_kills=${kills:-?} log=[$(field reason)] outcome=[$outcome] errors=[$errors]"
-  # A killed runner skips the oracle steps, so a missing verdict line is the kill's, not an unreadable log:
-  # INFRA_KILLED unless a step's own failing exit or a failed/timed-out test dominates it (then RED).
-  if [ "$log_verdict" = INFRA_KILLED ] && [ "${outcome#exit-}" = "$outcome" ]; then
-    echo "$name: INFRA_KILLED"
-    mark_killed
-    continue
-  elif [ "${kills:-0}" -gt 0 ] 2>/dev/null; then
-    grade=RED
-  elif [ -z "$log_verdict" ]; then
-    grade=UNREADABLE
+  proof=
+  grade=GREEN
+  if [ -n "$unread" ]; then
+    grade=UNREADABLE; proof="not downloaded: a declared step is missing or duplicated"
+  elif [ -n "$killed" ]; then
+    grade=INFRA_KILLED; [ -z "$failed" ] || grade=RED; proof="not downloaded: the job was cancelled"
   else
-    case $log_verdict in CLEAN|RPC_ONLY) log_ok=1 ;; *) log_ok= ;; esac
-    grade=RED
-    case $role in
-      serial)
-        if ! readable "$oracle" || ! readable "$floors" || [ -z "$files" ]; then
-          grade=UNREADABLE
-        elif [ "$oracle" = "COUNT_ORACLE GREEN expected=$expected actual=$expected" ] && floors_green "$floors" \
-             && [ -n "$log_ok" ] && [ "$outcome" = ok ] && [ $((passed + skipped)) -eq "$expected" ]; then
-          grade=GREEN
-        fi ;;
-      shard|single-fork)
-        if [ "$outcome" = ok ] && [ "$conclusion" = success ] && [ -n "$log_ok" ]; then grade=GREEN; fi ;;
-      aggregate)
-        if ! readable "$floors" || ! readable "$set_oracle"; then
-          grade=UNREADABLE
-        elif [ "$outcome" = ok ] && [ "$conclusion" = success ] && floors_green "$floors" \
-             && [ "$set_oracle" = "SET_ORACLE verdict=GREEN expected=$expected actual=$expected defects=0" ] \
-             && { [ "${merge%% *}" = CLEAN ] || [ "${merge%% *}" = RPC_ONLY ]; }; then
-          grade=GREEN
-        fi ;;
-    esac
+    dir=$(mktemp -d "$out_dir/CI-$(printf '%s' "$tag" | tr -c 'A-Za-z0-9._-' '-')-proof-$name.XXXXXX") \
+      && gh run download "$run" --repo "$gh_repo" -n "proof-$name" -D "$dir" > /dev/null 2>&1
+    if [ $? -ne 0 ]; then
+      grade=UNREADABLE; proof="MISSING: artifact proof-$name not downloaded"
+    else
+      proof_out=$("${cli[@]}" proof --head "$sha" --repo "$repo_dir" "$dir" 2> "$dir.stderr")
+      proof_status=$?
+      printf '%s\n' "$proof_out" | sed -n "s/^PROOF_DEFECT /$name: PROOF_DEFECT /p"
+      proof=$(printf '%s\n' "$proof_out" | grep '^PROOF ')
+      word=$(printf '%s\n' "$proof" | sed -n 's/^PROOF dir=.* verdict=\([A-Z]*\) head=.*/\1/p')
+      # fail closed: one PROOF line, and its word believed only beside its own exit status
+      if [ "$(count "$proof")" -ne 1 ]; then
+        grade=UNREADABLE; proof="UNREADABLE: cli.ts proof exited $proof_status with $(count "$proof") PROOF lines"
+      elif [ "$word:$proof_status" = GREEN:0 ] || [ "$word:$proof_status" = RED:1 ]; then
+        grade=$word
+      else
+        grade=UNREADABLE
+      fi
+    fi
+    if [ "$grade" != UNREADABLE ] && { [ -n "$failed$unproven" ] || [ "$job_conclusion" != success ]; }; then grade=RED; fi
   fi
+  echo "$name: steps=[$listed] proof=[$proof]"
   echo "$name: $grade"
   case $grade in
     RED) mark_red ;;
     UNREADABLE) mark_unreadable ;;
+    INFRA_KILLED) mark_killed ;;
   esac
 done <<< "$jobs"
 
-echo "VERDICT rc=$verdict (0=all GREEN 1=RED 2=UNREADABLE 3=INFRA_KILLED)"
-exit "$verdict"
+# the run's own conclusion: GREEN needs success; a cancelled run is INFRA_KILLED; any other conclusion is RED
+# unless a job's grade above already accounts for it (a killed job concludes its run `failure` too)
+case $conclusion in
+  success) ;;
+  cancelled) mark_killed ;;
+  *) if [ "$verdict" -eq 0 ]; then echo "RUN: RED (run conclusion $conclusion with every job GREEN)"; mark_red; fi ;;
+esac
+finish

package/skills/tickmarkr-overseer/scripts/classify-vitest-log.sh

@@ -1,133 +0,0 @@
-#!/bin/bash
-# classify-vitest-log.sh <log> [expected-files] — the ONE strict Vitest log classifier
-# (OBS-1184). The public CI wrapper (scripts/run-ci-vitest.sh) and grade-ci.sh both call it, so the badge
-# and the grade cannot disagree. Reads a raw Vitest log or a gh job log (gh's "job<TAB>step<TAB>timestamp "
-# prefix and ANSI styling — the raw ESC byte and gh's `^[` rendering of it — are stripped) and prints ONE line:
-#   VITEST_LOG verdict=<CLEAN|RPC_ONLY|RED|INFRA_KILLED> reason=<why> summaries=S passed=P skipped=K failed=F
-#     timedout=T unhandled=U runner_rpc_timeouts=R coverage_misses=C signals=G terminal_errors=E runner_kills=X
-# unhandled counts the unhandled errors that are NOT runner RPC timeouts. Exit 0 CLEAN or RPC_ONLY, 1 RED,
-# 3 INFRA_KILLED.
-#
-# INFRA_KILLED (D-897): the GitHub runner's own shutdown or cancellation annotation ("##[error]The runner
-# has received a shutdown signal…" / "##[error]The operation was canceled.") with no failed or timed-out
-# test and no coverage-threshold miss. It is never green and never passes a gate — the proof was not
-# finished — but it is not a test red either. Those failures dominate a kill (RED); a kill excuses only
-# the cut-off or short log it leaves behind (runner_kills counts the annotations either way). A failure
-# the reporter already printed before the kill cut its summary off counts too: a dot-reporter "x" (its
-# stream has no newline, so a test's "stdout | …"/"stderr | …" header may finish the same line), a
-# default-reporter "×" or failed module record "❯ file (N tests…)" (❯ marks a failed module even with
-# no failed test — a collect or hook error), a " FAIL " badge or the "Failed Tests/Suites N" banner (only
-# beside a kill — without one, the complete summaries' own counts decide). Only one printed after the last
-# complete summary counts: a complete summary's own counts settle every line before it. Test output never
-# counts: a console block runs from its header until the reporter speaks again — a dot run, a ⎯ banner or
-# a default-reporter module record (✓/❯/↓ file (N tests…)) right after a blank line, or the next summary —
-# so blank lines inside a test's output do not end it.
-# ponytail: a test that prints, after a blank line of its own, a line the reporter could have printed is
-# indistinguishable from the reporter; only that deliberate forgery escapes the block.
-#
-# The declared contract (D-893), when a caller supplies one: expected-files is the number of test files
-# the run must collect — the summaries' "(N)" totals must sum to it exactly. The caller derives it from
-# the declared projects' own candidate-tree discovery, so a never-started declared project is a short
-# count: project presence comes from that discovery, never from log text a test can print. A short count
-# is RED reason=partial: a complete summary of the projects that happened to start proves nothing about
-# the one that did not. With or without a contract, a summary whose finished files (failed + passed +
-# skipped + todo) fall short of its own "(N)" is RED reason=partial too: Vitest's total counts the queued
-# and running files an abort left unfinished.
-#
-# RED unless every Vitest summary is complete (Test Files, Tests and Duration lines, and every
-# "Unhandled Errors" block closed by the summary that follows it — a cut-off log is RED), with no
-# failed file or test, no timed-out test or hook, no coverage-threshold ERROR, no npm signal line,
-# no unhandled error other than a vitest-worker RPC timeout, and no terminal error: a structural
-# "Unhandled Error", "Startup Error" or "Collect Error" banner outside Vitest's counted block — how
-# Vitest reports an exception thrown after the summary, such as a coverage-reporting failure, before it
-# exits 1 (terminal_errors counts them; an RPC timeout never offsets one). RPC_ONLY means unhandled
-# errors were present and EVERY one is such a timeout (OBS-1058: the starved 2-core runner talking,
-# not a test).
-#
-# Which timeouts count: only one that is ITSELF an unhandled entry — the first payload line under a
-# per-error header, nothing later. The header is Vitest's own structural line — a run of ⎯ on each side
-# of "Unhandled Error", nothing else on the line — so prose that merely contains the words never opens
-# a window, and the payload must be Vitest's whole line (`Error: [vitest-worker]: Timeout calling
-# "<method>"`), so an assertion quoting it stays RED. Per-error headers are honoured only inside
-# Vitest's own block — after its structural "Unhandled Errors" line and "Vitest caught N" tally, until
-# the file summary — and each block's count never exceeds its N, so a test that prints a header and a
-# timeout to stdout forges nothing. The tallies must also sum to the summaries' own "Errors N" lines
-# (Vitest prints both from one list), so a forged or a missing block is RED as well.
-set -u
-
-log=${1:?log path required}
-fields='summaries=0 passed=0 skipped=0 failed=0 timedout=0 unhandled=0 runner_rpc_timeouts=0 coverage_misses=0 signals=0 terminal_errors=0 runner_kills=0'
-if [ ! -s "$log" ] || [ ! -r "$log" ]; then
-  echo "VITEST_LOG verdict=RED reason=unreadable $fields"
-  exit 1
-fi
-
-esc=$(printf '\033')
-awk -v esc="$esc" -v expected="${2:-}" '
-  function num(s, word) { return match(s, "[0-9]+ " word) ? substr(s, RSTART, RLENGTH) + 0 : 0 }
-  function close_block() { rpc += (found < cap ? found : cap); armed = 0; pending = 0 }
-  {
-    line = $0
-    gsub(/\^\[\[[0-9;]*m/, "", line); gsub(esc "\\[[0-9;]*m", "", line)
-    sub(/^[^\t]*\t[^\t]*\t[0-9T:.-]+Z[ ]?/, "", line)
-    if (line ~ /^[ \t]*(·|x|\*|-)*(stdout|stderr) \| /) {
-      if (line ~ /^[ \t]*(·|\*|-)*x/) reported++
-      console = 1
-    } else if (after_blank && line ~ /^[ \t]*((·|x|\*|-)*(·|x)(·|x|\*|-)*|(⎯)+( .* (⎯)+)?|(✓|❯|↓) .* [(][0-9]+ tests?( [|] [0-9]+ (failed|skipped))*[)].*)[ \t]*$/) console = 0
-    if (!console && line ~ /^[ \t]*((·|\*|-)*x(·|x|\*|-)*|× .*|❯ .* [(][0-9]+ tests?( [|] [0-9]+ (failed|skipped))*[)].*|FAIL[ \t].*|(⎯)+ Failed (Tests|Suites) [0-9]+ (⎯)+)[ \t]*$/) reported++
-    after_blank = line !~ /[^ \t]/
-  }
-  line ~ /^##\[error\](The runner has received a shutdown signal|The operation was cancell?ed)/ { kills++ }
-  # The Vitest timeout message carries its millisecond count; a test that PRINTS a fingerprint normalized
-  # to #ms (the daemon OBS-1106 notifications) is output, not a timed-out test. A real timeout also fails
-  # its test, so the summary failed count still reds it.
-  line ~ /(Test|Hook) timed out in [0-9]+ ?ms/ { timedout++ }
-  line ~ /ERROR: Coverage for .* does not meet .*threshold/ { coverage++ }
-  line ~ /^npm (error|ERR!) signal / { signals++ }
-  line ~ /^(⎯)+ Unhandled Errors (⎯)+[ \t]*$/ { if (armed) close_block(); opening = 1; next }
-  opening && line !~ /[^ \t]/ { next }
-  opening {
-    opening = 0
-    if (line ~ /^Vitest caught [0-9]+ unhandled error/) { armed = 1; found = 0; cap = num(line, "unhandled"); tally += cap }
-    else untallied++
-  }
-  line ~ /^[ \t]*Test Files[ \t]/ {
-    if (armed) close_block()
-    summaries++; in_summary = 1; saw_tests = 0; console = 0
-    failed += num(line, "failed"); passed += num(line, "passed"); skipped += num(line, "skipped")
-    finished = num(line, "failed") + num(line, "passed") + num(line, "skipped") + num(line, "todo")
-    total = match(line, /\([0-9]+\)[ \t]*$/) ? substr(line, RSTART + 1) + 0 : -1
-    if (total >= 0) collected += total
-    if (finished != total) unfinished++
-    next
-  }
-  in_summary && line ~ /^[ \t]*Tests[ \t]/ { saw_tests = 1; failed_tests += num(line, "failed"); next }
-  in_summary && line ~ /^[ \t]*Errors[ \t]+[0-9]+ error/ { summary_errors += num(line, "error"); next }
-  in_summary && line ~ /^[ \t]*Duration[ \t]/ { if (saw_tests) { complete++; reported = 0 }; in_summary = 0; next }
-  armed && line ~ /^(⎯)+ Unhandled Error (⎯)+[ \t]*$/ { pending = 1; next }
-  line ~ /^(⎯)+ (Unhandled|Startup|Collect) Error (⎯)+[ \t]*$/ { terminal++; next }
-  pending && line ~ /[^ \t]/ { pending = 0; if (line ~ /^Error: \[vitest-worker\]: Timeout calling "[A-Za-z]+"[ \t]*$/) found++ }
-  END {
-    unhandled = tally + untallied
-    if (summary_errors > unhandled) unhandled = summary_errors
-    other = unhandled - rpc
-    partial = unfinished || (expected != "" && collected != expected + 0)
-    # a kill always names a class: the failure that dominates it, or the kill itself
-    if (kills) reason = (failed || failed_tests || reported) ? "failed" : timedout ? "timed-out" : coverage ? "coverage-threshold" : "runner-shutdown"
-    else if (summaries == 0) reason = "no-summary"
-    else if (complete != summaries || opening || armed) reason = "truncated"
-    else if (failed || failed_tests) reason = "failed"
-    else if (timedout) reason = "timed-out"
-    else if (partial) reason = "partial"
-    else if (coverage) reason = "coverage-threshold"
-    else if (signals) reason = "signal"
-    else if (terminal) reason = "terminal-error"
-    else if (other > 0) reason = "unhandled-error"
-    else if (tally + untallied != summary_errors) reason = "unhandled-mismatch"
-    verdict = reason == "runner-shutdown" ? "INFRA_KILLED" : (reason != "" ? "RED" : (unhandled > 0 ? "RPC_ONLY" : "CLEAN"))
-    if (reason == "") reason = "none"
-    printf "VITEST_LOG verdict=%s reason=%s summaries=%d passed=%d skipped=%d failed=%d timedout=%d unhandled=%d runner_rpc_timeouts=%d coverage_misses=%d signals=%d terminal_errors=%d runner_kills=%d\n", \
-      verdict, reason, summaries, passed, skipped, failed, timedout, other, rpc, coverage, signals, terminal, kills
-    exit (verdict == "INFRA_KILLED" ? 3 : verdict == "RED")
-  }
-' "$log"