tickmarkr 2.8.1 → 2.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/codex.js +3 -2
- package/dist/gates/baseline.js +3 -3
- package/dist/run/daemon.js +2 -1
- package/dist/run/wall-budget.js +6 -1
- package/package.json +1 -2
- package/skills/tickmarkr-overseer/SKILL.md +34 -20
- package/skills/tickmarkr-overseer/scripts/grade-ci.sh +162 -166
- package/skills/tickmarkr-overseer/scripts/classify-vitest-log.sh +0 -133
package/dist/adapters/codex.js
CHANGED
|
@@ -121,8 +121,9 @@ function codexRetirement(upgrade) {
|
|
|
121
121
|
// literal-string fallback. Every emitted root is therefore a TOML basic string that parses back to the exact
|
|
122
122
|
// path (git's own trailing newline is stripped through a sentinel, never a path's). No common-root fallback;
|
|
123
123
|
// packed-refs is not granted (packed-ref ops fail closed); stash, tags, notes and branches outside the leaf
|
|
124
|
-
// are denied. Declared: sibling refs in the same leaf (other task branches, the run's integration
|
|
125
|
-
// under the default prefix) stay writable
|
|
124
|
+
// are denied. Declared (queue row 151): sibling refs in the same leaf (other task branches, the run's integration
|
|
125
|
+
// branch under the default prefix) stay writable; since 2.8.1 a write to the integration ref parks the merge and is
|
|
126
|
+
// never published.
|
|
126
127
|
// doctor's probe (codex-commit-check.ts) executes these exact bytes; it never rebuilds the grant.
|
|
127
128
|
export const CODEX_GIT_GRANT = `-c "sandbox_workspace_write.writable_roots=[$(tkr_common=$(git rev-parse --path-format=absolute --git-common-dir && echo /) && tkr_gitdir=$(git rev-parse --absolute-git-dir && echo /) && tkr_common=\${tkr_common%?/} && tkr_gitdir=\${tkr_gitdir%?/} && { tkr_head=$(git symbolic-ref -q HEAD); [ $? -le 1 ]; } && case $tkr_head in (refs/heads/*) tkr_leaf=\${tkr_head%/*};; (*) tkr_leaf=;; esac && [ "$(printf %s "$tkr_common$tkr_gitdir$tkr_leaf" | LC_ALL=C tr -d '\\0-\\37\\177"\\\\')" = "$tkr_common$tkr_gitdir$tkr_leaf" ] && { [ "$tkr_gitdir" = "$tkr_common" ] || printf '"%s",' "$tkr_gitdir"; } && printf '"%s/objects"' "$tkr_common" && { [ -z "$tkr_leaf" ] || printf ',"%s/%s","%s/logs/%s"' "$tkr_common" "$tkr_leaf" "$tkr_common" "$tkr_leaf"; })]"`;
|
|
128
129
|
// OBS-125: codex 0.144.x gates enabled hooks behind PER-PROJECT-PATH hook trust — a NEW gate distinct
|
package/dist/gates/baseline.js
CHANGED
|
@@ -363,9 +363,9 @@ const VOCAB_RE = /\b(?:error|fail(?:ed|ure|ing)?)\b/i;
|
|
|
363
363
|
// OBS-791: `[vitest-worker]: Timeout calling "<method>"` is the SAME birpc mechanism one layer up.
|
|
364
364
|
// Vitest's bundled birpc has a fixed 60s DEFAULT_TIMEOUT with no override (fixed upstream only in
|
|
365
365
|
// vitest 4.x), so a suite whose tests all pass can still die at teardown when the worker->host RPC
|
|
366
|
-
// window closes.
|
|
367
|
-
//
|
|
368
|
-
//
|
|
366
|
+
// window closes. Public CI's proof forgives this exact fingerprint (the RPC-only rule; since v2.8.2 decided
|
|
367
|
+
// from the run's own record, queue rows 148 and 149) and this classifier once called it a regression — the
|
|
368
|
+
// product charged a worker for the failure its own release proof forgives. The METHOD NAME IS DELIBERATELY NOT PINNED: the timeout is a
|
|
369
369
|
// property of the RPC window, not of `onTaskUpdate`, and pinning one method would forgive a run and
|
|
370
370
|
// charge its sibling for the same infrastructure event. This stays a CLOSED signature — the
|
|
371
371
|
// `[vitest-worker]: ` prefix is vitest's own RPC layer and never user assertion text — and the
|
package/dist/run/daemon.js
CHANGED
|
@@ -7306,7 +7306,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
7306
7306
|
throw error;
|
|
7307
7307
|
});
|
|
7308
7308
|
fatalPhase = "tip-verify";
|
|
7309
|
-
// OBS-34: post-merge integration-tip verify —
|
|
7309
|
+
// OBS-34: post-merge integration-tip verify — the battery forgives only what its baseline entry recorded red
|
|
7310
|
+
// (src/run/merge.ts, the same predicate as the gate battery).
|
|
7310
7311
|
const lastMergedTask = [...journal.read()].reverse().find((e) => e.event === "merge" && e.taskId)?.taskId;
|
|
7311
7312
|
// 2.8.1 T1 (ANL-023 §7.1): every close opens on the integration identity and closes on it below.
|
|
7312
7313
|
const identity = { repo: repoRoot, branch, expected: published }; // 2.8.1 T2: E
|
package/dist/run/wall-budget.js
CHANGED
|
@@ -74,7 +74,12 @@ export function wallBudget(events) {
|
|
|
74
74
|
return;
|
|
75
75
|
span("queue", opened, ts);
|
|
76
76
|
// One open wait per owner, closed in journal order: the list stays sorted and non-overlapping.
|
|
77
|
-
|
|
77
|
+
// Appended in place: a copy per close made a run with n waits cost O(n^2) (ANL-022 §3 T1 #5).
|
|
78
|
+
const list = queued.get(key);
|
|
79
|
+
if (list)
|
|
80
|
+
list.push([opened, ts]);
|
|
81
|
+
else
|
|
82
|
+
queued.set(key, [[opened, ts]]);
|
|
78
83
|
waits.delete(key);
|
|
79
84
|
};
|
|
80
85
|
const service = (gate, task, a, b) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tickmarkr",
|
|
3
|
-
"version": "2.8.
|
|
3
|
+
"version": "2.8.2",
|
|
4
4
|
"description": "Spec in, verified work out.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -77,7 +77,6 @@
|
|
|
77
77
|
"tsconfig.json",
|
|
78
78
|
"vitest.config.ts",
|
|
79
79
|
".github/pull_request_template.md",
|
|
80
|
-
"scripts/assert-test-file-count.sh",
|
|
81
80
|
"scripts/emit-schema.ts",
|
|
82
81
|
"scripts/probe-rig.mjs",
|
|
83
82
|
"scripts/run-ci-vitest.sh",
|
|
@@ -691,12 +691,13 @@ subject of a release proof is never *"the run"* and never *"the fix leg"* — it
|
|
|
691
691
|
the one commit that carries the run's work AND every leg merged into it. Name that commit's sha in the
|
|
692
692
|
proof, in the same act as the grade, and the proof is bound to that sha alone. **When the subject moves — a
|
|
693
693
|
later merge, a new integration tip, an amended or re-cut commit, a fresh export, a re-tag — the proof is
|
|
694
|
-
VOID, and the new subject is graded from scratch**,
|
|
695
|
-
the merged tree and not from either parent. Two greens assembled into one verdict are two
|
|
696
|
-
trees that never contained each other: the union's own conflict resolutions, its ordering
|
|
697
|
-
collected
|
|
698
|
-
the
|
|
699
|
-
pointed at bytes: **a proof that names no commit cannot
|
|
694
|
+
VOID, and the new subject is graded from scratch**, its tracked test set included, since that set is
|
|
695
|
+
recomputed from the merged tree and not from either parent. Two greens assembled into one verdict are two
|
|
696
|
+
claims about two trees that never contained each other: the union's own conflict resolutions, its ordering
|
|
697
|
+
and its collected test set are exactly what neither green observed — which is why `RELEASING.md` step 4
|
|
698
|
+
hands the grader the mirror's `HEAD` and the grader refuses a run of any other head before a single record
|
|
699
|
+
is downloaded. This is the void-condition duty pointed at bytes: **a proof that names no commit cannot
|
|
700
|
+
notice its subject leaving.**
|
|
700
701
|
|
|
701
702
|
**THE RECIPROCAL DUTY, and it is yours because you write both documents:** when you issue a ruling that
|
|
702
703
|
re-scopes, narrows, splits or re-owns anything, **the ruling must name every sealed document its subject
|
|
@@ -1059,7 +1060,8 @@ they are left implicit:
|
|
|
1059
1060
|
test exists — the acceptance oracle runs the sentence as vitest `-t` and a paraphrase matches zero tests.
|
|
1060
1061
|
- Every fix-leg verify carries `--author <the seat that wrote it>`; without it the author is HUMAN, every
|
|
1061
1062
|
vendor stays eligible, and a same-vendor approval is not the cross-vendor review the criterion names.
|
|
1062
|
-
- A grader change is proved on a
|
|
1063
|
+
- A grader change is proved on a recorded run — GitHub's structured run record and the proof groups its
|
|
1064
|
+
jobs uploaded — never on a synthetic job log: the grader reads no log text at all.
|
|
1063
1065
|
- A public-CI red is classified in a throttled Linux container first (`docker run --cpus=2 node:22`),
|
|
1064
1066
|
by measurement: starvation on the base tree is not a defect on the branch.
|
|
1065
1067
|
|
|
@@ -1603,19 +1605,31 @@ Create specialist seats using `orca terminal create --worktree path:<repo> --com
|
|
|
1603
1605
|
- **RUN BOTH RELEASE PROOFS BEFORE STANDING DOWN.** The first is the exported-tree suite, which
|
|
1604
1606
|
maintainers run from the private repo with its `verify:export` script — that tooling is deliberately
|
|
1605
1607
|
not part of this package, so the command does not exist in a public checkout; it must install,
|
|
1606
|
-
build, lint, and only then run the suite in the exported tree
|
|
1607
|
-
|
|
1608
|
-
smoke
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
`
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1608
|
+
build, lint, and only then run the suite in the exported tree, and only its zero exit writes the export
|
|
1609
|
+
proof that `export-public.sh --onto` reads before it writes the mirror commit. Then the installed-Orca
|
|
1610
|
+
smoke must exercise its live leg, decided from its own record and never from Vitest's exit status:
|
|
1611
|
+
`TICKMARKR_CI_RECORD=/tmp/tickmarkr-orca-smoke.record.json TICKMARKR_E2E=1 npx --no-install vitest run tests/e2e/orca-smoke.e2e.test.ts`,
|
|
1612
|
+
then
|
|
1613
|
+
`npx --no-install tsx scripts/ci/cli.ts verdict --require-test 'tests/e2e/orca-smoke.e2e.test.ts::e2e: orca driver smoke > live: the real-orca terminal leg — vitest-skipped without the gate or a reachable runtime' /tmp/tickmarkr-orca-smoke.record.json`
|
|
1614
|
+
must exit 0. The forced-stall proof (RELEASING.md step 6) is decided the same way: its record from
|
|
1615
|
+
`TICKMARKR_CI_RECORD=/tmp/tickmarkr-forced-stall.record.json TICKMARKR_TEST_FORCE_STALL=1 npx --no-install vitest run tests/e2e/forced-stall.e2e.test.ts -t 'the enabled forced-stall release proof makes the integration tip verifier recover a real RPC-stranded serial file as parallel exit1 then serial-only exit0 under distinct nonces, so a helper-only green or a full-suite retry fails'`,
|
|
1616
|
+
then
|
|
1617
|
+
`npx --no-install tsx scripts/ci/cli.ts verdict --require-test 'tests/e2e/forced-stall.e2e.test.ts::the enabled forced-stall release proof makes the integration tip verifier recover a real RPC-stranded serial file as parallel exit1 then serial-only exit0 under distinct nonces, so a helper-only green or a full-suite retry fails' /tmp/tickmarkr-forced-stall.record.json`
|
|
1618
|
+
must exit 0. A skipped smoke or stall leaf is not proof: Vitest exits 0 when the named leaf skips (the gate
|
|
1619
|
+
unset, Orca unreachable, a drifted `-t` title), and the verdict reads it RED. Record both results before
|
|
1620
|
+
declaring the release complete.
|
|
1621
|
+
|
|
1622
|
+
- **GRADE PUBLIC CI FROM ITS RUN RECORD AND PROOF GROUPS AT RUN-END ONLY (law 48).** A badge is not
|
|
1623
|
+
the grade, and an in-progress run is never graded. At run-end execute
|
|
1624
|
+
`scripts/grade-ci.sh <run-id> <mirror HEAD sha> <mirror clone> <tag>` and require GREEN from every job
|
|
1625
|
+
of the declared inventory it prints first — serial `test` and `test-macos`; a missing, duplicated or
|
|
1626
|
+
unknown job reads UNREADABLE, never a skip, and a sharded run reads UNREADABLE until its proof-group
|
|
1627
|
+
selection ships (ANL-022 §7.4). The grader is bound to the head: a run of any other commit reads
|
|
1628
|
+
UNREADABLE before anything is downloaded. It reads GitHub's structured step outcomes and conclusions and
|
|
1629
|
+
recomputes each job's uploaded proof group (`proof-<job>`: the Vitest records and the floors sidecar)
|
|
1630
|
+
against the test files that commit tracks in the mirror clone — there is no count to derive, and no job
|
|
1631
|
+
log text is read, so a log gh labels `UNKNOWN STEP` or a test that prints timeout text cannot move the
|
|
1632
|
+
grade. Its UNREADABLE state is a hard stop, not a green default.
|
|
1619
1633
|
|
|
1620
1634
|
- **RE-READ THE REGISTRY BEFORE RULING (law 51).** A successful Release run and the first
|
|
1621
1635
|
`npm view` are separate observations: registry propagation can briefly return the prior version.
|
|
@@ -1,45 +1,59 @@
|
|
|
1
1
|
#!/bin/bash
|
|
2
|
-
# grade-ci.sh <run-id> <expected-
|
|
3
|
-
#
|
|
4
|
-
#
|
|
5
|
-
#
|
|
2
|
+
# grade-ci.sh <run-id> <expected-sha> <repo-dir> [tag] — grade EVERY job of a public CI run of <expected-sha>
|
|
3
|
+
# from GitHub's structured run record and each job's uploaded proof group (v2.8.2 T3, ANL-022 §7.2b; queue rows
|
|
4
|
+
# 165 and 148). NO LOG TEXT IS READ: no job log, no step label, no `##[error]` annotation decides anything.
|
|
5
|
+
# Grade only at run-end. Exit 0 GREEN, 1 RED, 2 UNREADABLE, 3 INFRA_KILLED: a cancelled run, job or declared
|
|
6
|
+
# step with no failed declared step — never green, never a pass, only unproven.
|
|
6
7
|
# UNREADABLE dominates a mixed result, then RED: a failure anywhere dominates a kill.
|
|
7
8
|
#
|
|
9
|
+
# INPUT: <expected-sha> is 40 lowercase hex naming a commit in <repo-dir> — the mirror clone (RELEASING.md step
|
|
10
|
+
# 4) or the Release workflow's own checkout — else exit 2 before any gh call. No count argument: the manifest of
|
|
11
|
+
# tracked test files is recomputed from that commit.
|
|
12
|
+
# ONE READ: `gh run view <run-id> --json headSha,status,conclusion,jobs`. IDENTITY: UNREADABLE, before any
|
|
13
|
+
# download, unless the run's headSha is <expected-sha> and its status is completed.
|
|
14
|
+
#
|
|
8
15
|
# Two CLOSED inventories (v2.7.8 T4; the workflow renders these names): serial {test, test-macos}; sharded,
|
|
9
16
|
# for os in ubuntu-latest and macos-latest, `suite <os> k/4` for k 1..4, `single-fork <os>`, `aggregate <os>`.
|
|
10
17
|
# A run holding any sharded name is graded against the sharded inventory, else the serial one. The first
|
|
11
18
|
# line is `INVENTORY form=<serial|sharded> declared=<n> observed=<n>`; each missing, duplicated or unknown
|
|
12
19
|
# name prints `INVENTORY_DEFECT kind=<missing|duplicate|unknown> job=<json-quoted name>` and makes the run
|
|
13
|
-
# UNREADABLE: a job the inventory does not name is never skipped.
|
|
20
|
+
# UNREADABLE: a job the inventory does not name is never skipped. The sharded proof-group selection is ANL-022
|
|
21
|
+
# §7.4's, so until it ships a sharded run reads UNREADABLE after its INVENTORY line.
|
|
14
22
|
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
# `Merge coverage (one threshold check per OS)` classified CLEAN or RPC_ONLY.
|
|
30
|
-
# Exit annotations: SERIAL keeps the one exception run-ci-vitest.sh grants (a failure explained ONLY by steps
|
|
31
|
-
# that exited exactly 1 on a log of their own the classifier reads RPC_ONLY); any other failing annotation is
|
|
32
|
-
# never GREEN. Shard, single-fork and aggregate admit none — their wrapper applied the exception in-job.
|
|
33
|
-
# INFRA_KILLED is the base rule for every role.
|
|
23
|
+
# Per serial job, every part from the same read or the job's own artifact:
|
|
24
|
+
# steps each DECLARED step below present exactly once in .jobs[].steps[] (missing or duplicated ->
|
|
25
|
+
# UNREADABLE); GREEN needs every one `success`. A cancelled job or declared step is INFRA_KILLED
|
|
26
|
+
# unless a declared step concluded `failure` (then RED); the steps a kill skipped do not count.
|
|
27
|
+
# conclusions GREEN also needs the job's and the run's conclusion `success`, so a failing undeclared step (an
|
|
28
|
+
# action's post-step) reds the grade; a cancelled run is INFRA_KILLED, and any other run conclusion
|
|
29
|
+
# no job's grade accounts for is RED.
|
|
30
|
+
# proof `gh run download <run-id> -n proof-<job>` into a fresh directory (missing -> UNREADABLE), then
|
|
31
|
+
# the grader's own checkout's `tsx scripts/ci/cli.ts proof --head <expected-sha> --repo <repo-dir>
|
|
32
|
+
# <dir>` (that checkout is the git toplevel holding this script; without its cli.ts, exit 2):
|
|
33
|
+
# the group's original records read CLEAN or RPC_ONLY at <expected-sha>, the tracked set recomputed
|
|
34
|
+
# from the commit holds every file once and finished, and the floors sidecar reads GREEN.
|
|
35
|
+
# Output: the INVENTORY lines, `RUN head=… expected=… status=… conclusion=…`, per job `<job>: steps=[…]
|
|
36
|
+
# proof=[<PROOF line>]` then `<job>: <GRADE>`, and a final `VERDICT rc=<n> …`.
|
|
34
37
|
set -u
|
|
35
38
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
+
[ "$#" -ge 3 ] && [ "$#" -le 4 ] \
|
|
40
|
+
|| { echo "UNREADABLE: usage: grade-ci.sh <run-id> <expected-sha> <repo-dir> [tag]"; exit 2; }
|
|
41
|
+
run=$1
|
|
42
|
+
sha=$2
|
|
43
|
+
repo_dir=$3
|
|
44
|
+
tag=${4:-$run}
|
|
39
45
|
here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
40
|
-
|
|
46
|
+
# the grader's own checkout — the same for the canonical copy and the installed .claude/skills one
|
|
47
|
+
checkout=$(git -C "$here" rev-parse --show-toplevel 2>/dev/null)
|
|
48
|
+
gh_repo=${TKR_GRADE_CI_REPO:-alzahrani-khalid/tickmarkr}
|
|
41
49
|
out_dir=${TKR_GRADE_CI_DIR:-${TKR_STATE_DIR:-.tickmarkr}/overseer/diag}
|
|
42
|
-
|
|
50
|
+
|
|
51
|
+
[[ $sha =~ ^[0-9a-f]{40}$ ]] || { echo "UNREADABLE: the expected head is not 40 lowercase hex"; exit 2; }
|
|
52
|
+
git -C "$repo_dir" cat-file -e "$sha^{commit}" 2>/dev/null \
|
|
53
|
+
|| { echo "UNREADABLE: $sha is not a commit in $repo_dir"; exit 2; }
|
|
54
|
+
[ -n "$checkout" ] && [ -f "$checkout/scripts/ci/cli.ts" ] \
|
|
55
|
+
|| { echo "UNREADABLE: the grader's own checkout holds no scripts/ci/cli.ts to recompute the proof"; exit 2; }
|
|
56
|
+
mkdir -p "$out_dir" || { echo "UNREADABLE: cannot create the proof directory $out_dir"; exit 2; }
|
|
43
57
|
|
|
44
58
|
serial_inventory=$'test\ntest-macos'
|
|
45
59
|
sharded_inventory=$(for os in ubuntu-latest macos-latest; do
|
|
@@ -47,21 +61,24 @@ sharded_inventory=$(for os in ubuntu-latest macos-latest; do
|
|
|
47
61
|
echo "single-fork $os"
|
|
48
62
|
echo "aggregate $os"
|
|
49
63
|
done)
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
64
|
+
# the serial job's declared steps as GitHub names them (an unnamed `run:` step is `Run <command>`)
|
|
65
|
+
serial_steps='Run npm ci
|
|
66
|
+
Run npm run build
|
|
67
|
+
Run npm run lint
|
|
68
|
+
Run coverage proof (parallel suite project)
|
|
69
|
+
Recheck coverage floors
|
|
70
|
+
Run single-fork projects (independent of the step above)
|
|
71
|
+
Assert collected test-file set
|
|
72
|
+
Upload proof group'
|
|
73
|
+
cli=("$checkout/node_modules/.bin/tsx" "$checkout/scripts/ci/cli.ts")
|
|
57
74
|
|
|
58
75
|
verdict=0
|
|
59
76
|
mark_unreadable() { verdict=2; }
|
|
60
77
|
mark_red() { [ "$verdict" -eq 2 ] || verdict=1; }
|
|
61
78
|
mark_killed() { [ "$verdict" -ne 0 ] || verdict=3; }
|
|
62
|
-
|
|
79
|
+
finish() { echo "VERDICT rc=$verdict (0=all GREEN 1=RED 2=UNREADABLE 3=INFRA_KILLED)"; exit "$verdict"; }
|
|
63
80
|
count() { printf '%s' "$1" | grep -c ''; }
|
|
64
|
-
# json <name> — one JSON string; awk -v decodes
|
|
81
|
+
# json <name> — one JSON string; awk -v decodes the @tsv escapes (\t \n \r \\) first
|
|
65
82
|
json() {
|
|
66
83
|
LC_ALL=C awk -v s="$1" 'BEGIN {
|
|
67
84
|
for (i = 1; i < 32; i++) esc[sprintf("%c", i)] = sprintf("\\u%04x", i)
|
|
@@ -71,28 +88,37 @@ json() {
|
|
|
71
88
|
}'
|
|
72
89
|
}
|
|
73
90
|
defect() { echo "INVENTORY_DEFECT kind=$1 job=$(json "$2")"; mark_unreadable; }
|
|
74
|
-
# step_log <log> <step> — the job-log lines gh's step column attributes to <step>, as gh printed them
|
|
75
|
-
step_log() { awk -F'\t' -v s="$2" 'NF >= 3 && $2 == s' "$1"; }
|
|
76
|
-
# verdict_line <log> <step> <family> <format> — the ONE <family> line <step> printed, in <format>;
|
|
77
|
-
# else MISSING, MULTIPLE(n) or MALFORMED, each UNREADABLE
|
|
78
|
-
verdict_line() {
|
|
79
|
-
local lines
|
|
80
|
-
lines=$(step_log "$1" "$2" | awk -v esc="$(printf '\033')" '{
|
|
81
|
-
gsub(/\^\[\[[0-9;]*m/, ""); gsub(esc "\\[[0-9;]*m", "")
|
|
82
|
-
sub(/^[^\t]*\t[^\t]*\t[0-9T:.-]+Z ?/, ""); sub(/[ \t\r]+$/, ""); print
|
|
83
|
-
}' | grep -E "^$3( |\$)")
|
|
84
|
-
case $(count "$lines") in
|
|
85
|
-
0) echo MISSING ;;
|
|
86
|
-
1) if printf '%s\n' "$lines" | grep -qxE "$4"; then printf '%s\n' "$lines"; else echo MALFORMED; fi ;;
|
|
87
|
-
*) echo "MULTIPLE($(count "$lines"))" ;;
|
|
88
|
-
esac
|
|
89
|
-
}
|
|
90
|
-
readable() { case $1 in MISSING|MULTIPLE*|MALFORMED) return 1 ;; esac; }
|
|
91
|
-
floors_green() { printf '%s\n' "$1" | grep -qxE 'COVERAGE_FLOORS verdict=GREEN floors=[0-9]+ failing=0'; }
|
|
92
91
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
92
|
+
run_json=$(gh run view "$run" --repo "$gh_repo" --json headSha,status,conclusion,jobs) \
|
|
93
|
+
|| { echo "UNREADABLE: gh run view $run"; mark_unreadable; finish; }
|
|
94
|
+
# the read as @tsv-escaped rows — RUN <head> <status> <conclusion>, JOB <index> <name> <status> <conclusion>,
|
|
95
|
+
# STEP <job index> <name> <conclusion>; a null or empty value prints `null`; any other shape fails closed
|
|
96
|
+
rows=$(printf '%s' "$run_json" | node -e '
|
|
97
|
+
const field = (v) => v === null || v === undefined || v === "" ? "null"
|
|
98
|
+
: String(v).replace(/\\/g, "\\\\").replace(/\t/g, "\\t").replace(/\n/g, "\\n").replace(/\r/g, "\\r");
|
|
99
|
+
const scalar = (v) => v === null || v === undefined || typeof v === "string" || typeof v === "number";
|
|
100
|
+
let text = "";
|
|
101
|
+
process.stdin.setEncoding("utf8").on("data", (d) => { text += d; }).on("end", () => {
|
|
102
|
+
try {
|
|
103
|
+
const run = JSON.parse(text);
|
|
104
|
+
const object = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
|
|
105
|
+
if (!object(run) || !Array.isArray(run.jobs) || ![run.headSha, run.status, run.conclusion].every(scalar)) throw 0;
|
|
106
|
+
const out = [["RUN", run.headSha, run.status, run.conclusion]];
|
|
107
|
+
run.jobs.forEach((job, i) => {
|
|
108
|
+
if (!object(job) || ![job.name, job.status, job.conclusion].every(scalar)) throw 0;
|
|
109
|
+
if (job.steps !== undefined && job.steps !== null && !Array.isArray(job.steps)) throw 0;
|
|
110
|
+
out.push(["JOB", i, job.name, job.status, job.conclusion]);
|
|
111
|
+
for (const step of job.steps ?? []) {
|
|
112
|
+
if (!object(step) || ![step.name, step.conclusion].every(scalar)) throw 0;
|
|
113
|
+
out.push(["STEP", i, step.name, step.conclusion]);
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
process.stdout.write(out.map((row) => row.map(field).join("\t") + "\n").join(""));
|
|
117
|
+
} catch { process.exit(2); }
|
|
118
|
+
});') || { echo "UNREADABLE: gh run view $run returned no readable run record"; mark_unreadable; finish; }
|
|
119
|
+
IFS=$'\t' read -r _ head status conclusion <<< "$(printf '%s\n' "$rows" | sed -n $'s/^RUN\t/-\t/p')"
|
|
120
|
+
jobs=$(printf '%s\n' "$rows" | sed -n $'s/^JOB\t//p')
|
|
121
|
+
steps=$(printf '%s\n' "$rows" | sed -n $'s/^STEP\t//p')
|
|
96
122
|
# an empty job list is an inventory like any other: serial, observed=0, every declared job missing
|
|
97
123
|
|
|
98
124
|
names=$(printf '%s\n' "$jobs" | cut -f2)
|
|
@@ -113,126 +139,96 @@ while IFS= read -r name; do
|
|
|
113
139
|
printf '%s\n' "$declared" | grep -qxF -- "$name" || defect unknown "$name"
|
|
114
140
|
done < <([ -z "$jobs" ] || printf '%s\n' "$names" | awk '!seen[$0]++')
|
|
115
141
|
|
|
116
|
-
[
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
suite\ *) role=shard ;;
|
|
122
|
-
single-fork\ *) role=single-fork ;;
|
|
123
|
-
*) role=aggregate ;;
|
|
124
|
-
esac
|
|
142
|
+
if [ "$form" = sharded ]; then
|
|
143
|
+
echo "SHARDED: UNREADABLE (the sharded proof-group selection is ANL-022 §7.4's and has not shipped)"
|
|
144
|
+
mark_unreadable
|
|
145
|
+
finish
|
|
146
|
+
fi
|
|
125
147
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
148
|
+
echo "RUN head=$head expected=$sha status=$status conclusion=$conclusion"
|
|
149
|
+
if [ "$head" != "$sha" ]; then
|
|
150
|
+
echo "RUN: UNREADABLE (the run's head is not the expected head; nothing downloaded)"
|
|
151
|
+
mark_unreadable
|
|
152
|
+
finish
|
|
153
|
+
fi
|
|
154
|
+
if [ "$status" != completed ]; then
|
|
155
|
+
echo "RUN: UNREADABLE (the run has not reached run-end; nothing downloaded)"
|
|
156
|
+
mark_unreadable
|
|
157
|
+
finish
|
|
158
|
+
fi
|
|
132
159
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
160
|
+
[ -z "$jobs" ] || while IFS=$'\t' read -r index name job_status job_conclusion; do
|
|
161
|
+
# an unknown or duplicated job has no role to grade: its INVENTORY_DEFECT already made the run UNREADABLE
|
|
162
|
+
[ "$(printf '%s\n' "$names" | grep -cxF -- "$name")" -eq 1 ] && printf '%s\n' "$declared" | grep -qxF -- "$name" || continue
|
|
163
|
+
|
|
164
|
+
echo "$name: status=$job_status conclusion=$job_conclusion"
|
|
165
|
+
if [ "$job_status" != completed ]; then
|
|
166
|
+
echo "$name: UNREADABLE (job has not reached run-end)"
|
|
137
167
|
mark_unreadable
|
|
138
168
|
continue
|
|
139
169
|
fi
|
|
140
170
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
# exit annotation, or — SERIAL only — a failure explained ONLY by steps that exited exactly 1 on a log
|
|
160
|
-
# of their own (gh's step column) that the classifier calls RPC_ONLY. Any other code (2, a signal's
|
|
161
|
-
# 137), any annotation on another role, an unexplained failure, or a cancelled, timed-out or missing
|
|
162
|
-
# conclusion is RED.
|
|
163
|
-
exits=$(grep -E '##\[error\]Process completed with exit code [0-9]+' "$log")
|
|
164
|
-
case "$conclusion" in
|
|
165
|
-
success) outcome=ok ;;
|
|
166
|
-
failure) if [ -n "$exits" ]; then outcome=ok; else outcome=unexplained-failure; fi ;;
|
|
167
|
-
*) outcome="conclusion-${conclusion:-none}" ;;
|
|
168
|
-
esac
|
|
169
|
-
if [ -n "$exits" ]; then
|
|
170
|
-
while IFS= read -r exit_line; do
|
|
171
|
-
code=$(printf '%s\n' "$exit_line" | sed -E 's/.*Process completed with exit code ([0-9]+).*/\1/')
|
|
172
|
-
if [ "$code" != 1 ] || [ "$role" != serial ]; then outcome="exit-$code"; break; fi
|
|
173
|
-
step=$(printf '%s\n' "$exit_line" | awk -F'\t' 'NF >= 3 { print $2 }')
|
|
174
|
-
step_log "$log" "$step" > "$log.step"
|
|
175
|
-
case $(bash "$here/classify-vitest-log.sh" "$log.step" 2>&1) in
|
|
176
|
-
"VITEST_LOG verdict=RPC_ONLY "*) ;;
|
|
177
|
-
*) outcome="exit-1-not-rpc-only"; break ;;
|
|
178
|
-
esac
|
|
179
|
-
done <<< "$exits"
|
|
180
|
-
fi
|
|
181
|
-
|
|
182
|
-
oracles=
|
|
183
|
-
case $role in
|
|
184
|
-
serial)
|
|
185
|
-
oracle=$(verdict_line "$log" "$COUNT_STEP" COUNT_ORACLE "$COUNT_FORMAT")
|
|
186
|
-
floors=$(verdict_line "$log" "$FLOORS_STEP" COVERAGE_FLOORS "$FLOORS_FORMAT")
|
|
187
|
-
oracles="oracle=[$oracle] floors=[$floors] " ;;
|
|
188
|
-
aggregate)
|
|
189
|
-
floors=$(verdict_line "$log" "$FLOORS_STEP" COVERAGE_FLOORS "$FLOORS_FORMAT")
|
|
190
|
-
set_oracle=$(verdict_line "$log" "$SET_STEP" SET_ORACLE "$SET_FORMAT")
|
|
191
|
-
step_log "$log" "$MERGE_STEP" > "$log.merge"
|
|
192
|
-
merge=$(bash "$here/classify-vitest-log.sh" "$log.merge" 2>&1 | sed -n 's/^VITEST_LOG verdict=\([^ ]*\) reason=\([^ ]*\) .*/\1 \2/p')
|
|
193
|
-
oracles="floors=[$floors] set=[$set_oracle] merge=[$merge] " ;;
|
|
194
|
-
esac
|
|
171
|
+
own=$(printf '%s\n' "$steps" | awk -F'\t' -v i="$index" '$1 == i')
|
|
172
|
+
listed=; unread=; failed=; unproven=; killed=
|
|
173
|
+
[ "$job_conclusion" != cancelled ] || killed=1
|
|
174
|
+
while IFS= read -r step; do
|
|
175
|
+
matches=$(printf '%s\n' "$own" | awk -F'\t' -v s="$step" '$2 == s { print $3 }')
|
|
176
|
+
case $(count "$matches") in
|
|
177
|
+
0) outcome=MISSING; unread=1 ;;
|
|
178
|
+
1) outcome=$matches ;;
|
|
179
|
+
*) outcome="MULTIPLE($(count "$matches"))"; unread=1 ;;
|
|
180
|
+
esac
|
|
181
|
+
case $outcome in
|
|
182
|
+
success|MISSING|MULTIPLE*) ;;
|
|
183
|
+
cancelled) killed=1 ;;
|
|
184
|
+
failure) failed=1 ;;
|
|
185
|
+
*) unproven=1 ;; # skipped, or no conclusion: never success
|
|
186
|
+
esac
|
|
187
|
+
listed="$listed${listed:+ | }$step: $outcome"
|
|
188
|
+
done <<< "$serial_steps"
|
|
195
189
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
continue
|
|
203
|
-
elif [ "${kills:-0}" -gt 0 ] 2>/dev/null; then
|
|
204
|
-
grade=RED
|
|
205
|
-
elif [ -z "$log_verdict" ]; then
|
|
206
|
-
grade=UNREADABLE
|
|
190
|
+
proof=
|
|
191
|
+
grade=GREEN
|
|
192
|
+
if [ -n "$unread" ]; then
|
|
193
|
+
grade=UNREADABLE; proof="not downloaded: a declared step is missing or duplicated"
|
|
194
|
+
elif [ -n "$killed" ]; then
|
|
195
|
+
grade=INFRA_KILLED; [ -z "$failed" ] || grade=RED; proof="not downloaded: the job was cancelled"
|
|
207
196
|
else
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
esac
|
|
197
|
+
dir=$(mktemp -d "$out_dir/CI-$(printf '%s' "$tag" | tr -c 'A-Za-z0-9._-' '-')-proof-$name.XXXXXX") \
|
|
198
|
+
&& gh run download "$run" --repo "$gh_repo" -n "proof-$name" -D "$dir" > /dev/null 2>&1
|
|
199
|
+
if [ $? -ne 0 ]; then
|
|
200
|
+
grade=UNREADABLE; proof="MISSING: artifact proof-$name not downloaded"
|
|
201
|
+
else
|
|
202
|
+
proof_out=$("${cli[@]}" proof --head "$sha" --repo "$repo_dir" "$dir" 2> "$dir.stderr")
|
|
203
|
+
proof_status=$?
|
|
204
|
+
printf '%s\n' "$proof_out" | sed -n "s/^PROOF_DEFECT /$name: PROOF_DEFECT /p"
|
|
205
|
+
proof=$(printf '%s\n' "$proof_out" | grep '^PROOF ')
|
|
206
|
+
word=$(printf '%s\n' "$proof" | sed -n 's/^PROOF dir=.* verdict=\([A-Z]*\) head=.*/\1/p')
|
|
207
|
+
# fail closed: one PROOF line, and its word believed only beside its own exit status
|
|
208
|
+
if [ "$(count "$proof")" -ne 1 ]; then
|
|
209
|
+
grade=UNREADABLE; proof="UNREADABLE: cli.ts proof exited $proof_status with $(count "$proof") PROOF lines"
|
|
210
|
+
elif [ "$word:$proof_status" = GREEN:0 ] || [ "$word:$proof_status" = RED:1 ]; then
|
|
211
|
+
grade=$word
|
|
212
|
+
else
|
|
213
|
+
grade=UNREADABLE
|
|
214
|
+
fi
|
|
215
|
+
fi
|
|
216
|
+
if [ "$grade" != UNREADABLE ] && { [ -n "$failed$unproven" ] || [ "$job_conclusion" != success ]; }; then grade=RED; fi
|
|
229
217
|
fi
|
|
218
|
+
echo "$name: steps=[$listed] proof=[$proof]"
|
|
230
219
|
echo "$name: $grade"
|
|
231
220
|
case $grade in
|
|
232
221
|
RED) mark_red ;;
|
|
233
222
|
UNREADABLE) mark_unreadable ;;
|
|
223
|
+
INFRA_KILLED) mark_killed ;;
|
|
234
224
|
esac
|
|
235
225
|
done <<< "$jobs"
|
|
236
226
|
|
|
237
|
-
|
|
238
|
-
|
|
227
|
+
# the run's own conclusion: GREEN needs success; a cancelled run is INFRA_KILLED; any other conclusion is RED
|
|
228
|
+
# unless a job's grade above already accounts for it (a killed job concludes its run `failure` too)
|
|
229
|
+
case $conclusion in
|
|
230
|
+
success) ;;
|
|
231
|
+
cancelled) mark_killed ;;
|
|
232
|
+
*) if [ "$verdict" -eq 0 ]; then echo "RUN: RED (run conclusion $conclusion with every job GREEN)"; mark_red; fi ;;
|
|
233
|
+
esac
|
|
234
|
+
finish
|
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
#!/bin/bash
|
|
2
|
-
# classify-vitest-log.sh <log> [expected-files] — the ONE strict Vitest log classifier
|
|
3
|
-
# (OBS-1184). The public CI wrapper (scripts/run-ci-vitest.sh) and grade-ci.sh both call it, so the badge
|
|
4
|
-
# and the grade cannot disagree. Reads a raw Vitest log or a gh job log (gh's "job<TAB>step<TAB>timestamp "
|
|
5
|
-
# prefix and ANSI styling — the raw ESC byte and gh's `^[` rendering of it — are stripped) and prints ONE line:
|
|
6
|
-
# VITEST_LOG verdict=<CLEAN|RPC_ONLY|RED|INFRA_KILLED> reason=<why> summaries=S passed=P skipped=K failed=F
|
|
7
|
-
# timedout=T unhandled=U runner_rpc_timeouts=R coverage_misses=C signals=G terminal_errors=E runner_kills=X
|
|
8
|
-
# unhandled counts the unhandled errors that are NOT runner RPC timeouts. Exit 0 CLEAN or RPC_ONLY, 1 RED,
|
|
9
|
-
# 3 INFRA_KILLED.
|
|
10
|
-
#
|
|
11
|
-
# INFRA_KILLED (D-897): the GitHub runner's own shutdown or cancellation annotation ("##[error]The runner
|
|
12
|
-
# has received a shutdown signal…" / "##[error]The operation was canceled.") with no failed or timed-out
|
|
13
|
-
# test and no coverage-threshold miss. It is never green and never passes a gate — the proof was not
|
|
14
|
-
# finished — but it is not a test red either. Those failures dominate a kill (RED); a kill excuses only
|
|
15
|
-
# the cut-off or short log it leaves behind (runner_kills counts the annotations either way). A failure
|
|
16
|
-
# the reporter already printed before the kill cut its summary off counts too: a dot-reporter "x" (its
|
|
17
|
-
# stream has no newline, so a test's "stdout | …"/"stderr | …" header may finish the same line), a
|
|
18
|
-
# default-reporter "×" or failed module record "❯ file (N tests…)" (❯ marks a failed module even with
|
|
19
|
-
# no failed test — a collect or hook error), a " FAIL " badge or the "Failed Tests/Suites N" banner (only
|
|
20
|
-
# beside a kill — without one, the complete summaries' own counts decide). Only one printed after the last
|
|
21
|
-
# complete summary counts: a complete summary's own counts settle every line before it. Test output never
|
|
22
|
-
# counts: a console block runs from its header until the reporter speaks again — a dot run, a ⎯ banner or
|
|
23
|
-
# a default-reporter module record (✓/❯/↓ file (N tests…)) right after a blank line, or the next summary —
|
|
24
|
-
# so blank lines inside a test's output do not end it.
|
|
25
|
-
# ponytail: a test that prints, after a blank line of its own, a line the reporter could have printed is
|
|
26
|
-
# indistinguishable from the reporter; only that deliberate forgery escapes the block.
|
|
27
|
-
#
|
|
28
|
-
# The declared contract (D-893), when a caller supplies one: expected-files is the number of test files
|
|
29
|
-
# the run must collect — the summaries' "(N)" totals must sum to it exactly. The caller derives it from
|
|
30
|
-
# the declared projects' own candidate-tree discovery, so a never-started declared project is a short
|
|
31
|
-
# count: project presence comes from that discovery, never from log text a test can print. A short count
|
|
32
|
-
# is RED reason=partial: a complete summary of the projects that happened to start proves nothing about
|
|
33
|
-
# the one that did not. With or without a contract, a summary whose finished files (failed + passed +
|
|
34
|
-
# skipped + todo) fall short of its own "(N)" is RED reason=partial too: Vitest's total counts the queued
|
|
35
|
-
# and running files an abort left unfinished.
|
|
36
|
-
#
|
|
37
|
-
# RED unless every Vitest summary is complete (Test Files, Tests and Duration lines, and every
|
|
38
|
-
# "Unhandled Errors" block closed by the summary that follows it — a cut-off log is RED), with no
|
|
39
|
-
# failed file or test, no timed-out test or hook, no coverage-threshold ERROR, no npm signal line,
|
|
40
|
-
# no unhandled error other than a vitest-worker RPC timeout, and no terminal error: a structural
|
|
41
|
-
# "Unhandled Error", "Startup Error" or "Collect Error" banner outside Vitest's counted block — how
|
|
42
|
-
# Vitest reports an exception thrown after the summary, such as a coverage-reporting failure, before it
|
|
43
|
-
# exits 1 (terminal_errors counts them; an RPC timeout never offsets one). RPC_ONLY means unhandled
|
|
44
|
-
# errors were present and EVERY one is such a timeout (OBS-1058: the starved 2-core runner talking,
|
|
45
|
-
# not a test).
|
|
46
|
-
#
|
|
47
|
-
# Which timeouts count: only one that is ITSELF an unhandled entry — the first payload line under a
|
|
48
|
-
# per-error header, nothing later. The header is Vitest's own structural line — a run of ⎯ on each side
|
|
49
|
-
# of "Unhandled Error", nothing else on the line — so prose that merely contains the words never opens
|
|
50
|
-
# a window, and the payload must be Vitest's whole line (`Error: [vitest-worker]: Timeout calling
|
|
51
|
-
# "<method>"`), so an assertion quoting it stays RED. Per-error headers are honoured only inside
|
|
52
|
-
# Vitest's own block — after its structural "Unhandled Errors" line and "Vitest caught N" tally, until
|
|
53
|
-
# the file summary — and each block's count never exceeds its N, so a test that prints a header and a
|
|
54
|
-
# timeout to stdout forges nothing. The tallies must also sum to the summaries' own "Errors N" lines
|
|
55
|
-
# (Vitest prints both from one list), so a forged or a missing block is RED as well.
|
|
56
|
-
set -u
|
|
57
|
-
|
|
58
|
-
log=${1:?log path required}
|
|
59
|
-
fields='summaries=0 passed=0 skipped=0 failed=0 timedout=0 unhandled=0 runner_rpc_timeouts=0 coverage_misses=0 signals=0 terminal_errors=0 runner_kills=0'
|
|
60
|
-
if [ ! -s "$log" ] || [ ! -r "$log" ]; then
|
|
61
|
-
echo "VITEST_LOG verdict=RED reason=unreadable $fields"
|
|
62
|
-
exit 1
|
|
63
|
-
fi
|
|
64
|
-
|
|
65
|
-
esc=$(printf '\033')
|
|
66
|
-
awk -v esc="$esc" -v expected="${2:-}" '
|
|
67
|
-
function num(s, word) { return match(s, "[0-9]+ " word) ? substr(s, RSTART, RLENGTH) + 0 : 0 }
|
|
68
|
-
function close_block() { rpc += (found < cap ? found : cap); armed = 0; pending = 0 }
|
|
69
|
-
{
|
|
70
|
-
line = $0
|
|
71
|
-
gsub(/\^\[\[[0-9;]*m/, "", line); gsub(esc "\\[[0-9;]*m", "", line)
|
|
72
|
-
sub(/^[^\t]*\t[^\t]*\t[0-9T:.-]+Z[ ]?/, "", line)
|
|
73
|
-
if (line ~ /^[ \t]*(·|x|\*|-)*(stdout|stderr) \| /) {
|
|
74
|
-
if (line ~ /^[ \t]*(·|\*|-)*x/) reported++
|
|
75
|
-
console = 1
|
|
76
|
-
} else if (after_blank && line ~ /^[ \t]*((·|x|\*|-)*(·|x)(·|x|\*|-)*|(⎯)+( .* (⎯)+)?|(✓|❯|↓) .* [(][0-9]+ tests?( [|] [0-9]+ (failed|skipped))*[)].*)[ \t]*$/) console = 0
|
|
77
|
-
if (!console && line ~ /^[ \t]*((·|\*|-)*x(·|x|\*|-)*|× .*|❯ .* [(][0-9]+ tests?( [|] [0-9]+ (failed|skipped))*[)].*|FAIL[ \t].*|(⎯)+ Failed (Tests|Suites) [0-9]+ (⎯)+)[ \t]*$/) reported++
|
|
78
|
-
after_blank = line !~ /[^ \t]/
|
|
79
|
-
}
|
|
80
|
-
line ~ /^##\[error\](The runner has received a shutdown signal|The operation was cancell?ed)/ { kills++ }
|
|
81
|
-
# The Vitest timeout message carries its millisecond count; a test that PRINTS a fingerprint normalized
|
|
82
|
-
# to #ms (the daemon OBS-1106 notifications) is output, not a timed-out test. A real timeout also fails
|
|
83
|
-
# its test, so the summary failed count still reds it.
|
|
84
|
-
line ~ /(Test|Hook) timed out in [0-9]+ ?ms/ { timedout++ }
|
|
85
|
-
line ~ /ERROR: Coverage for .* does not meet .*threshold/ { coverage++ }
|
|
86
|
-
line ~ /^npm (error|ERR!) signal / { signals++ }
|
|
87
|
-
line ~ /^(⎯)+ Unhandled Errors (⎯)+[ \t]*$/ { if (armed) close_block(); opening = 1; next }
|
|
88
|
-
opening && line !~ /[^ \t]/ { next }
|
|
89
|
-
opening {
|
|
90
|
-
opening = 0
|
|
91
|
-
if (line ~ /^Vitest caught [0-9]+ unhandled error/) { armed = 1; found = 0; cap = num(line, "unhandled"); tally += cap }
|
|
92
|
-
else untallied++
|
|
93
|
-
}
|
|
94
|
-
line ~ /^[ \t]*Test Files[ \t]/ {
|
|
95
|
-
if (armed) close_block()
|
|
96
|
-
summaries++; in_summary = 1; saw_tests = 0; console = 0
|
|
97
|
-
failed += num(line, "failed"); passed += num(line, "passed"); skipped += num(line, "skipped")
|
|
98
|
-
finished = num(line, "failed") + num(line, "passed") + num(line, "skipped") + num(line, "todo")
|
|
99
|
-
total = match(line, /\([0-9]+\)[ \t]*$/) ? substr(line, RSTART + 1) + 0 : -1
|
|
100
|
-
if (total >= 0) collected += total
|
|
101
|
-
if (finished != total) unfinished++
|
|
102
|
-
next
|
|
103
|
-
}
|
|
104
|
-
in_summary && line ~ /^[ \t]*Tests[ \t]/ { saw_tests = 1; failed_tests += num(line, "failed"); next }
|
|
105
|
-
in_summary && line ~ /^[ \t]*Errors[ \t]+[0-9]+ error/ { summary_errors += num(line, "error"); next }
|
|
106
|
-
in_summary && line ~ /^[ \t]*Duration[ \t]/ { if (saw_tests) { complete++; reported = 0 }; in_summary = 0; next }
|
|
107
|
-
armed && line ~ /^(⎯)+ Unhandled Error (⎯)+[ \t]*$/ { pending = 1; next }
|
|
108
|
-
line ~ /^(⎯)+ (Unhandled|Startup|Collect) Error (⎯)+[ \t]*$/ { terminal++; next }
|
|
109
|
-
pending && line ~ /[^ \t]/ { pending = 0; if (line ~ /^Error: \[vitest-worker\]: Timeout calling "[A-Za-z]+"[ \t]*$/) found++ }
|
|
110
|
-
END {
|
|
111
|
-
unhandled = tally + untallied
|
|
112
|
-
if (summary_errors > unhandled) unhandled = summary_errors
|
|
113
|
-
other = unhandled - rpc
|
|
114
|
-
partial = unfinished || (expected != "" && collected != expected + 0)
|
|
115
|
-
# a kill always names a class: the failure that dominates it, or the kill itself
|
|
116
|
-
if (kills) reason = (failed || failed_tests || reported) ? "failed" : timedout ? "timed-out" : coverage ? "coverage-threshold" : "runner-shutdown"
|
|
117
|
-
else if (summaries == 0) reason = "no-summary"
|
|
118
|
-
else if (complete != summaries || opening || armed) reason = "truncated"
|
|
119
|
-
else if (failed || failed_tests) reason = "failed"
|
|
120
|
-
else if (timedout) reason = "timed-out"
|
|
121
|
-
else if (partial) reason = "partial"
|
|
122
|
-
else if (coverage) reason = "coverage-threshold"
|
|
123
|
-
else if (signals) reason = "signal"
|
|
124
|
-
else if (terminal) reason = "terminal-error"
|
|
125
|
-
else if (other > 0) reason = "unhandled-error"
|
|
126
|
-
else if (tally + untallied != summary_errors) reason = "unhandled-mismatch"
|
|
127
|
-
verdict = reason == "runner-shutdown" ? "INFRA_KILLED" : (reason != "" ? "RED" : (unhandled > 0 ? "RPC_ONLY" : "CLEAN"))
|
|
128
|
-
if (reason == "") reason = "none"
|
|
129
|
-
printf "VITEST_LOG verdict=%s reason=%s summaries=%d passed=%d skipped=%d failed=%d timedout=%d unhandled=%d runner_rpc_timeouts=%d coverage_misses=%d signals=%d terminal_errors=%d runner_kills=%d\n", \
|
|
130
|
-
verdict, reason, summaries, passed, skipped, failed, timedout, other, rpc, coverage, signals, terminal, kills
|
|
131
|
-
exit (verdict == "INFRA_KILLED" ? 3 : verdict == "RED")
|
|
132
|
-
}
|
|
133
|
-
' "$log"
|