peaks-loop 4.1.21 → 4.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +3 -6
- package/CHANGELOG.md +191 -0
- package/dist/cli/commands/mut-commands-helpers.d.ts +6 -6
- package/dist/cli/commands/mut-commands-helpers.js +44 -26
- package/dist/cli/commands/observability-commands-hotspots.d.ts +54 -0
- package/dist/cli/commands/observability-commands-hotspots.js +222 -0
- package/dist/cli/commands/observability-commands-near-miss.d.ts +28 -0
- package/dist/cli/commands/observability-commands-near-miss.js +105 -0
- package/dist/cli/commands/observability-commands-queries.d.ts +61 -18
- package/dist/cli/commands/observability-commands-queries.js +88 -47
- package/dist/cli/commands/observability-commands.js +79 -20
- package/dist/services/artifacts/artifact-prerequisites.d.ts +3 -0
- package/dist/services/artifacts/artifact-prerequisites.js +1 -1
- package/dist/services/artifacts/prerequisite-path-census.d.ts +42 -0
- package/dist/services/artifacts/prerequisite-path-census.js +79 -0
- package/dist/services/artifacts/prerequisite-transition-event.js +13 -1
- package/dist/services/artifacts/rd-gate-event.d.ts +106 -0
- package/dist/services/artifacts/rd-gate-event.js +211 -0
- package/dist/services/artifacts/rd-transition-gates.d.ts +6 -0
- package/dist/services/artifacts/rd-transition-gates.js +65 -24
- package/dist/services/distribution/json-layout.d.ts +20 -0
- package/dist/services/distribution/json-layout.js +195 -0
- package/dist/services/distribution/plugin-manifest.d.ts +31 -2
- package/dist/services/distribution/plugin-manifest.js +40 -4
- package/dist/services/observability/aggregation-types.d.ts +21 -0
- package/dist/services/observability/aggregation.d.ts +12 -2
- package/dist/services/observability/aggregation.js +26 -2
- package/dist/services/observability/compliance-judge.d.ts +77 -0
- package/dist/services/observability/compliance-judge.js +143 -0
- package/dist/services/observability/compliance-spec.d.ts +96 -0
- package/dist/services/observability/compliance-spec.js +29 -0
- package/dist/services/observability/compliance-specs/peaks-race-code.spec.d.ts +37 -0
- package/dist/services/observability/compliance-specs/peaks-race-code.spec.js +66 -0
- package/dist/services/observability/compliance-specs/peaks-rd.spec.d.ts +50 -0
- package/dist/services/observability/compliance-specs/peaks-rd.spec.js +111 -0
- package/dist/services/observability/hotspot-attribution-summary.d.ts +84 -0
- package/dist/services/observability/hotspot-attribution-summary.js +74 -0
- package/dist/services/observability/hotspot-rows.d.ts +47 -0
- package/dist/services/observability/hotspot-rows.js +89 -0
- package/dist/services/observability/hotspot-shared.d.ts +212 -0
- package/dist/services/observability/hotspot-shared.js +122 -0
- package/dist/services/observability/hotspot-sources.d.ts +55 -0
- package/dist/services/observability/hotspot-sources.js +142 -0
- package/dist/services/observability/hotspots.d.ts +47 -0
- package/dist/services/observability/hotspots.js +47 -0
- package/dist/services/observability/near-miss-attempts.d.ts +175 -0
- package/dist/services/observability/near-miss-attempts.js +223 -0
- package/dist/services/observability/near-miss-coverage.d.ts +109 -0
- package/dist/services/observability/near-miss-coverage.js +108 -0
- package/dist/services/observability/near-miss-denominator.d.ts +81 -0
- package/dist/services/observability/near-miss-denominator.js +75 -0
- package/dist/services/observability/near-miss-difference.d.ts +98 -0
- package/dist/services/observability/near-miss-difference.js +132 -0
- package/dist/services/observability/near-miss-facts.d.ts +165 -0
- package/dist/services/observability/near-miss-facts.js +101 -0
- package/dist/services/observability/near-miss-gate-events.d.ts +75 -0
- package/dist/services/observability/near-miss-gate-events.js +106 -0
- package/dist/services/observability/near-miss-gates.d.ts +108 -0
- package/dist/services/observability/near-miss-gates.js +135 -0
- package/dist/services/observability/near-miss-records.d.ts +93 -0
- package/dist/services/observability/near-miss-records.js +87 -0
- package/dist/services/observability/near-miss.d.ts +148 -0
- package/dist/services/observability/near-miss.js +196 -0
- package/dist/services/observability/observability-schema.d.ts +17 -2
- package/dist/services/observability/observability-schema.js +21 -1
- package/dist/services/observability/owner-skill-map.d.ts +92 -0
- package/dist/services/observability/owner-skill-map.js +144 -0
- package/dist/services/observability/report-formatter-hotspots.d.ts +32 -0
- package/dist/services/observability/report-formatter-hotspots.js +76 -0
- package/dist/services/observability/report-formatter-near-miss.d.ts +48 -0
- package/dist/services/observability/report-formatter-near-miss.js +234 -0
- package/dist/services/observability/report-formatter.d.ts +49 -3
- package/dist/services/observability/report-formatter.js +55 -14
- package/dist/services/observability/retro-attribution.d.ts +129 -0
- package/dist/services/observability/retro-attribution.js +134 -0
- package/dist/services/observability/session-metric-rows.d.ts +153 -0
- package/dist/services/observability/session-metric-rows.js +184 -0
- package/package.json +5 -5
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/hotspots.ts`
|
|
3
|
+
*
|
|
4
|
+
* The hotspot ranking: which artifact does a skill keep failing to deliver,
|
|
5
|
+
* and who owns it.
|
|
6
|
+
*
|
|
7
|
+
* WHY THIS EXISTS. The programme's premise is that the QA verdict says
|
|
8
|
+
* "pass" without saying WHICH check was skipped, so a reader cannot go and
|
|
9
|
+
* fix the skill that is actually failing. This ranking names the artifact and
|
|
10
|
+
* its owner, so the next step ("raise this to a hook") has a target.
|
|
11
|
+
*
|
|
12
|
+
* TWO SOURCES, AND THE CONFIDENCE COLUMN SAYS WHICH. A FORWARD event (v2)
|
|
13
|
+
* carries the missing list as it was MEASURED: the path, the mode and the
|
|
14
|
+
* owner are all facts, and the row's confidence is `exact`. A HISTORICAL
|
|
15
|
+
* event (v1) carries only a COUNT, so the artifact is a CANDIDATE recovered
|
|
16
|
+
* by the retro-attribution, and the row's confidence is `inferred` (the count
|
|
17
|
+
* matched the whole prerequisite list) or `ambiguous` (the count did not
|
|
18
|
+
* localise to one entry). The spec (§2.2 S2.2) puts all three tiers in one
|
|
19
|
+
* ranking and requires them to be LABELLED, because an `inferred` row says
|
|
20
|
+
* "this artifact was in the missing set", NOT "this artifact was proven
|
|
21
|
+
* missing". A reader who sorts by count must be able to see which rows are
|
|
22
|
+
* measurements and which are reconstructions.
|
|
23
|
+
*
|
|
24
|
+
* WHY THE ROW SAYS WHAT ITS COUNT COUNTS. The confidence label names HOW the
|
|
25
|
+
* row was known, but a reader sorting by `count` sees a NUMBER and reaches
|
|
26
|
+
* for the obvious meaning — how many times the artifact was missing. For a
|
|
27
|
+
* reconstructed row that meaning is wrong: one real failure contributes to
|
|
28
|
+
* every candidate the attribution could not narrow, so the count is CANDIDATE
|
|
29
|
+
* MENTIONS. The row therefore carries `countMeaning` beside the count, so the
|
|
30
|
+
* number cannot be read the wrong way without the reader having to know the
|
|
31
|
+
* rule.
|
|
32
|
+
*
|
|
33
|
+
* A ROW WITH NO OWNER IS STILL A ROW. An artifact whose directory the
|
|
34
|
+
* owner map does not know is a NEW artifact type, and a new artifact type is
|
|
35
|
+
* exactly what a reader needs to see — the gate is that it is COUNTED, not
|
|
36
|
+
* that it is attributed. Dropping it would hide a whole class of failures
|
|
37
|
+
* behind a mapping gap, which is the failure mode the owner map exists to
|
|
38
|
+
* prevent.
|
|
39
|
+
*
|
|
40
|
+
* WHY THE READERS LIVE NEXT DOOR. `hotspot-rows.ts` holds the reading of
|
|
41
|
+
* the event stream, so this file states what a row MEANS and what the
|
|
42
|
+
* ranking guarantees — the same split `aggregation.ts` and
|
|
43
|
+
* `aggregation-helpers.ts` make in this directory.
|
|
44
|
+
*/
|
|
45
|
+
export { aggregateHotspots } from './hotspot-rows.js';
|
|
46
|
+
export { summarizeAttribution } from './hotspot-attribution-summary.js';
|
|
47
|
+
export { canonicalArtifact, countMeaningOf } from './hotspot-shared.js';
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/near-miss-attempts.ts`
|
|
3
|
+
*
|
|
4
|
+
* The prereq ATTEMPTS a slice's events record, the moment the slice was
|
|
5
|
+
* declared PASSING, and the keys whose check was still unsatisfied at that
|
|
6
|
+
* moment.
|
|
7
|
+
*
|
|
8
|
+
* WHY A SEPARATE MODULE. The near-miss metric needs to know, for one slice,
|
|
9
|
+
* when it was declared passing and which required checks had not been
|
|
10
|
+
* satisfied by then. That is a reading of the event stream, and it is the
|
|
11
|
+
* reading the `prereq-skipped` reason is built on — so it lives next door, and
|
|
12
|
+
* `near-miss.ts` holds the metric.
|
|
13
|
+
*
|
|
14
|
+
* WHY THE STREAM HOLDS TWO EVENTS PER ATTEMPT. A transition attempt produces
|
|
15
|
+
* TWO `slice-transition` events: the one the prereq check itself emits,
|
|
16
|
+
* carrying the verdict (`{artifactRole, to, prereqOk, missingCount}`), and the
|
|
17
|
+
* one the transition emits after the artifact is written, carrying the
|
|
18
|
+
* arrival (`{from, to, artifactRole}`) and NO verdict. Reading the second as
|
|
19
|
+
* an attempt would double-count every attempt and pair a failure with an
|
|
20
|
+
* unrelated transition, so the attempt IS the verdict event and the verdict
|
|
21
|
+
* -less one is ignored.
|
|
22
|
+
*
|
|
23
|
+
* WHY THE REFERENCE MOMENT IS THE SUCCESSFUL PASS. The question the near-miss
|
|
24
|
+
* literature asks is "at the moment the slice was declared passing, had the
|
|
25
|
+
* required check been satisfied?" — so the reference is the PASS, not the
|
|
26
|
+
* first `verdict-issued` event and not the last attempt at a key. A
|
|
27
|
+
* `verdict-issued` attempt whose prereq check FAILED is not a pass at all; it
|
|
28
|
+
* is a refused transition recorded under the same `to` state. The pass moment
|
|
29
|
+
* is therefore the first `verdict-issued` attempt whose prereq check
|
|
30
|
+
* SUCCEEDED, and a slice with no such attempt never passed — it is in
|
|
31
|
+
* neither the numerator nor the denominator.
|
|
32
|
+
*
|
|
33
|
+
* WHY A KEY IS "UNSATISFIED" RATHER THAN "FAILED". A check that refused and
|
|
34
|
+
* was then satisfied on a retry is a gate working as designed, and the
|
|
35
|
+
* earlier question ("was there ever a failure?") cannot tell the two apart.
|
|
36
|
+
* What the near-miss question asks is whether the key was satisfied WHEN THE
|
|
37
|
+
* SLICE PASSED — and a key whose only successful attempt comes after the pass
|
|
38
|
+
* moment was still unsatisfied at the moment the slice was declared passing.
|
|
39
|
+
*
|
|
40
|
+
* WHY THE KEY IS `artifactRole -> to`. A retry re-runs the SAME check: the
|
|
41
|
+
* same artifact role, moving to the same state. Pairing on the key is what
|
|
42
|
+
* says "this later success answered that earlier failure". Pairing on time
|
|
43
|
+
* alone would pair a failure with any later success, including one for a
|
|
44
|
+
* different transition.
|
|
45
|
+
*/
|
|
46
|
+
import type { ObservabilityEvent } from './observability-service.js';
|
|
47
|
+
/** The state a QA verdict is issued at. */
|
|
48
|
+
export declare const VERDICT_STATE = "verdict-issued";
|
|
49
|
+
/** One prereq verdict the stream recorded for a transition attempt. */
|
|
50
|
+
export type PrereqAttempt = {
|
|
51
|
+
/** The position of the attempt in the slice's event stream. */
|
|
52
|
+
readonly index: number;
|
|
53
|
+
/** The transition key: `artifactRole -> to`. */
|
|
54
|
+
readonly key: string;
|
|
55
|
+
/** Whether the check passed. */
|
|
56
|
+
readonly passed: boolean;
|
|
57
|
+
readonly sessionId: string;
|
|
58
|
+
readonly ts: string;
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* The state of one transition key AT THE PASS MOMENT.
|
|
62
|
+
*
|
|
63
|
+
* WHY THE STATE IS NOT ONE THING. "The check was unsatisfied" covers two
|
|
64
|
+
* materially different situations, and a reader has to be able to tell them
|
|
65
|
+
* apart:
|
|
66
|
+
*
|
|
67
|
+
* - `refused` — the check WAS run at the pass moment and said no. The gate
|
|
68
|
+
* was live, it fired, and the slice passed anyway.
|
|
69
|
+
* - `unattempted` — the check had NOT been run at the pass moment at all.
|
|
70
|
+
* The role that owns the check had not yet attempted its transition.
|
|
71
|
+
*
|
|
72
|
+
* On this repository's recorded history one of the twelve findings is
|
|
73
|
+
* `refused` and eleven are `unattempted` — and the second is the more alarming
|
|
74
|
+
* of the two, because it means the pass was declared before the owning role
|
|
75
|
+
* had even tried. Calling all twelve "refusing" would send a reader hunting
|
|
76
|
+
* for a gate that was firing when it was not.
|
|
77
|
+
*/
|
|
78
|
+
export type UnsatisfiedKeyState = 'refused' | 'unattempted';
|
|
79
|
+
/**
|
|
80
|
+
* The transition key an event belongs to, or null when it names no transition.
|
|
81
|
+
*
|
|
82
|
+
* The key is the pair that identifies the CHECK, not the attempt: a retry
|
|
83
|
+
* re-runs the same check, so the key is what pairs a failure with the
|
|
84
|
+
* success that later answered it.
|
|
85
|
+
*/
|
|
86
|
+
export declare function attemptKey(event: ObservabilityEvent): string | null;
|
|
87
|
+
/**
|
|
88
|
+
* The prereq verdict an event carries, or null when it carries none.
|
|
89
|
+
*
|
|
90
|
+
* `null` is the verdict-LESS event: the transition itself, which says where
|
|
91
|
+
* the slice went but nothing about whether the check let it go.
|
|
92
|
+
*/
|
|
93
|
+
export declare function prereqVerdictOf(event: ObservabilityEvent): boolean | null;
|
|
94
|
+
/**
|
|
95
|
+
* The `verdict-issued` attempt of a slice whose prereq verdict this build
|
|
96
|
+
* cannot READ, or null when there is none.
|
|
97
|
+
*
|
|
98
|
+
* WHY THIS IS NOT "NO VERDICT". The event schema keeps `detail` loose, so a
|
|
99
|
+
* record CAN carry a `prereqOk` that is not a boolean — the string `"true"`,
|
|
100
|
+
* a number, a null. The two cases are different facts: a record with no
|
|
101
|
+
* `prereqOk` at all is the verdict-LESS arrival event, while a record with a
|
|
102
|
+
* `prereqOk` this build cannot read is a verdict that was RECORDED and is
|
|
103
|
+
* UNREADABLE. Reading the second as the first makes the slice vanish: it has
|
|
104
|
+
* no pass moment, so it is in neither the numerator nor the denominator, and
|
|
105
|
+
* no coverage count mentions it.
|
|
106
|
+
*
|
|
107
|
+
* THE SCOPE IS THE VERDICT TRANSITION, because that is the attempt that
|
|
108
|
+
* decides whether the slice passed at all. A malformed verdict on another
|
|
109
|
+
* transition does not make the slice's pass unreadable.
|
|
110
|
+
*
|
|
111
|
+
* `beforeIndex` bounds the search to the attempts that come BEFORE the pass
|
|
112
|
+
* moment: the reference moment is the FIRST successful `verdict-issued`
|
|
113
|
+
* attempt, so an unreadable attempt after it cannot have decided the pass.
|
|
114
|
+
*/
|
|
115
|
+
export declare function unreadableVerdictOf(events: readonly ObservabilityEvent[], beforeIndex?: number): UnreadableVerdict | null;
|
|
116
|
+
/** A prereq verdict the record carries in a shape this build cannot read. */
|
|
117
|
+
export type UnreadableVerdict = {
|
|
118
|
+
/** The transition key: `artifactRole -> to`. */
|
|
119
|
+
readonly key: string;
|
|
120
|
+
readonly sessionId: string;
|
|
121
|
+
readonly ts: string;
|
|
122
|
+
/** The value the record carried, spelled out so a reader can see it. */
|
|
123
|
+
readonly value: string;
|
|
124
|
+
};
|
|
125
|
+
/** The attempts a slice's events record, in stream order. */
|
|
126
|
+
export declare function collectAttempts(events: readonly ObservabilityEvent[]): readonly PrereqAttempt[];
|
|
127
|
+
/**
|
|
128
|
+
* The moment the slice was declared PASSING.
|
|
129
|
+
*
|
|
130
|
+
* The pass moment is the first `verdict-issued` attempt whose prereq check
|
|
131
|
+
* SUCCEEDED. A `verdict-issued` attempt whose check failed is a refused
|
|
132
|
+
* transition, not a pass, so it does not end the wait — the slice was still
|
|
133
|
+
* trying when it refused.
|
|
134
|
+
*
|
|
135
|
+
* The POSITION is returned alongside the timestamp because two events can
|
|
136
|
+
* share a millisecond, and the comparison this moment is used for — "was
|
|
137
|
+
* this key satisfied by then?" — is a question about order, which a
|
|
138
|
+
* timestamp cannot answer when the clock does not advance.
|
|
139
|
+
*/
|
|
140
|
+
export declare function passMomentOf(attempts: readonly PrereqAttempt[]): {
|
|
141
|
+
readonly index: number;
|
|
142
|
+
readonly ts: string;
|
|
143
|
+
} | null;
|
|
144
|
+
/**
|
|
145
|
+
* The keys whose check was STILL UNSATISFIED when the slice was declared
|
|
146
|
+
* passing, each with its state at that moment.
|
|
147
|
+
*
|
|
148
|
+
* A key qualifies when it recorded at least one failed attempt AND had no
|
|
149
|
+
* successful attempt at or before the pass moment. The absence of a success
|
|
150
|
+
* is what shows it was still unsatisfied when the slice passed.
|
|
151
|
+
*
|
|
152
|
+
* THE STATE AT THE PASS MOMENT IS NOT ONE THING, and the difference is the
|
|
153
|
+
* finding:
|
|
154
|
+
*
|
|
155
|
+
* - `refused` — the check had run at the pass moment and said no. The gate
|
|
156
|
+
* was live, it fired, and the slice passed anyway.
|
|
157
|
+
* - `unattempted` — the check had NOT run at the pass moment. The owning
|
|
158
|
+
* role had not yet attempted its transition, and the slice was declared
|
|
159
|
+
* passing without it.
|
|
160
|
+
*
|
|
161
|
+
* On this repository's recorded history one of the twelve findings is
|
|
162
|
+
* `refused` and eleven are `unattempted`. Calling all twelve "refusing"
|
|
163
|
+
* would send a reader after a gate that was firing when it was not.
|
|
164
|
+
*
|
|
165
|
+
* The failure is not required to PRECEDE the pass moment: a key whose only
|
|
166
|
+
* failure came after the pass is still a key that was unsatisfied when the
|
|
167
|
+
* slice was declared passing, and that is the finding.
|
|
168
|
+
*/
|
|
169
|
+
export declare function unsatisfiedKeys(attempts: readonly PrereqAttempt[], passIndex: number): readonly {
|
|
170
|
+
readonly key: string;
|
|
171
|
+
readonly state: UnsatisfiedKeyState;
|
|
172
|
+
readonly refusalCount: number;
|
|
173
|
+
}[];
|
|
174
|
+
/** The failed attempts of one unsatisfied key — the evidence for the finding. */
|
|
175
|
+
export declare function failuresOf(attempts: readonly PrereqAttempt[], key: string): readonly PrereqAttempt[];
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/near-miss-attempts.ts`
|
|
3
|
+
*
|
|
4
|
+
* The prereq ATTEMPTS a slice's events record, the moment the slice was
|
|
5
|
+
* declared PASSING, and the keys whose check was still unsatisfied at that
|
|
6
|
+
* moment.
|
|
7
|
+
*
|
|
8
|
+
* WHY A SEPARATE MODULE. The near-miss metric needs to know, for one slice,
|
|
9
|
+
* when it was declared passing and which required checks had not been
|
|
10
|
+
* satisfied by then. That is a reading of the event stream, and it is the
|
|
11
|
+
* reading the `prereq-skipped` reason is built on — so it lives next door, and
|
|
12
|
+
* `near-miss.ts` holds the metric.
|
|
13
|
+
*
|
|
14
|
+
* WHY THE STREAM HOLDS TWO EVENTS PER ATTEMPT. A transition attempt produces
|
|
15
|
+
* TWO `slice-transition` events: the one the prereq check itself emits,
|
|
16
|
+
* carrying the verdict (`{artifactRole, to, prereqOk, missingCount}`), and the
|
|
17
|
+
* one the transition emits after the artifact is written, carrying the
|
|
18
|
+
* arrival (`{from, to, artifactRole}`) and NO verdict. Reading the second as
|
|
19
|
+
* an attempt would double-count every attempt and pair a failure with an
|
|
20
|
+
* unrelated transition, so the attempt IS the verdict event and the verdict
|
|
21
|
+
* -less one is ignored.
|
|
22
|
+
*
|
|
23
|
+
* WHY THE REFERENCE MOMENT IS THE SUCCESSFUL PASS. The question the near-miss
|
|
24
|
+
* literature asks is "at the moment the slice was declared passing, had the
|
|
25
|
+
* required check been satisfied?" — so the reference is the PASS, not the
|
|
26
|
+
* first `verdict-issued` event and not the last attempt at a key. A
|
|
27
|
+
* `verdict-issued` attempt whose prereq check FAILED is not a pass at all; it
|
|
28
|
+
* is a refused transition recorded under the same `to` state. The pass moment
|
|
29
|
+
* is therefore the first `verdict-issued` attempt whose prereq check
|
|
30
|
+
* SUCCEEDED, and a slice with no such attempt never passed — it is in
|
|
31
|
+
* neither the numerator nor the denominator.
|
|
32
|
+
*
|
|
33
|
+
* WHY A KEY IS "UNSATISFIED" RATHER THAN "FAILED". A check that refused and
|
|
34
|
+
* was then satisfied on a retry is a gate working as designed, and the
|
|
35
|
+
* earlier question ("was there ever a failure?") cannot tell the two apart.
|
|
36
|
+
* What the near-miss question asks is whether the key was satisfied WHEN THE
|
|
37
|
+
* SLICE PASSED — and a key whose only successful attempt comes after the pass
|
|
38
|
+
* moment was still unsatisfied at the moment the slice was declared passing.
|
|
39
|
+
*
|
|
40
|
+
* WHY THE KEY IS `artifactRole -> to`. A retry re-runs the SAME check: the
|
|
41
|
+
* same artifact role, moving to the same state. Pairing on the key is what
|
|
42
|
+
* says "this later success answered that earlier failure". Pairing on time
|
|
43
|
+
* alone would pair a failure with any later success, including one for a
|
|
44
|
+
* different transition.
|
|
45
|
+
*/
|
|
46
|
+
/** The state a QA verdict is issued at. */
|
|
47
|
+
export const VERDICT_STATE = 'verdict-issued';
|
|
48
|
+
/**
|
|
49
|
+
* The transition key an event belongs to, or null when it names no transition.
|
|
50
|
+
*
|
|
51
|
+
* The key is the pair that identifies the CHECK, not the attempt: a retry
|
|
52
|
+
* re-runs the same check, so the key is what pairs a failure with the
|
|
53
|
+
* success that later answered it.
|
|
54
|
+
*/
|
|
55
|
+
export function attemptKey(event) {
|
|
56
|
+
const detail = event.detail;
|
|
57
|
+
const role = typeof detail.artifactRole === 'string' ? detail.artifactRole : null;
|
|
58
|
+
const to = typeof detail.to === 'string' ? detail.to : null;
|
|
59
|
+
if (role === null || to === null)
|
|
60
|
+
return null;
|
|
61
|
+
return `${role} -> ${to}`;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* The prereq verdict an event carries, or null when it carries none.
|
|
65
|
+
*
|
|
66
|
+
* `null` is the verdict-LESS event: the transition itself, which says where
|
|
67
|
+
* the slice went but nothing about whether the check let it go.
|
|
68
|
+
*/
|
|
69
|
+
export function prereqVerdictOf(event) {
|
|
70
|
+
const detail = event.detail;
|
|
71
|
+
return typeof detail.prereqOk === 'boolean' ? detail.prereqOk : null;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* The `verdict-issued` attempt of a slice whose prereq verdict this build
|
|
75
|
+
* cannot READ, or null when there is none.
|
|
76
|
+
*
|
|
77
|
+
* WHY THIS IS NOT "NO VERDICT". The event schema keeps `detail` loose, so a
|
|
78
|
+
* record CAN carry a `prereqOk` that is not a boolean — the string `"true"`,
|
|
79
|
+
* a number, a null. The two cases are different facts: a record with no
|
|
80
|
+
* `prereqOk` at all is the verdict-LESS arrival event, while a record with a
|
|
81
|
+
* `prereqOk` this build cannot read is a verdict that was RECORDED and is
|
|
82
|
+
* UNREADABLE. Reading the second as the first makes the slice vanish: it has
|
|
83
|
+
* no pass moment, so it is in neither the numerator nor the denominator, and
|
|
84
|
+
* no coverage count mentions it.
|
|
85
|
+
*
|
|
86
|
+
* THE SCOPE IS THE VERDICT TRANSITION, because that is the attempt that
|
|
87
|
+
* decides whether the slice passed at all. A malformed verdict on another
|
|
88
|
+
* transition does not make the slice's pass unreadable.
|
|
89
|
+
*
|
|
90
|
+
* `beforeIndex` bounds the search to the attempts that come BEFORE the pass
|
|
91
|
+
* moment: the reference moment is the FIRST successful `verdict-issued`
|
|
92
|
+
* attempt, so an unreadable attempt after it cannot have decided the pass.
|
|
93
|
+
*/
|
|
94
|
+
export function unreadableVerdictOf(events, beforeIndex) {
|
|
95
|
+
for (const [index, event] of events.entries()) {
|
|
96
|
+
if (beforeIndex !== undefined && index >= beforeIndex)
|
|
97
|
+
return null;
|
|
98
|
+
if (event.category !== 'slice-transition')
|
|
99
|
+
continue;
|
|
100
|
+
const detail = event.detail;
|
|
101
|
+
if (detail.to !== VERDICT_STATE)
|
|
102
|
+
continue;
|
|
103
|
+
if (!('prereqOk' in detail))
|
|
104
|
+
continue;
|
|
105
|
+
if (typeof detail.prereqOk === 'boolean')
|
|
106
|
+
continue;
|
|
107
|
+
return {
|
|
108
|
+
key: attemptKey(event) ?? `-> ${VERDICT_STATE}`,
|
|
109
|
+
sessionId: event.sessionId,
|
|
110
|
+
ts: event.ts,
|
|
111
|
+
value: describePrereqValue(detail.prereqOk)
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* How a prereq verdict that is not a boolean is described to a reader.
|
|
118
|
+
*
|
|
119
|
+
* The value is named, not summarised: a reader who is told "the verdict was
|
|
120
|
+
* unreadable" has to open the record to learn what it actually said, and
|
|
121
|
+
* `"true"` read as a boolean is exactly the reading this build must not make.
|
|
122
|
+
*/
|
|
123
|
+
function describePrereqValue(value) {
|
|
124
|
+
if (value === null)
|
|
125
|
+
return 'null';
|
|
126
|
+
if (typeof value === 'string')
|
|
127
|
+
return `the string "${value}"`;
|
|
128
|
+
if (typeof value === 'number')
|
|
129
|
+
return `the number ${String(value)}`;
|
|
130
|
+
if (typeof value === 'boolean')
|
|
131
|
+
return `the boolean ${String(value)}`;
|
|
132
|
+
return `a ${typeof value}`;
|
|
133
|
+
}
|
|
134
|
+
/** The attempts a slice's events record, in stream order. */
|
|
135
|
+
export function collectAttempts(events) {
|
|
136
|
+
const attempts = [];
|
|
137
|
+
for (const [index, event] of events.entries()) {
|
|
138
|
+
if (event.category !== 'slice-transition')
|
|
139
|
+
continue;
|
|
140
|
+
const verdict = prereqVerdictOf(event);
|
|
141
|
+
if (verdict === null)
|
|
142
|
+
continue;
|
|
143
|
+
const key = attemptKey(event);
|
|
144
|
+
if (key === null)
|
|
145
|
+
continue;
|
|
146
|
+
attempts.push({ index, key, passed: verdict, sessionId: event.sessionId, ts: event.ts });
|
|
147
|
+
}
|
|
148
|
+
return attempts;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* The moment the slice was declared PASSING.
|
|
152
|
+
*
|
|
153
|
+
* The pass moment is the first `verdict-issued` attempt whose prereq check
|
|
154
|
+
* SUCCEEDED. A `verdict-issued` attempt whose check failed is a refused
|
|
155
|
+
* transition, not a pass, so it does not end the wait — the slice was still
|
|
156
|
+
* trying when it refused.
|
|
157
|
+
*
|
|
158
|
+
* The POSITION is returned alongside the timestamp because two events can
|
|
159
|
+
* share a millisecond, and the comparison this moment is used for — "was
|
|
160
|
+
* this key satisfied by then?" — is a question about order, which a
|
|
161
|
+
* timestamp cannot answer when the clock does not advance.
|
|
162
|
+
*/
|
|
163
|
+
export function passMomentOf(attempts) {
|
|
164
|
+
const pass = attempts.find((attempt) => attempt.key.endsWith(`-> ${VERDICT_STATE}`) && attempt.passed);
|
|
165
|
+
return pass === undefined ? null : { index: pass.index, ts: pass.ts };
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* The keys whose check was STILL UNSATISFIED when the slice was declared
|
|
169
|
+
* passing, each with its state at that moment.
|
|
170
|
+
*
|
|
171
|
+
* A key qualifies when it recorded at least one failed attempt AND had no
|
|
172
|
+
* successful attempt at or before the pass moment. The absence of a success
|
|
173
|
+
* is what shows it was still unsatisfied when the slice passed.
|
|
174
|
+
*
|
|
175
|
+
* THE STATE AT THE PASS MOMENT IS NOT ONE THING, and the difference is the
|
|
176
|
+
* finding:
|
|
177
|
+
*
|
|
178
|
+
* - `refused` — the check had run at the pass moment and said no. The gate
|
|
179
|
+
* was live, it fired, and the slice passed anyway.
|
|
180
|
+
* - `unattempted` — the check had NOT run at the pass moment. The owning
|
|
181
|
+
* role had not yet attempted its transition, and the slice was declared
|
|
182
|
+
* passing without it.
|
|
183
|
+
*
|
|
184
|
+
* On this repository's recorded history one of the twelve findings is
|
|
185
|
+
* `refused` and eleven are `unattempted`. Calling all twelve "refusing"
|
|
186
|
+
* would send a reader after a gate that was firing when it was not.
|
|
187
|
+
*
|
|
188
|
+
* The failure is not required to PRECEDE the pass moment: a key whose only
|
|
189
|
+
* failure came after the pass is still a key that was unsatisfied when the
|
|
190
|
+
* slice was declared passing, and that is the finding.
|
|
191
|
+
*/
|
|
192
|
+
export function unsatisfiedKeys(attempts, passIndex) {
|
|
193
|
+
const keys = [];
|
|
194
|
+
const byKey = new Map();
|
|
195
|
+
for (const attempt of attempts) {
|
|
196
|
+
const bucket = byKey.get(attempt.key);
|
|
197
|
+
if (bucket === undefined)
|
|
198
|
+
byKey.set(attempt.key, [attempt]);
|
|
199
|
+
else
|
|
200
|
+
bucket.push(attempt);
|
|
201
|
+
}
|
|
202
|
+
for (const [key, bucket] of byKey) {
|
|
203
|
+
const refusals = bucket.filter((attempt) => !attempt.passed);
|
|
204
|
+
if (refusals.length === 0)
|
|
205
|
+
continue;
|
|
206
|
+
const satisfiedByPass = bucket.some((attempt) => attempt.passed && attempt.index <= passIndex);
|
|
207
|
+
if (satisfiedByPass)
|
|
208
|
+
continue;
|
|
209
|
+
// The state is read at the PASS MOMENT, not over the whole history: a
|
|
210
|
+
// check that refused only AFTER the pass had not run at all by then.
|
|
211
|
+
const refusedAtPass = refusals.some((attempt) => attempt.index <= passIndex);
|
|
212
|
+
keys.push({
|
|
213
|
+
key,
|
|
214
|
+
state: refusedAtPass ? 'refused' : 'unattempted',
|
|
215
|
+
refusalCount: refusals.length
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
return keys;
|
|
219
|
+
}
|
|
220
|
+
/** The failed attempts of one unsatisfied key — the evidence for the finding. */
|
|
221
|
+
export function failuresOf(attempts, key) {
|
|
222
|
+
return attempts.filter((attempt) => attempt.key === key && !attempt.passed);
|
|
223
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/near-miss-coverage.ts`
|
|
3
|
+
*
|
|
4
|
+
* What the near-miss metric could and could not OBSERVE.
|
|
5
|
+
*
|
|
6
|
+
* WHY THIS EXISTS. The near-miss rule has three classes, and each class is
|
|
7
|
+
* read from a different kind of event. A scope that holds none of a class's
|
|
8
|
+
* events cannot produce a finding in that class — and a rate that reads as
|
|
9
|
+
* "0 near-misses" when the class was never observable is exactly the failure
|
|
10
|
+
* mode this whole programme exists to close: an ABSENCE read as a RESULT.
|
|
11
|
+
*
|
|
12
|
+
* So the report carries, per class: whether the class had events to observe,
|
|
13
|
+
* how many there were, how many slices they reached, and the rate the class
|
|
14
|
+
* supports. A class with no events has a `null` rate, not a zero. A consumer
|
|
15
|
+
* that reads the rate alone can still be misled, so the coverage travels with
|
|
16
|
+
* it in the same envelope.
|
|
17
|
+
*
|
|
18
|
+
* WHY THE CLASS RATE'S DENOMINATOR IS THE OBSERVED SLICES. A class rate over
|
|
19
|
+
* ALL the passing slices would divide a finding by a population the class
|
|
20
|
+
* could not judge — a gate class that saw five slices out of a hundred would
|
|
21
|
+
* report a rate of one in a hundred, which reads as "rare" when it is "one in
|
|
22
|
+
* the five we could look at". The observed population is the one the finding
|
|
23
|
+
* was drawn from, and it is reported alongside, so a consumer can compute the
|
|
24
|
+
* other reading if it wants it.
|
|
25
|
+
*
|
|
26
|
+
* WHY THE SCOPE COUNT AND THE SLICE COUNT ARE BOTH HERE. The scope count says
|
|
27
|
+
* how much evidence the class had at all; the slice count says how much of
|
|
28
|
+
* the passing population it reached. A class with a hundred events on one
|
|
29
|
+
* slice is observable and reaches one slice — a very different fact from a
|
|
30
|
+
* class with a hundred events spread over a hundred slices.
|
|
31
|
+
*
|
|
32
|
+
* WHY THE SLICE-LEVEL UNOBSERVABILITY IS HERE TOO. `observable` is a SCOPE
|
|
33
|
+
* fact: the class had events somewhere. The rate, though, is computed over the
|
|
34
|
+
* slices the class reached — and the slices it did NOT reach are exactly the
|
|
35
|
+
* ones a skipped check would hide in, because a slice that recorded nothing
|
|
36
|
+
* about the class cannot be judged on it at all. So each class carries the
|
|
37
|
+
* count of passing slices that produced nothing of its kind: a reader who sees
|
|
38
|
+
* a rate of 100% over one slice must also see how many slices said nothing.
|
|
39
|
+
*/
|
|
40
|
+
import type { NearMissReason, SliceFacts } from './near-miss-facts.js';
|
|
41
|
+
/** What one reason class could and could not be observed for. */
|
|
42
|
+
export type NearMissClassCoverage = {
|
|
43
|
+
readonly reason: NearMissReason;
|
|
44
|
+
/**
|
|
45
|
+
* Whether the class had any evidence at all.
|
|
46
|
+
*
|
|
47
|
+
* `false` means the class was NOT OBSERVABLE in this scope, so its zero is
|
|
48
|
+
* an absence of data and not a result.
|
|
49
|
+
*/
|
|
50
|
+
readonly observable: boolean;
|
|
51
|
+
/** How many events of this class's kind the scope held. */
|
|
52
|
+
readonly scopeEventCount: number;
|
|
53
|
+
/** How many of the passing slices the class reached. */
|
|
54
|
+
readonly observedSliceCount: number;
|
|
55
|
+
/**
|
|
56
|
+
* How many of the passing slices produced NO event of this class's kind.
|
|
57
|
+
*
|
|
58
|
+
* These slices could not be judged on the class at all: a slice that
|
|
59
|
+
* recorded nothing about it cannot be found to have skipped it either. The
|
|
60
|
+
* count travels with the rate so the rate can never be read without the
|
|
61
|
+
* slices it could not speak for.
|
|
62
|
+
*/
|
|
63
|
+
readonly unobservedSliceCount: number;
|
|
64
|
+
/**
|
|
65
|
+
* How many UNMEASURED slices' verdict for this class could not be read.
|
|
66
|
+
*
|
|
67
|
+
* These slices recorded the class's kind of event, but the record carries a
|
|
68
|
+
* value this build cannot read — so the slice is in neither the numerator
|
|
69
|
+
* nor the denominator, and the class's rate does not cover it. The count
|
|
70
|
+
* travels with the rate so the rate can never be read without them.
|
|
71
|
+
*
|
|
72
|
+
* THE SCOPE IS THE UNMEASURED POPULATION, not every slice with an
|
|
73
|
+
* unreadable verdict: a slice that PASSED on a readable verdict is in the
|
|
74
|
+
* denominator, and the class's rate covers it.
|
|
75
|
+
*/
|
|
76
|
+
readonly unreadableSliceCount: number;
|
|
77
|
+
/** How many slices this class flagged. */
|
|
78
|
+
readonly nearMissSliceCount: number;
|
|
79
|
+
/**
|
|
80
|
+
* The rate this class supports over the slices it reached, or `null` when
|
|
81
|
+
* the class reached none.
|
|
82
|
+
*/
|
|
83
|
+
readonly rate: number | null;
|
|
84
|
+
};
|
|
85
|
+
/** The classes the metric knows, in a fixed order. */
|
|
86
|
+
export declare const NEAR_MISS_REASONS: readonly NearMissReason[];
|
|
87
|
+
/** The inputs the coverage is computed over. */
|
|
88
|
+
export type CoverageInput = {
|
|
89
|
+
/** Every event in the scope. */
|
|
90
|
+
readonly events: readonly {
|
|
91
|
+
category: string;
|
|
92
|
+
}[];
|
|
93
|
+
/** The facts of every PASSING slice — the metric's denominator. */
|
|
94
|
+
readonly passingFacts: readonly SliceFacts[];
|
|
95
|
+
/** The reasons each near-miss slice carries. */
|
|
96
|
+
readonly reasonsBySlice: ReadonlyMap<string, readonly NearMissReason[]>;
|
|
97
|
+
/** The facts of every slice whose PASS verdict could not be read. */
|
|
98
|
+
readonly unmeasuredFacts: readonly SliceFacts[];
|
|
99
|
+
};
|
|
100
|
+
/**
|
|
101
|
+
* The coverage of each reason class over the scope.
|
|
102
|
+
*
|
|
103
|
+
* A class with no evidence in the scope is reported as NOT OBSERVABLE with a
|
|
104
|
+
* `null` rate, so a zero can never be read as a result the class did not
|
|
105
|
+
* produce.
|
|
106
|
+
*/
|
|
107
|
+
export declare function computeCoverage(input: CoverageInput): readonly NearMissClassCoverage[];
|
|
108
|
+
/** The classes that could not be observed, as the phrases a reader needs. */
|
|
109
|
+
export declare function unobservableReasons(coverage: readonly NearMissClassCoverage[]): readonly NearMissReason[];
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/near-miss-coverage.ts`
|
|
3
|
+
*
|
|
4
|
+
* What the near-miss metric could and could not OBSERVE.
|
|
5
|
+
*
|
|
6
|
+
* WHY THIS EXISTS. The near-miss rule has three classes, and each class is
|
|
7
|
+
* read from a different kind of event. A scope that holds none of a class's
|
|
8
|
+
* events cannot produce a finding in that class — and a rate that reads as
|
|
9
|
+
* "0 near-misses" when the class was never observable is exactly the failure
|
|
10
|
+
* mode this whole programme exists to close: an ABSENCE read as a RESULT.
|
|
11
|
+
*
|
|
12
|
+
* So the report carries, per class: whether the class had events to observe,
|
|
13
|
+
* how many there were, how many slices they reached, and the rate the class
|
|
14
|
+
* supports. A class with no events has a `null` rate, not a zero. A consumer
|
|
15
|
+
* that reads the rate alone can still be misled, so the coverage travels with
|
|
16
|
+
* it in the same envelope.
|
|
17
|
+
*
|
|
18
|
+
* WHY THE CLASS RATE'S DENOMINATOR IS THE OBSERVED SLICES. A class rate over
|
|
19
|
+
* ALL the passing slices would divide a finding by a population the class
|
|
20
|
+
* could not judge — a gate class that saw five slices out of a hundred would
|
|
21
|
+
* report a rate of one in a hundred, which reads as "rare" when it is "one in
|
|
22
|
+
* the five we could look at". The observed population is the one the finding
|
|
23
|
+
* was drawn from, and it is reported alongside, so a consumer can compute the
|
|
24
|
+
* other reading if it wants it.
|
|
25
|
+
*
|
|
26
|
+
* WHY THE SCOPE COUNT AND THE SLICE COUNT ARE BOTH HERE. The scope count says
|
|
27
|
+
* how much evidence the class had at all; the slice count says how much of
|
|
28
|
+
* the passing population it reached. A class with a hundred events on one
|
|
29
|
+
* slice is observable and reaches one slice — a very different fact from a
|
|
30
|
+
* class with a hundred events spread over a hundred slices.
|
|
31
|
+
*
|
|
32
|
+
* WHY THE SLICE-LEVEL UNOBSERVABILITY IS HERE TOO. `observable` is a SCOPE
|
|
33
|
+
* fact: the class had events somewhere. The rate, though, is computed over the
|
|
34
|
+
* slices the class reached — and the slices it did NOT reach are exactly the
|
|
35
|
+
* ones a skipped check would hide in, because a slice that recorded nothing
|
|
36
|
+
* about the class cannot be judged on it at all. So each class carries the
|
|
37
|
+
* count of passing slices that produced nothing of its kind: a reader who sees
|
|
38
|
+
* a rate of 100% over one slice must also see how many slices said nothing.
|
|
39
|
+
*/
|
|
40
|
+
/** The classes the metric knows, in a fixed order. */
|
|
41
|
+
export const NEAR_MISS_REASONS = [
|
|
42
|
+
'gate-never-evaluated',
|
|
43
|
+
'prereq-skipped',
|
|
44
|
+
'tool-blocked'
|
|
45
|
+
];
|
|
46
|
+
/** The evidence each class is read from. */
|
|
47
|
+
function evidenceOf(events) {
|
|
48
|
+
const gateEvents = events.filter((event) => event.category === 'gate').length;
|
|
49
|
+
const toolTraces = events.filter((event) => event.category === 'tool-trace').length;
|
|
50
|
+
const prereqAttempts = events.filter((event) => {
|
|
51
|
+
if (event.category !== 'slice-transition')
|
|
52
|
+
return false;
|
|
53
|
+
const detail = event.detail;
|
|
54
|
+
return typeof detail?.prereqOk === 'boolean';
|
|
55
|
+
}).length;
|
|
56
|
+
return [
|
|
57
|
+
{
|
|
58
|
+
reason: 'gate-never-evaluated',
|
|
59
|
+
scopeEventCount: gateEvents,
|
|
60
|
+
produced: (facts) => facts.observed.gateEvents,
|
|
61
|
+
unreadable: (facts) => facts.unreadableGates.size > 0
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
reason: 'prereq-skipped',
|
|
65
|
+
scopeEventCount: prereqAttempts,
|
|
66
|
+
produced: (facts) => facts.observed.prereqAttempts,
|
|
67
|
+
unreadable: (facts) => facts.unreadableVerdict !== null
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
reason: 'tool-blocked',
|
|
71
|
+
scopeEventCount: toolTraces,
|
|
72
|
+
produced: (facts) => facts.observed.toolTraces,
|
|
73
|
+
unreadable: () => false
|
|
74
|
+
}
|
|
75
|
+
];
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* The coverage of each reason class over the scope.
|
|
79
|
+
*
|
|
80
|
+
* A class with no evidence in the scope is reported as NOT OBSERVABLE with a
|
|
81
|
+
* `null` rate, so a zero can never be read as a result the class did not
|
|
82
|
+
* produce.
|
|
83
|
+
*/
|
|
84
|
+
export function computeCoverage(input) {
|
|
85
|
+
const evidence = evidenceOf(input.events);
|
|
86
|
+
return evidence.map((item) => {
|
|
87
|
+
const produced = input.passingFacts.map((facts) => item.produced(facts));
|
|
88
|
+
const observedSliceCount = produced.filter((count) => count > 0).length;
|
|
89
|
+
const unobservedSliceCount = produced.filter((count) => count === 0).length;
|
|
90
|
+
const unreadableSliceCount = input.unmeasuredFacts.filter((facts) => item.unreadable(facts)).length;
|
|
91
|
+
const nearMissSliceCount = [...input.reasonsBySlice.values()].filter((reasons) => reasons.includes(item.reason)).length;
|
|
92
|
+
const observable = item.scopeEventCount > 0;
|
|
93
|
+
return {
|
|
94
|
+
reason: item.reason,
|
|
95
|
+
observable,
|
|
96
|
+
scopeEventCount: item.scopeEventCount,
|
|
97
|
+
observedSliceCount,
|
|
98
|
+
unobservedSliceCount,
|
|
99
|
+
unreadableSliceCount,
|
|
100
|
+
nearMissSliceCount,
|
|
101
|
+
rate: observedSliceCount === 0 ? null : nearMissSliceCount / observedSliceCount
|
|
102
|
+
};
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
/** The classes that could not be observed, as the phrases a reader needs. */
|
|
106
|
+
export function unobservableReasons(coverage) {
|
|
107
|
+
return coverage.filter((item) => !item.observable).map((item) => item.reason);
|
|
108
|
+
}
|