peaks-loop 4.1.22 → 4.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +12 -2
- package/CHANGELOG.md +184 -0
- package/dist/cli/commands/code-auto-compact-command.js +19 -1
- package/dist/cli/commands/code-gate-step-08-command.js +103 -105
- package/dist/cli/commands/code-gate-step-08-remedy.d.ts +10 -2
- package/dist/cli/commands/code-gate-step-08-remedy.js +11 -3
- package/dist/cli/commands/gate-commands-enforce.d.ts +23 -8
- package/dist/cli/commands/gate-commands-enforce.js +27 -12
- package/dist/cli/commands/observability-commands-hotspots.d.ts +54 -0
- package/dist/cli/commands/observability-commands-hotspots.js +222 -0
- package/dist/cli/commands/observability-commands-near-miss.d.ts +28 -0
- package/dist/cli/commands/observability-commands-near-miss.js +105 -0
- package/dist/cli/commands/observability-commands-queries.d.ts +61 -18
- package/dist/cli/commands/observability-commands-queries.js +88 -47
- package/dist/cli/commands/observability-commands-session-row.d.ts +48 -0
- package/dist/cli/commands/observability-commands-session-row.js +201 -0
- package/dist/cli/commands/observability-commands.js +81 -20
- package/dist/services/artifacts/artifact-prerequisites.d.ts +3 -0
- package/dist/services/artifacts/artifact-prerequisites.js +1 -1
- package/dist/services/artifacts/prerequisite-path-census.d.ts +47 -0
- package/dist/services/artifacts/prerequisite-path-census.js +100 -0
- package/dist/services/artifacts/prerequisite-path-resolver.js +32 -2
- package/dist/services/artifacts/prerequisite-transition-event.js +13 -1
- package/dist/services/artifacts/rd-gate-event.d.ts +106 -0
- package/dist/services/artifacts/rd-gate-event.js +211 -0
- package/dist/services/artifacts/rd-transition-gates.d.ts +6 -0
- package/dist/services/artifacts/rd-transition-gates.js +65 -24
- package/dist/services/hooks/output.d.ts +67 -1
- package/dist/services/hooks/output.js +74 -1
- package/dist/services/observability/aggregation-types.d.ts +21 -0
- package/dist/services/observability/aggregation.d.ts +12 -2
- package/dist/services/observability/aggregation.js +28 -3
- package/dist/services/observability/compliance-judge.d.ts +77 -0
- package/dist/services/observability/compliance-judge.js +143 -0
- package/dist/services/observability/compliance-spec.d.ts +96 -0
- package/dist/services/observability/compliance-spec.js +29 -0
- package/dist/services/observability/compliance-specs/peaks-race-code.spec.d.ts +37 -0
- package/dist/services/observability/compliance-specs/peaks-race-code.spec.js +66 -0
- package/dist/services/observability/compliance-specs/peaks-rd.spec.d.ts +50 -0
- package/dist/services/observability/compliance-specs/peaks-rd.spec.js +111 -0
- package/dist/services/observability/hotspot-attribution-summary.d.ts +84 -0
- package/dist/services/observability/hotspot-attribution-summary.js +74 -0
- package/dist/services/observability/hotspot-rows.d.ts +47 -0
- package/dist/services/observability/hotspot-rows.js +89 -0
- package/dist/services/observability/hotspot-shared.d.ts +207 -0
- package/dist/services/observability/hotspot-shared.js +127 -0
- package/dist/services/observability/hotspot-sources.d.ts +55 -0
- package/dist/services/observability/hotspot-sources.js +142 -0
- package/dist/services/observability/hotspots.d.ts +47 -0
- package/dist/services/observability/hotspots.js +47 -0
- package/dist/services/observability/near-miss-attempts.d.ts +175 -0
- package/dist/services/observability/near-miss-attempts.js +223 -0
- package/dist/services/observability/near-miss-coverage.d.ts +109 -0
- package/dist/services/observability/near-miss-coverage.js +108 -0
- package/dist/services/observability/near-miss-denominator.d.ts +81 -0
- package/dist/services/observability/near-miss-denominator.js +75 -0
- package/dist/services/observability/near-miss-difference.d.ts +98 -0
- package/dist/services/observability/near-miss-difference.js +135 -0
- package/dist/services/observability/near-miss-facts.d.ts +165 -0
- package/dist/services/observability/near-miss-facts.js +101 -0
- package/dist/services/observability/near-miss-gate-events.d.ts +75 -0
- package/dist/services/observability/near-miss-gate-events.js +106 -0
- package/dist/services/observability/near-miss-gates.d.ts +108 -0
- package/dist/services/observability/near-miss-gates.js +135 -0
- package/dist/services/observability/near-miss-records.d.ts +93 -0
- package/dist/services/observability/near-miss-records.js +87 -0
- package/dist/services/observability/near-miss.d.ts +148 -0
- package/dist/services/observability/near-miss.js +203 -0
- package/dist/services/observability/observability-schema.d.ts +17 -2
- package/dist/services/observability/observability-schema.js +21 -1
- package/dist/services/observability/owner-skill-map.d.ts +92 -0
- package/dist/services/observability/owner-skill-map.js +164 -0
- package/dist/services/observability/repair-cycle-metric.js +6 -3
- package/dist/services/observability/report-formatter-hotspots.d.ts +32 -0
- package/dist/services/observability/report-formatter-hotspots.js +76 -0
- package/dist/services/observability/report-formatter-near-miss.d.ts +48 -0
- package/dist/services/observability/report-formatter-near-miss.js +234 -0
- package/dist/services/observability/report-formatter.d.ts +49 -3
- package/dist/services/observability/report-formatter.js +55 -14
- package/dist/services/observability/retro-attribution.d.ts +129 -0
- package/dist/services/observability/retro-attribution.js +134 -0
- package/dist/services/observability/session-metric-rows.d.ts +153 -0
- package/dist/services/observability/session-metric-rows.js +184 -0
- package/dist/services/skills/presence-role.js +9 -2
- package/dist/services/standards/loop-engineering-readiness-lint.d.ts +15 -1
- package/dist/services/standards/loop-engineering-readiness-lint.js +27 -9
- package/dist/shared/incrementing-number.d.ts +57 -0
- package/dist/shared/incrementing-number.js +13 -0
- package/dist/skills/peaks-maker/index.js +43 -30
- package/output-styles/peaks-skill-swarm.md +2 -2
- package/package.json +5 -5
- package/scripts/install-skills-bundles.mjs +6 -1
- package/scripts/install-skills.mjs +10 -1
- package/skills/bee/peaks-perf-audit/SKILL.md +1 -1
- package/skills/bee/peaks-prd/SKILL.md +1 -1
- package/skills/bee/peaks-qa/SKILL.md +1 -1
- package/skills/bee/peaks-rd/SKILL.md +1 -1
- package/skills/bee/peaks-reviewer/SKILL.md +1 -1
- package/skills/bee/peaks-sc/SKILL.md +1 -1
- package/skills/bee/peaks-security-audit/SKILL.md +1 -1
- package/skills/bee/peaks-txt/SKILL.md +1 -1
- package/skills/bee/peaks-ui/SKILL.md +1 -1
- package/skills/peaks-audit/SKILL.md +1 -1
- package/skills/peaks-code/SKILL.md +1 -1
- package/skills/peaks-content/SKILL.md +1 -1
- package/skills/peaks-doctor/SKILL.md +1 -1
- package/skills/peaks-final-review/SKILL.md +1 -1
- package/skills/peaks-ide/SKILL.md +1 -1
- package/skills/peaks-issue-fix-orchestrator/SKILL.md +1 -1
- package/skills/peaks-maker/SKILL.md +125 -0
- package/skills/peaks-mut/SKILL.md +297 -0
- package/skills/peaks-race-code/SKILL.md +1 -1
- package/skills/peaks-resume/SKILL.md +1 -1
- package/skills/peaks-slice-decompose/SKILL.md +1 -1
- package/skills/peaks-solo/SKILL.md +1 -1
- package/skills/peaks-sop/SKILL.md +1 -1
- package/skills/peaks-status/SKILL.md +1 -1
- package/skills/peaks-test/SKILL.md +1 -1
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/near-miss-coverage.ts`
|
|
3
|
+
*
|
|
4
|
+
* What the near-miss metric could and could not OBSERVE.
|
|
5
|
+
*
|
|
6
|
+
* WHY THIS EXISTS. The near-miss rule has three classes, and each class is
|
|
7
|
+
* read from a different kind of event. A scope that holds none of a class's
|
|
8
|
+
* events cannot produce a finding in that class — and a rate that reads as
|
|
9
|
+
* "0 near-misses" when the class was never observable is exactly the failure
|
|
10
|
+
* mode this whole programme exists to close: an ABSENCE read as a RESULT.
|
|
11
|
+
*
|
|
12
|
+
* So the report carries, per class: whether the class had events to observe,
|
|
13
|
+
* how many there were, how many slices they reached, and the rate the class
|
|
14
|
+
* supports. A class with no events has a `null` rate, not a zero. A consumer
|
|
15
|
+
* that reads the rate alone can still be misled, so the coverage travels with
|
|
16
|
+
* it in the same envelope.
|
|
17
|
+
*
|
|
18
|
+
* WHY THE CLASS RATE'S DENOMINATOR IS THE OBSERVED SLICES. A class rate over
|
|
19
|
+
* ALL the passing slices would divide a finding by a population the class
|
|
20
|
+
* could not judge — a gate class that saw five slices out of a hundred would
|
|
21
|
+
* report a rate of one in a hundred, which reads as "rare" when it is "one in
|
|
22
|
+
* the five we could look at". The observed population is the one the finding
|
|
23
|
+
* was drawn from, and it is reported alongside, so a consumer can compute the
|
|
24
|
+
* other reading if it wants it.
|
|
25
|
+
*
|
|
26
|
+
* WHY THE SCOPE COUNT AND THE SLICE COUNT ARE BOTH HERE. The scope count says
|
|
27
|
+
* how much evidence the class had at all; the slice count says how much of
|
|
28
|
+
* the passing population it reached. A class with a hundred events on one
|
|
29
|
+
* slice is observable and reaches one slice — a very different fact from a
|
|
30
|
+
* class with a hundred events spread over a hundred slices.
|
|
31
|
+
*
|
|
32
|
+
* WHY THE SLICE-LEVEL UNOBSERVABILITY IS HERE TOO. `observable` is a SCOPE
|
|
33
|
+
* fact: the class had events somewhere. The rate, though, is computed over the
|
|
34
|
+
* slices the class reached — and the slices it did NOT reach are exactly the
|
|
35
|
+
* ones a skipped check would hide in, because a slice that recorded nothing
|
|
36
|
+
* about the class cannot be judged on it at all. So each class carries the
|
|
37
|
+
* count of passing slices that produced nothing of its kind: a reader who sees
|
|
38
|
+
* a rate of 100% over one slice must also see how many slices said nothing.
|
|
39
|
+
*/
|
|
40
|
+
import type { NearMissReason, SliceFacts } from './near-miss-facts.js';
|
|
41
|
+
/** What one reason class could and could not be observed for. */
|
|
42
|
+
export type NearMissClassCoverage = {
|
|
43
|
+
readonly reason: NearMissReason;
|
|
44
|
+
/**
|
|
45
|
+
* Whether the class had any evidence at all.
|
|
46
|
+
*
|
|
47
|
+
* `false` means the class was NOT OBSERVABLE in this scope, so its zero is
|
|
48
|
+
* an absence of data and not a result.
|
|
49
|
+
*/
|
|
50
|
+
readonly observable: boolean;
|
|
51
|
+
/** How many events of this class's kind the scope held. */
|
|
52
|
+
readonly scopeEventCount: number;
|
|
53
|
+
/** How many of the passing slices the class reached. */
|
|
54
|
+
readonly observedSliceCount: number;
|
|
55
|
+
/**
|
|
56
|
+
* How many of the passing slices produced NO event of this class's kind.
|
|
57
|
+
*
|
|
58
|
+
* These slices could not be judged on the class at all: a slice that
|
|
59
|
+
* recorded nothing about it cannot be found to have skipped it either. The
|
|
60
|
+
* count travels with the rate so the rate can never be read without the
|
|
61
|
+
* slices it could not speak for.
|
|
62
|
+
*/
|
|
63
|
+
readonly unobservedSliceCount: number;
|
|
64
|
+
/**
|
|
65
|
+
* How many UNMEASURED slices' verdict for this class could not be read.
|
|
66
|
+
*
|
|
67
|
+
* These slices recorded the class's kind of event, but the record carries a
|
|
68
|
+
* value this build cannot read — so the slice is in neither the numerator
|
|
69
|
+
* nor the denominator, and the class's rate does not cover it. The count
|
|
70
|
+
* travels with the rate so the rate can never be read without them.
|
|
71
|
+
*
|
|
72
|
+
* THE SCOPE IS THE UNMEASURED POPULATION, not every slice with an
|
|
73
|
+
* unreadable verdict: a slice that PASSED on a readable verdict is in the
|
|
74
|
+
* denominator, and the class's rate covers it.
|
|
75
|
+
*/
|
|
76
|
+
readonly unreadableSliceCount: number;
|
|
77
|
+
/** How many slices this class flagged. */
|
|
78
|
+
readonly nearMissSliceCount: number;
|
|
79
|
+
/**
|
|
80
|
+
* The rate this class supports over the slices it reached, or `null` when
|
|
81
|
+
* the class reached none.
|
|
82
|
+
*/
|
|
83
|
+
readonly rate: number | null;
|
|
84
|
+
};
|
|
85
|
+
/** The classes the metric knows, in a fixed order. */
|
|
86
|
+
export declare const NEAR_MISS_REASONS: readonly NearMissReason[];
|
|
87
|
+
/** The inputs the coverage is computed over. */
|
|
88
|
+
export type CoverageInput = {
|
|
89
|
+
/** Every event in the scope. */
|
|
90
|
+
readonly events: readonly {
|
|
91
|
+
category: string;
|
|
92
|
+
}[];
|
|
93
|
+
/** The facts of every PASSING slice — the metric's denominator. */
|
|
94
|
+
readonly passingFacts: readonly SliceFacts[];
|
|
95
|
+
/** The reasons each near-miss slice carries. */
|
|
96
|
+
readonly reasonsBySlice: ReadonlyMap<string, readonly NearMissReason[]>;
|
|
97
|
+
/** The facts of every slice whose PASS verdict could not be read. */
|
|
98
|
+
readonly unmeasuredFacts: readonly SliceFacts[];
|
|
99
|
+
};
|
|
100
|
+
/**
|
|
101
|
+
* The coverage of each reason class over the scope.
|
|
102
|
+
*
|
|
103
|
+
* A class with no evidence in the scope is reported as NOT OBSERVABLE with a
|
|
104
|
+
* `null` rate, so a zero can never be read as a result the class did not
|
|
105
|
+
* produce.
|
|
106
|
+
*/
|
|
107
|
+
export declare function computeCoverage(input: CoverageInput): readonly NearMissClassCoverage[];
|
|
108
|
+
/** The classes that could not be observed, as the phrases a reader needs. */
|
|
109
|
+
export declare function unobservableReasons(coverage: readonly NearMissClassCoverage[]): readonly NearMissReason[];
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/near-miss-coverage.ts`
|
|
3
|
+
*
|
|
4
|
+
* What the near-miss metric could and could not OBSERVE.
|
|
5
|
+
*
|
|
6
|
+
* WHY THIS EXISTS. The near-miss rule has three classes, and each class is
|
|
7
|
+
* read from a different kind of event. A scope that holds none of a class's
|
|
8
|
+
* events cannot produce a finding in that class — and a rate that reads as
|
|
9
|
+
* "0 near-misses" when the class was never observable is exactly the failure
|
|
10
|
+
* mode this whole programme exists to close: an ABSENCE read as a RESULT.
|
|
11
|
+
*
|
|
12
|
+
* So the report carries, per class: whether the class had events to observe,
|
|
13
|
+
* how many there were, how many slices they reached, and the rate the class
|
|
14
|
+
* supports. A class with no events has a `null` rate, not a zero. A consumer
|
|
15
|
+
* that reads the rate alone can still be misled, so the coverage travels with
|
|
16
|
+
* it in the same envelope.
|
|
17
|
+
*
|
|
18
|
+
* WHY THE CLASS RATE'S DENOMINATOR IS THE OBSERVED SLICES. A class rate over
|
|
19
|
+
* ALL the passing slices would divide a finding by a population the class
|
|
20
|
+
* could not judge — a gate class that saw five slices out of a hundred would
|
|
21
|
+
* report a rate of one in a hundred, which reads as "rare" when it is "one in
|
|
22
|
+
* the five we could look at". The observed population is the one the finding
|
|
23
|
+
* was drawn from, and it is reported alongside, so a consumer can compute the
|
|
24
|
+
* other reading if it wants it.
|
|
25
|
+
*
|
|
26
|
+
* WHY THE SCOPE COUNT AND THE SLICE COUNT ARE BOTH HERE. The scope count says
|
|
27
|
+
* how much evidence the class had at all; the slice count says how much of
|
|
28
|
+
* the passing population it reached. A class with a hundred events on one
|
|
29
|
+
* slice is observable and reaches one slice — a very different fact from a
|
|
30
|
+
* class with a hundred events spread over a hundred slices.
|
|
31
|
+
*
|
|
32
|
+
* WHY THE SLICE-LEVEL UNOBSERVABILITY IS HERE TOO. `observable` is a SCOPE
|
|
33
|
+
* fact: the class had events somewhere. The rate, though, is computed over the
|
|
34
|
+
* slices the class reached — and the slices it did NOT reach are exactly the
|
|
35
|
+
* ones a skipped check would hide in, because a slice that recorded nothing
|
|
36
|
+
* about the class cannot be judged on it at all. So each class carries the
|
|
37
|
+
* count of passing slices that produced nothing of its kind: a reader who sees
|
|
38
|
+
* a rate of 100% over one slice must also see how many slices said nothing.
|
|
39
|
+
*/
|
|
40
|
+
/** The classes the metric knows, in a fixed order. */
|
|
41
|
+
export const NEAR_MISS_REASONS = [
|
|
42
|
+
'gate-never-evaluated',
|
|
43
|
+
'prereq-skipped',
|
|
44
|
+
'tool-blocked'
|
|
45
|
+
];
|
|
46
|
+
/** The evidence each class is read from. */
|
|
47
|
+
function evidenceOf(events) {
|
|
48
|
+
const gateEvents = events.filter((event) => event.category === 'gate').length;
|
|
49
|
+
const toolTraces = events.filter((event) => event.category === 'tool-trace').length;
|
|
50
|
+
const prereqAttempts = events.filter((event) => {
|
|
51
|
+
if (event.category !== 'slice-transition')
|
|
52
|
+
return false;
|
|
53
|
+
const detail = event.detail;
|
|
54
|
+
return typeof detail?.prereqOk === 'boolean';
|
|
55
|
+
}).length;
|
|
56
|
+
return [
|
|
57
|
+
{
|
|
58
|
+
reason: 'gate-never-evaluated',
|
|
59
|
+
scopeEventCount: gateEvents,
|
|
60
|
+
produced: (facts) => facts.observed.gateEvents,
|
|
61
|
+
unreadable: (facts) => facts.unreadableGates.size > 0
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
reason: 'prereq-skipped',
|
|
65
|
+
scopeEventCount: prereqAttempts,
|
|
66
|
+
produced: (facts) => facts.observed.prereqAttempts,
|
|
67
|
+
unreadable: (facts) => facts.unreadableVerdict !== null
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
reason: 'tool-blocked',
|
|
71
|
+
scopeEventCount: toolTraces,
|
|
72
|
+
produced: (facts) => facts.observed.toolTraces,
|
|
73
|
+
unreadable: () => false
|
|
74
|
+
}
|
|
75
|
+
];
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* The coverage of each reason class over the scope.
|
|
79
|
+
*
|
|
80
|
+
* A class with no evidence in the scope is reported as NOT OBSERVABLE with a
|
|
81
|
+
* `null` rate, so a zero can never be read as a result the class did not
|
|
82
|
+
* produce.
|
|
83
|
+
*/
|
|
84
|
+
export function computeCoverage(input) {
|
|
85
|
+
const evidence = evidenceOf(input.events);
|
|
86
|
+
return evidence.map((item) => {
|
|
87
|
+
const produced = input.passingFacts.map((facts) => item.produced(facts));
|
|
88
|
+
const observedSliceCount = produced.filter((count) => count > 0).length;
|
|
89
|
+
const unobservedSliceCount = produced.filter((count) => count === 0).length;
|
|
90
|
+
const unreadableSliceCount = input.unmeasuredFacts.filter((facts) => item.unreadable(facts)).length;
|
|
91
|
+
const nearMissSliceCount = [...input.reasonsBySlice.values()].filter((reasons) => reasons.includes(item.reason)).length;
|
|
92
|
+
const observable = item.scopeEventCount > 0;
|
|
93
|
+
return {
|
|
94
|
+
reason: item.reason,
|
|
95
|
+
observable,
|
|
96
|
+
scopeEventCount: item.scopeEventCount,
|
|
97
|
+
observedSliceCount,
|
|
98
|
+
unobservedSliceCount,
|
|
99
|
+
unreadableSliceCount,
|
|
100
|
+
nearMissSliceCount,
|
|
101
|
+
rate: observedSliceCount === 0 ? null : nearMissSliceCount / observedSliceCount
|
|
102
|
+
};
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
/** The classes that could not be observed, as the phrases a reader needs. */
|
|
106
|
+
export function unobservableReasons(coverage) {
|
|
107
|
+
return coverage.filter((item) => !item.observable).map((item) => item.reason);
|
|
108
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/near-miss-denominator.ts`
|
|
3
|
+
*
|
|
4
|
+
* The denominator the near-miss rate divides by, and its reconciliation with
|
|
5
|
+
* the status summary's success count.
|
|
6
|
+
*
|
|
7
|
+
* WHY THIS EXISTS. "Passed" has TWO definitions in this repository, and they
|
|
8
|
+
* do not count the same slices:
|
|
9
|
+
*
|
|
10
|
+
* - the STATUS SUMMARY counts a slice successful when its FINAL state is one
|
|
11
|
+
* of the terminal-happy states (`handed-off`, `verdict-issued`, …);
|
|
12
|
+
* - the NEAR-MISS metric counts a slice passed when it recorded a
|
|
13
|
+
* SUCCESSFUL `verdict-issued` attempt — the attempt whose prereq check
|
|
14
|
+
* passed, which is the moment the slice was declared passing.
|
|
15
|
+
*
|
|
16
|
+
* On this repository's recorded history the status summary counts 103
|
|
17
|
+
* successes and the near-miss denominator holds 94 slices. The gap is REAL
|
|
18
|
+
* and it is not noise, so the output states both numbers and says what the
|
|
19
|
+
* gap is made of. Silently aligning one to the other by fiat would hide the
|
|
20
|
+
* very thing the reader needs to see.
|
|
21
|
+
*
|
|
22
|
+
* WHY THE GAP'S COMPOSITION LIVES NEXT DOOR. Counting what the gap is made
|
|
23
|
+
* of, and naming the cause of each part, is a different question from the one
|
|
24
|
+
* this module answers — and the answer is long enough to need its own file.
|
|
25
|
+
* `near-miss-difference.ts` holds that part.
|
|
26
|
+
*
|
|
27
|
+
* WHY THE ROLLUPS COME FROM THE AGGREGATION. The comparison is only
|
|
28
|
+
* meaningful if the status side is computed by the SAME code the status
|
|
29
|
+
* summary uses, so the reconciliation cannot drift from the number it
|
|
30
|
+
* reconciles with. `aggregateSlices` is that code.
|
|
31
|
+
*
|
|
32
|
+
* WHY THE RECONCILIATION IS A COUNT AND NOT A STORY. Each side of the gap is
|
|
33
|
+
* counted, so a reader can check the arithmetic: the status count, minus the
|
|
34
|
+
* slices it counts that have no pass moment, plus the slices that have a
|
|
35
|
+
* pass moment it does not count, is the near-miss denominator.
|
|
36
|
+
*/
|
|
37
|
+
import { type NearMissDifferenceCounts } from './near-miss-difference.js';
|
|
38
|
+
import type { RequestTypeReading } from './hotspot-shared.js';
|
|
39
|
+
import type { ObservabilityEvent } from './observability-service.js';
|
|
40
|
+
/**
|
|
41
|
+
* The definition of "passed" the near-miss metric divides by, and how it
|
|
42
|
+
* relates to the status summary's success count.
|
|
43
|
+
*/
|
|
44
|
+
export type NearMissDenominator = {
|
|
45
|
+
/** What "passed" means here, in words a reader can check. */
|
|
46
|
+
readonly definition: string;
|
|
47
|
+
/** How many slices that definition admits — the rate's denominator. */
|
|
48
|
+
readonly sliceCount: number;
|
|
49
|
+
/**
|
|
50
|
+
* The status summary's success count over the same events.
|
|
51
|
+
*
|
|
52
|
+
* It is reported even when it AGREES, so a reader who compares the two
|
|
53
|
+
* numbers is never left guessing whether they were compared at all.
|
|
54
|
+
*/
|
|
55
|
+
readonly statusSuccessCount: number;
|
|
56
|
+
/**
|
|
57
|
+
* The slices that are in NEITHER the numerator nor the denominator
|
|
58
|
+
* because their pass verdict could not be read.
|
|
59
|
+
*
|
|
60
|
+
* They are reported so the denominator's composition is visible: a reader
|
|
61
|
+
* who adds up the slices and finds some missing must be able to see WHERE
|
|
62
|
+
* they went rather than infer it.
|
|
63
|
+
*/
|
|
64
|
+
readonly unmeasuredSliceCount: number;
|
|
65
|
+
/** The reconciliation, or null when the two counts agree. */
|
|
66
|
+
readonly difference: NearMissDenominatorDifference | null;
|
|
67
|
+
};
|
|
68
|
+
/** What the gap between the two "passed" definitions is made of. */
|
|
69
|
+
export type NearMissDenominatorDifference = NearMissDifferenceCounts & {
|
|
70
|
+
/** The reconciliation, spelled out. */
|
|
71
|
+
readonly note: string;
|
|
72
|
+
};
|
|
73
|
+
/** The definition, stated once, so the output and the source agree. */
|
|
74
|
+
export declare const PASSED_DEFINITION: string;
|
|
75
|
+
/**
|
|
76
|
+
* The denominator, and its reconciliation with the status summary.
|
|
77
|
+
*
|
|
78
|
+
* The status side comes from the aggregation's own rollup, so the number this
|
|
79
|
+
* reconciles with is the number the status summary reports.
|
|
80
|
+
*/
|
|
81
|
+
export declare function describeDenominator(events: readonly ObservabilityEvent[], sliceCount: number, requestTypes?: ReadonlyMap<string, RequestTypeReading>, unmeasuredSliceCount?: number): NearMissDenominator;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/near-miss-denominator.ts`
|
|
3
|
+
*
|
|
4
|
+
* The denominator the near-miss rate divides by, and its reconciliation with
|
|
5
|
+
* the status summary's success count.
|
|
6
|
+
*
|
|
7
|
+
* WHY THIS EXISTS. "Passed" has TWO definitions in this repository, and they
|
|
8
|
+
* do not count the same slices:
|
|
9
|
+
*
|
|
10
|
+
* - the STATUS SUMMARY counts a slice successful when its FINAL state is one
|
|
11
|
+
* of the terminal-happy states (`handed-off`, `verdict-issued`, …);
|
|
12
|
+
* - the NEAR-MISS metric counts a slice passed when it recorded a
|
|
13
|
+
* SUCCESSFUL `verdict-issued` attempt — the attempt whose prereq check
|
|
14
|
+
* passed, which is the moment the slice was declared passing.
|
|
15
|
+
*
|
|
16
|
+
* On this repository's recorded history the status summary counts 103
|
|
17
|
+
* successes and the near-miss denominator holds 94 slices. The gap is REAL
|
|
18
|
+
* and it is not noise, so the output states both numbers and says what the
|
|
19
|
+
* gap is made of. Silently aligning one to the other by fiat would hide the
|
|
20
|
+
* very thing the reader needs to see.
|
|
21
|
+
*
|
|
22
|
+
* WHY THE GAP'S COMPOSITION LIVES NEXT DOOR. Counting what the gap is made
|
|
23
|
+
* of, and naming the cause of each part, is a different question from the one
|
|
24
|
+
* this module answers — and the answer is long enough to need its own file.
|
|
25
|
+
* `near-miss-difference.ts` holds that part.
|
|
26
|
+
*
|
|
27
|
+
* WHY THE ROLLUPS COME FROM THE AGGREGATION. The comparison is only
|
|
28
|
+
* meaningful if the status side is computed by the SAME code the status
|
|
29
|
+
* summary uses, so the reconciliation cannot drift from the number it
|
|
30
|
+
* reconciles with. `aggregateSlices` is that code.
|
|
31
|
+
*
|
|
32
|
+
* WHY THE RECONCILIATION IS A COUNT AND NOT A STORY. Each side of the gap is
|
|
33
|
+
* counted, so a reader can check the arithmetic: the status count, minus the
|
|
34
|
+
* slices it counts that have no pass moment, plus the slices that have a
|
|
35
|
+
* pass moment it does not count, is the near-miss denominator.
|
|
36
|
+
*/
|
|
37
|
+
import { aggregateSlices } from './aggregation.js';
|
|
38
|
+
import { countDifferences, differenceNote } from './near-miss-difference.js';
|
|
39
|
+
/** The definition, stated once, so the output and the source agree. */
|
|
40
|
+
export const PASSED_DEFINITION = 'a slice that recorded a SUCCESSFUL `verdict-issued` attempt — a `verdict-issued` ' +
|
|
41
|
+
'transition attempt whose prereq check passed, which is the moment the slice was ' +
|
|
42
|
+
'declared passing. A `verdict-issued` attempt whose check FAILED is a refused ' +
|
|
43
|
+
'transition, not a pass. A slice whose `verdict-issued` attempt records a prereq ' +
|
|
44
|
+
'verdict this build CANNOT READ is in NEITHER: whether it passed is unknown, so it ' +
|
|
45
|
+
'is reported as unmeasured rather than counted here.';
|
|
46
|
+
/**
|
|
47
|
+
* The denominator, and its reconciliation with the status summary.
|
|
48
|
+
*
|
|
49
|
+
* The status side comes from the aggregation's own rollup, so the number this
|
|
50
|
+
* reconciles with is the number the status summary reports.
|
|
51
|
+
*/
|
|
52
|
+
export function describeDenominator(events, sliceCount, requestTypes = new Map(), unmeasuredSliceCount = 0) {
|
|
53
|
+
const rollups = aggregateSlices(events);
|
|
54
|
+
const statusSuccessCount = rollups.filter((rollup) => rollup.success).length;
|
|
55
|
+
if (statusSuccessCount === sliceCount) {
|
|
56
|
+
return {
|
|
57
|
+
definition: PASSED_DEFINITION,
|
|
58
|
+
sliceCount,
|
|
59
|
+
statusSuccessCount,
|
|
60
|
+
unmeasuredSliceCount,
|
|
61
|
+
difference: null
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
const counts = countDifferences(events, rollups, requestTypes);
|
|
65
|
+
return {
|
|
66
|
+
definition: PASSED_DEFINITION,
|
|
67
|
+
sliceCount,
|
|
68
|
+
statusSuccessCount,
|
|
69
|
+
unmeasuredSliceCount,
|
|
70
|
+
difference: {
|
|
71
|
+
...counts,
|
|
72
|
+
note: differenceNote(statusSuccessCount, sliceCount, counts)
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/near-miss-difference.ts`
|
|
3
|
+
*
|
|
4
|
+
* WHAT THE GAP between the two "passed" definitions is made of.
|
|
5
|
+
*
|
|
6
|
+
* WHY THIS IS ITS OWN MODULE. The reconciliation is a COUNTING question —
|
|
7
|
+
* each side of the gap is counted so a reader can check the arithmetic — and
|
|
8
|
+
* it is a different question from the one the denominator answers. Keeping
|
|
9
|
+
* them apart is what lets the denominator stay the short statement of "what
|
|
10
|
+
* this count means" and lets this module be the long statement of "why the
|
|
11
|
+
* two counts differ".
|
|
12
|
+
*
|
|
13
|
+
* WHY THE NOTE NAMES A CAUSE RATHER THAN A MISSING TRACE. Seventeen of the
|
|
14
|
+
* gap slices recorded no successful verdict attempt, and seven of those
|
|
15
|
+
* reached a `verdict-issued` event with no prereq verdict beside it. It is
|
|
16
|
+
* tempting to read that as "the check ran and left no record" — but it is
|
|
17
|
+
* not what happened, and saying so would send a reader hunting for an
|
|
18
|
+
* emitter bug that does not exist. The prereq check RETURNS EARLY when the
|
|
19
|
+
* request type's table has no row for the transition being checked, before
|
|
20
|
+
* it emits anything. The `docs` and `chore` types use a minimal table with
|
|
21
|
+
* no `qa:verdict-issued` row, so that transition has NO prerequisite to
|
|
22
|
+
* verify it against and the check has nothing to record. The pass was never
|
|
23
|
+
* gated, rather than gated-and-unrecorded.
|
|
24
|
+
*
|
|
25
|
+
* WHY AN UNREADABLE VERDICT IS COUNTED APART. A `verdict-issued` attempt
|
|
26
|
+
* whose prereq verdict was RECORDED in a shape this build cannot read is a
|
|
27
|
+
* different cause from one that was never recorded: the second was never
|
|
28
|
+
* gated, the first was gated and its verdict is unreadable. Folding them
|
|
29
|
+
* together would attribute an unreadable verdict to a missing prerequisite
|
|
30
|
+
* row, which is a cause the record does not support.
|
|
31
|
+
*/
|
|
32
|
+
import { type RequestType } from '../artifacts/artifact-prerequisites.js';
|
|
33
|
+
import type { SliceRollup } from './aggregation.js';
|
|
34
|
+
import type { RequestTypeReading } from './hotspot-shared.js';
|
|
35
|
+
import type { ObservabilityEvent } from './observability-service.js';
|
|
36
|
+
/** The counts that make the gap up, and the note that spells them out. */
|
|
37
|
+
export type NearMissDifferenceCounts = {
|
|
38
|
+
/**
|
|
39
|
+
* Slices the status summary counts as successful that recorded no
|
|
40
|
+
* successful verdict attempt.
|
|
41
|
+
*/
|
|
42
|
+
readonly successWithoutPassMoment: number;
|
|
43
|
+
/**
|
|
44
|
+
* Of those, the ones that reached a `verdict-issued` EVENT whose prereq
|
|
45
|
+
* verdict was never recorded.
|
|
46
|
+
*/
|
|
47
|
+
readonly successWithoutPassMomentButReachedVerdict: number;
|
|
48
|
+
/**
|
|
49
|
+
* Of those, the ones whose `verdict-issued` attempt recorded a prereq
|
|
50
|
+
* verdict that could not be READ.
|
|
51
|
+
*/
|
|
52
|
+
readonly successWithoutPassMomentButUnreadableVerdict: number;
|
|
53
|
+
/**
|
|
54
|
+
* The request types whose slices recorded no successful verdict attempt.
|
|
55
|
+
*
|
|
56
|
+
* WHY THIS IS THE CAUSE, NOT A SYMPTOM. The prereq event is emitted by the
|
|
57
|
+
* prerequisite check, and the check RETURNS EARLY when the request type's
|
|
58
|
+
* table has no row for the transition being checked — so no event is
|
|
59
|
+
* emitted at all. The `docs` and `chore` types use a minimal table with no
|
|
60
|
+
* `qa:verdict-issued` row, so the verdict transition has NO prerequisite to
|
|
61
|
+
* verify it against and the check has nothing to record.
|
|
62
|
+
*
|
|
63
|
+
* So the pass was not "unverified" — it was never gated, because there was
|
|
64
|
+
* no gate there. Saying "the check left no trace" would send a reader
|
|
65
|
+
* hunting for an emitter bug that does not exist.
|
|
66
|
+
*/
|
|
67
|
+
readonly typesWithoutVerdictPrerequisite: readonly RequestType[];
|
|
68
|
+
/**
|
|
69
|
+
* The slices of those types — the concrete referents for the count.
|
|
70
|
+
*/
|
|
71
|
+
readonly slicesWithoutVerdictPrerequisite: readonly string[];
|
|
72
|
+
/**
|
|
73
|
+
* Slices that recorded a successful verdict attempt but the status summary
|
|
74
|
+
* does not count as successful — they were sent back for rework after the
|
|
75
|
+
* verdict, so their final state is not a terminal-happy one.
|
|
76
|
+
*/
|
|
77
|
+
readonly passMomentWithoutStatusSuccess: number;
|
|
78
|
+
};
|
|
79
|
+
/**
|
|
80
|
+
* The counts that make the gap between the two "passed" definitions up.
|
|
81
|
+
*
|
|
82
|
+
* Each side of the gap is counted, so a reader can check the arithmetic: the
|
|
83
|
+
* status count, minus the slices it counts that have no pass moment, plus the
|
|
84
|
+
* slices that have a pass moment it does not count, is the near-miss
|
|
85
|
+
* denominator.
|
|
86
|
+
*/
|
|
87
|
+
export declare function countDifferences(events: readonly ObservabilityEvent[], rollups: readonly SliceRollup[], requestTypes: ReadonlyMap<string, RequestTypeReading>): NearMissDifferenceCounts;
|
|
88
|
+
/**
|
|
89
|
+
* The reconciliation, in words.
|
|
90
|
+
*
|
|
91
|
+
* WHY IT NAMES THE CAUSE. The slices that reached a `verdict-issued` event
|
|
92
|
+
* with no recorded prereq verdict were NOT "unverified" — the request types
|
|
93
|
+
* they had have no `qa:verdict-issued` prerequisite row at all, so the
|
|
94
|
+
* transition had no check to verify it against and the check had nothing to
|
|
95
|
+
* record. Saying the check "left no trace" would send a reader hunting for
|
|
96
|
+
* an emitter bug that does not exist.
|
|
97
|
+
*/
|
|
98
|
+
export declare function differenceNote(statusSuccessCount: number, sliceCount: number, counts: NearMissDifferenceCounts): string;
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/near-miss-difference.ts`
|
|
3
|
+
*
|
|
4
|
+
* WHAT THE GAP between the two "passed" definitions is made of.
|
|
5
|
+
*
|
|
6
|
+
* WHY THIS IS ITS OWN MODULE. The reconciliation is a COUNTING question —
|
|
7
|
+
* each side of the gap is counted so a reader can check the arithmetic — and
|
|
8
|
+
* it is a different question from the one the denominator answers. Keeping
|
|
9
|
+
* them apart is what lets the denominator stay the short statement of "what
|
|
10
|
+
* this count means" and lets this module be the long statement of "why the
|
|
11
|
+
* two counts differ".
|
|
12
|
+
*
|
|
13
|
+
* WHY THE NOTE NAMES A CAUSE RATHER THAN A MISSING TRACE. Seventeen of the
|
|
14
|
+
* gap slices recorded no successful verdict attempt, and seven of those
|
|
15
|
+
* reached a `verdict-issued` event with no prereq verdict beside it. It is
|
|
16
|
+
* tempting to read that as "the check ran and left no record" — but it is
|
|
17
|
+
* not what happened, and saying so would send a reader hunting for an
|
|
18
|
+
* emitter bug that does not exist. The prereq check RETURNS EARLY when the
|
|
19
|
+
* request type's table has no row for the transition being checked, before
|
|
20
|
+
* it emits anything. The `docs` and `chore` types use a minimal table with
|
|
21
|
+
* no `qa:verdict-issued` row, so that transition has NO prerequisite to
|
|
22
|
+
* verify it against and the check has nothing to record. The pass was never
|
|
23
|
+
* gated, rather than gated-and-unrecorded.
|
|
24
|
+
*
|
|
25
|
+
* WHY AN UNREADABLE VERDICT IS COUNTED APART. A `verdict-issued` attempt
|
|
26
|
+
* whose prereq verdict was RECORDED in a shape this build cannot read is a
|
|
27
|
+
* different cause from one that was never recorded: the second was never
|
|
28
|
+
* gated, the first was gated and its verdict is unreadable. Folding them
|
|
29
|
+
* together would attribute an unreadable verdict to a missing prerequisite
|
|
30
|
+
* row, which is a cause the record does not support.
|
|
31
|
+
*/
|
|
32
|
+
import { collectAttempts, passMomentOf, unreadableVerdictOf, VERDICT_STATE } from './near-miss-attempts.js';
|
|
33
|
+
import { getPrerequisitesFor } from '../artifacts/artifact-prerequisites.js';
|
|
34
|
+
import { sliceIdentity } from '../../shared/incrementing-number.js';
|
|
35
|
+
/**
|
|
36
|
+
* Whether a request type's prerequisite table has a row for the verdict
|
|
37
|
+
* transition.
|
|
38
|
+
*
|
|
39
|
+
* The prereq check RETURNS EARLY when the table has no row for the
|
|
40
|
+
* transition being checked — before it emits anything — so a type with no
|
|
41
|
+
* `qa:verdict-issued` row has no event to record for that transition. The
|
|
42
|
+
* `docs` and `chore` types use a minimal table with no such row.
|
|
43
|
+
*/
|
|
44
|
+
function hasVerdictPrerequisite(type) {
|
|
45
|
+
return getPrerequisitesFor('qa', 'verdict-issued', type).length > 0;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The counts that make the gap between the two "passed" definitions up.
|
|
49
|
+
*
|
|
50
|
+
* Each side of the gap is counted, so a reader can check the arithmetic: the
|
|
51
|
+
* status count, minus the slices it counts that have no pass moment, plus the
|
|
52
|
+
* slices that have a pass moment it does not count, is the near-miss
|
|
53
|
+
* denominator.
|
|
54
|
+
*/
|
|
55
|
+
export function countDifferences(events, rollups, requestTypes) {
|
|
56
|
+
let successWithoutPassMoment = 0;
|
|
57
|
+
let successWithoutPassMomentButReachedVerdict = 0;
|
|
58
|
+
let successWithoutPassMomentButUnreadableVerdict = 0;
|
|
59
|
+
let passMomentWithoutStatusSuccess = 0;
|
|
60
|
+
const types = new Set();
|
|
61
|
+
const slices = [];
|
|
62
|
+
for (const rollup of rollups) {
|
|
63
|
+
const sliceEvents = events.filter((event) => event.category === 'slice-transition' &&
|
|
64
|
+
event.sliceRid !== undefined &&
|
|
65
|
+
sliceIdentity(event.sliceRid).logical === rollup.sliceRid);
|
|
66
|
+
const attempts = collectAttempts(sliceEvents);
|
|
67
|
+
const hasPassMoment = passMomentOf(attempts) !== null;
|
|
68
|
+
if (rollup.success && !hasPassMoment) {
|
|
69
|
+
successWithoutPassMoment += 1;
|
|
70
|
+
const reachedVerdict = sliceEvents.some((event) => event.detail.to === VERDICT_STATE);
|
|
71
|
+
if (reachedVerdict) {
|
|
72
|
+
// A verdict that was RECORDED but cannot be READ is a different
|
|
73
|
+
// cause from one that was never recorded, so it is counted apart.
|
|
74
|
+
if (unreadableVerdictOf(sliceEvents) !== null) {
|
|
75
|
+
successWithoutPassMomentButUnreadableVerdict += 1;
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
successWithoutPassMomentButReachedVerdict += 1;
|
|
79
|
+
const reading = requestTypes.get(rollup.sliceRid);
|
|
80
|
+
// The cause: the type's prerequisite table has NO row for the
|
|
81
|
+
// `qa -> verdict-issued` transition, so the check had nothing to
|
|
82
|
+
// verify and emitted no event. The pass was never gated, rather
|
|
83
|
+
// than gated-and-unrecorded.
|
|
84
|
+
if (reading !== undefined && !hasVerdictPrerequisite(reading.requestType)) {
|
|
85
|
+
types.add(reading.requestType);
|
|
86
|
+
slices.push(rollup.sliceRid);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (!rollup.success && hasPassMoment)
|
|
92
|
+
passMomentWithoutStatusSuccess += 1;
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
successWithoutPassMoment,
|
|
96
|
+
successWithoutPassMomentButReachedVerdict,
|
|
97
|
+
successWithoutPassMomentButUnreadableVerdict,
|
|
98
|
+
typesWithoutVerdictPrerequisite: [...types].sort(),
|
|
99
|
+
slicesWithoutVerdictPrerequisite: slices.sort(),
|
|
100
|
+
passMomentWithoutStatusSuccess
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* The reconciliation, in words.
|
|
105
|
+
*
|
|
106
|
+
* WHY IT NAMES THE CAUSE. The slices that reached a `verdict-issued` event
|
|
107
|
+
* with no recorded prereq verdict were NOT "unverified" — the request types
|
|
108
|
+
* they had have no `qa:verdict-issued` prerequisite row at all, so the
|
|
109
|
+
* transition had no check to verify it against and the check had nothing to
|
|
110
|
+
* record. Saying the check "left no trace" would send a reader hunting for
|
|
111
|
+
* an emitter bug that does not exist.
|
|
112
|
+
*/
|
|
113
|
+
export function differenceNote(statusSuccessCount, sliceCount, counts) {
|
|
114
|
+
const base = `The status summary counts ${statusSuccessCount} slices successful (final state is a ` +
|
|
115
|
+
`terminal-happy state); this metric counts ${sliceCount} passed (a successful ` +
|
|
116
|
+
`\`verdict-issued\` attempt). The gap is ${counts.successWithoutPassMoment} slice(s) the ` +
|
|
117
|
+
'status counts successful with no successful verdict attempt';
|
|
118
|
+
const cause = counts.typesWithoutVerdictPrerequisite.length === 0
|
|
119
|
+
? ` — ${counts.successWithoutPassMomentButReachedVerdict} of which reached a ` +
|
|
120
|
+
'`verdict-issued` event whose prereq verdict was never recorded'
|
|
121
|
+
: ` — ${counts.successWithoutPassMomentButReachedVerdict} of which reached a ` +
|
|
122
|
+
'`verdict-issued` event whose prereq verdict was never recorded, because their ' +
|
|
123
|
+
`request type(s) ${counts.typesWithoutVerdictPrerequisite.map((type) => `\`${type}\``).join(', ')} ` +
|
|
124
|
+
'have NO `qa:verdict-issued` prerequisite row: the transition had no check to verify ' +
|
|
125
|
+
'it against, so the check had nothing to record. The pass was never gated, rather than ' +
|
|
126
|
+
'gated-and-unrecorded';
|
|
127
|
+
const unreadable = counts.successWithoutPassMomentButUnreadableVerdict === 0
|
|
128
|
+
? ''
|
|
129
|
+
: ` — and ${counts.successWithoutPassMomentButUnreadableVerdict} of which recorded a ` +
|
|
130
|
+
'`verdict-issued` attempt whose prereq verdict could not be READ, so they are reported ' +
|
|
131
|
+
'as unmeasured rather than as never gated';
|
|
132
|
+
const rework = ` — less ${counts.passMomentWithoutStatusSuccess} slice(s) that did pass but were sent ` +
|
|
133
|
+
'back for rework afterwards, so their final state is not a terminal-happy one.';
|
|
134
|
+
return `${base}${cause}${unreadable}${rework}`;
|
|
135
|
+
}
|