peaks-loop 4.1.21 → 4.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +3 -6
- package/CHANGELOG.md +191 -0
- package/dist/cli/commands/mut-commands-helpers.d.ts +6 -6
- package/dist/cli/commands/mut-commands-helpers.js +44 -26
- package/dist/cli/commands/observability-commands-hotspots.d.ts +54 -0
- package/dist/cli/commands/observability-commands-hotspots.js +222 -0
- package/dist/cli/commands/observability-commands-near-miss.d.ts +28 -0
- package/dist/cli/commands/observability-commands-near-miss.js +105 -0
- package/dist/cli/commands/observability-commands-queries.d.ts +61 -18
- package/dist/cli/commands/observability-commands-queries.js +88 -47
- package/dist/cli/commands/observability-commands.js +79 -20
- package/dist/services/artifacts/artifact-prerequisites.d.ts +3 -0
- package/dist/services/artifacts/artifact-prerequisites.js +1 -1
- package/dist/services/artifacts/prerequisite-path-census.d.ts +42 -0
- package/dist/services/artifacts/prerequisite-path-census.js +79 -0
- package/dist/services/artifacts/prerequisite-transition-event.js +13 -1
- package/dist/services/artifacts/rd-gate-event.d.ts +106 -0
- package/dist/services/artifacts/rd-gate-event.js +211 -0
- package/dist/services/artifacts/rd-transition-gates.d.ts +6 -0
- package/dist/services/artifacts/rd-transition-gates.js +65 -24
- package/dist/services/distribution/json-layout.d.ts +20 -0
- package/dist/services/distribution/json-layout.js +195 -0
- package/dist/services/distribution/plugin-manifest.d.ts +31 -2
- package/dist/services/distribution/plugin-manifest.js +40 -4
- package/dist/services/observability/aggregation-types.d.ts +21 -0
- package/dist/services/observability/aggregation.d.ts +12 -2
- package/dist/services/observability/aggregation.js +26 -2
- package/dist/services/observability/compliance-judge.d.ts +77 -0
- package/dist/services/observability/compliance-judge.js +143 -0
- package/dist/services/observability/compliance-spec.d.ts +96 -0
- package/dist/services/observability/compliance-spec.js +29 -0
- package/dist/services/observability/compliance-specs/peaks-race-code.spec.d.ts +37 -0
- package/dist/services/observability/compliance-specs/peaks-race-code.spec.js +66 -0
- package/dist/services/observability/compliance-specs/peaks-rd.spec.d.ts +50 -0
- package/dist/services/observability/compliance-specs/peaks-rd.spec.js +111 -0
- package/dist/services/observability/hotspot-attribution-summary.d.ts +84 -0
- package/dist/services/observability/hotspot-attribution-summary.js +74 -0
- package/dist/services/observability/hotspot-rows.d.ts +47 -0
- package/dist/services/observability/hotspot-rows.js +89 -0
- package/dist/services/observability/hotspot-shared.d.ts +212 -0
- package/dist/services/observability/hotspot-shared.js +122 -0
- package/dist/services/observability/hotspot-sources.d.ts +55 -0
- package/dist/services/observability/hotspot-sources.js +142 -0
- package/dist/services/observability/hotspots.d.ts +47 -0
- package/dist/services/observability/hotspots.js +47 -0
- package/dist/services/observability/near-miss-attempts.d.ts +175 -0
- package/dist/services/observability/near-miss-attempts.js +223 -0
- package/dist/services/observability/near-miss-coverage.d.ts +109 -0
- package/dist/services/observability/near-miss-coverage.js +108 -0
- package/dist/services/observability/near-miss-denominator.d.ts +81 -0
- package/dist/services/observability/near-miss-denominator.js +75 -0
- package/dist/services/observability/near-miss-difference.d.ts +98 -0
- package/dist/services/observability/near-miss-difference.js +132 -0
- package/dist/services/observability/near-miss-facts.d.ts +165 -0
- package/dist/services/observability/near-miss-facts.js +101 -0
- package/dist/services/observability/near-miss-gate-events.d.ts +75 -0
- package/dist/services/observability/near-miss-gate-events.js +106 -0
- package/dist/services/observability/near-miss-gates.d.ts +108 -0
- package/dist/services/observability/near-miss-gates.js +135 -0
- package/dist/services/observability/near-miss-records.d.ts +93 -0
- package/dist/services/observability/near-miss-records.js +87 -0
- package/dist/services/observability/near-miss.d.ts +148 -0
- package/dist/services/observability/near-miss.js +196 -0
- package/dist/services/observability/observability-schema.d.ts +17 -2
- package/dist/services/observability/observability-schema.js +21 -1
- package/dist/services/observability/owner-skill-map.d.ts +92 -0
- package/dist/services/observability/owner-skill-map.js +144 -0
- package/dist/services/observability/report-formatter-hotspots.d.ts +32 -0
- package/dist/services/observability/report-formatter-hotspots.js +76 -0
- package/dist/services/observability/report-formatter-near-miss.d.ts +48 -0
- package/dist/services/observability/report-formatter-near-miss.js +234 -0
- package/dist/services/observability/report-formatter.d.ts +49 -3
- package/dist/services/observability/report-formatter.js +55 -14
- package/dist/services/observability/retro-attribution.d.ts +129 -0
- package/dist/services/observability/retro-attribution.js +134 -0
- package/dist/services/observability/session-metric-rows.d.ts +153 -0
- package/dist/services/observability/session-metric-rows.js +184 -0
- package/package.json +5 -5
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/compliance-spec.ts`
|
|
3
|
+
*
|
|
4
|
+
* The vocabulary of a compliance spec: what a skill PROMISED to do, and how
|
|
5
|
+
* each promise is checked.
|
|
6
|
+
*
|
|
7
|
+
* WHY A VOCABULARY OF ITS OWN. The near-miss metric asks how often a passing
|
|
8
|
+
* slice was bought by skipping a check. Answering it needs the checks written
|
|
9
|
+
* down per SKILL, not inferred from a gate: a gate says which artifacts must
|
|
10
|
+
* EXIST at a transition, while a compliance spec says which steps the skill's
|
|
11
|
+
* own contract promised. A spec therefore carries steps that no gate checks at
|
|
12
|
+
* all (the fast lane writes no request artifacts) and steps whose artifacts
|
|
13
|
+
* belong to another skill (the review fan-out writes into qa/ and audit/, and
|
|
14
|
+
* the owner map names who owes each one).
|
|
15
|
+
*
|
|
16
|
+
* THE VERDICT IS NOT A JUDGEMENT. A step is executed when the artifact it
|
|
17
|
+
* names is on disk — nothing else. `judgeCompliance` (compliance-judge.ts)
|
|
18
|
+
* asks the caller whether a path exists, so one spec can be judged against a
|
|
19
|
+
* real workspace, a fixture, or a hypothetical.
|
|
20
|
+
*
|
|
21
|
+
* THE PATHS ARE SESSION-RELATIVE, with the request id still a placeholder.
|
|
22
|
+
* They are the same strings the request-artifact prerequisite table stores
|
|
23
|
+
* (that table's own relativePath values, e.g. `rd/code-review-<rid>.md`), so a
|
|
24
|
+
* spec can be checked against the table mechanically and one spec can be
|
|
25
|
+
* judged for many slices. The CALLER resolves a path against a session
|
|
26
|
+
* directory and a request id before probing the filesystem; the judge itself
|
|
27
|
+
* never touches a disk.
|
|
28
|
+
*/
|
|
29
|
+
/** One step a skill declares, and the deterministic check that says it ran. */
|
|
30
|
+
export type ComplianceStep = {
|
|
31
|
+
/** Stable id of the step inside its spec. */
|
|
32
|
+
readonly id: string;
|
|
33
|
+
/** What the step is, in the words of the skill that declares it. */
|
|
34
|
+
readonly description: string;
|
|
35
|
+
/**
|
|
36
|
+
* Whether a missing artifact counts against the compliance rate. A step
|
|
37
|
+
* whose obligation is conditional on the slice (a perf baseline on a slice
|
|
38
|
+
* with no perf surface) is not required, so its absence is not a failure.
|
|
39
|
+
*/
|
|
40
|
+
readonly required: boolean;
|
|
41
|
+
/**
|
|
42
|
+
* The skill that OWNS the artifact this step proves — the owner map's
|
|
43
|
+
* answer, or the spec's own skill where the map claims no owner. It names
|
|
44
|
+
* the skill a reader must tell when the step did not happen.
|
|
45
|
+
*/
|
|
46
|
+
readonly ownerSkill: string;
|
|
47
|
+
/**
|
|
48
|
+
* The deterministic check: the artifact whose presence means the step ran.
|
|
49
|
+
*
|
|
50
|
+
* THE PATH CARRIES AN UNRESOLVED `<rid>` PLACEHOLDER. The judge hands this
|
|
51
|
+
* string through verbatim; the CALLER substitutes the request id before
|
|
52
|
+
* probing the filesystem. A caller that probes the raw string asks the
|
|
53
|
+
* filesystem about a path containing the literal `<rid>`, every step misses,
|
|
54
|
+
* and the compliance rate reads 0 without a single error — so the
|
|
55
|
+
* substitution belongs to the caller, and it is stated here as well as on
|
|
56
|
+
* the judge's `exists` parameter.
|
|
57
|
+
*/
|
|
58
|
+
readonly verify: {
|
|
59
|
+
readonly artifactExists: string;
|
|
60
|
+
};
|
|
61
|
+
/**
|
|
62
|
+
* The step this one must come after. A step whose prerequisite did not
|
|
63
|
+
* execute cannot be credited on the strength of its own artifact alone: the
|
|
64
|
+
* audit that exists without the handoff it read is not evidence of an audit.
|
|
65
|
+
*
|
|
66
|
+
* The prerequisite's own `required` flag is not consulted: a step is
|
|
67
|
+
* demoted whenever its prerequisite did not execute, optional or not.
|
|
68
|
+
*/
|
|
69
|
+
readonly afterStep?: string;
|
|
70
|
+
};
|
|
71
|
+
/** The steps one skill declares, and the skill they belong to. */
|
|
72
|
+
export type ComplianceSpec = {
|
|
73
|
+
/** Stable id of the spec. */
|
|
74
|
+
readonly id: string;
|
|
75
|
+
/** Where the steps are declared — a pointer into the skill's own file. */
|
|
76
|
+
readonly source: string;
|
|
77
|
+
/** The skill whose contract this spec measures. */
|
|
78
|
+
readonly owner: string;
|
|
79
|
+
/** The declared steps, in the order the skill declares them. */
|
|
80
|
+
readonly steps: readonly ComplianceStep[];
|
|
81
|
+
};
|
|
82
|
+
/**
|
|
83
|
+
* A path a spec verifies that the request-artifact prerequisite table does
|
|
84
|
+
* NOT name.
|
|
85
|
+
*
|
|
86
|
+
* WHY THE LIST IS DATA AND NOT A COMMENT. A comment cannot be checked, and an
|
|
87
|
+
* uncheckable exception is an exception that grows. Naming the gap makes the
|
|
88
|
+
* absence visible to a reader AND lets a test refuse it the moment the
|
|
89
|
+
* contract catches up — at which point the declaration must go.
|
|
90
|
+
*/
|
|
91
|
+
export type ComplianceContractGap = {
|
|
92
|
+
/** The session-relative path the step verifies. */
|
|
93
|
+
readonly path: string;
|
|
94
|
+
/** Why the contract does not name it yet. */
|
|
95
|
+
readonly reason: string;
|
|
96
|
+
};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/compliance-spec.ts`
|
|
3
|
+
*
|
|
4
|
+
* The vocabulary of a compliance spec: what a skill PROMISED to do, and how
|
|
5
|
+
* each promise is checked.
|
|
6
|
+
*
|
|
7
|
+
* WHY A VOCABULARY OF ITS OWN. The near-miss metric asks how often a passing
|
|
8
|
+
* slice was bought by skipping a check. Answering it needs the checks written
|
|
9
|
+
* down per SKILL, not inferred from a gate: a gate says which artifacts must
|
|
10
|
+
* EXIST at a transition, while a compliance spec says which steps the skill's
|
|
11
|
+
* own contract promised. A spec therefore carries steps that no gate checks at
|
|
12
|
+
* all (the fast lane writes no request artifacts) and steps whose artifacts
|
|
13
|
+
* belong to another skill (the review fan-out writes into qa/ and audit/, and
|
|
14
|
+
* the owner map names who owes each one).
|
|
15
|
+
*
|
|
16
|
+
* THE VERDICT IS NOT A JUDGEMENT. A step is executed when the artifact it
|
|
17
|
+
* names is on disk — nothing else. `judgeCompliance` (compliance-judge.ts)
|
|
18
|
+
* asks the caller whether a path exists, so one spec can be judged against a
|
|
19
|
+
* real workspace, a fixture, or a hypothetical.
|
|
20
|
+
*
|
|
21
|
+
* THE PATHS ARE SESSION-RELATIVE, with the request id still a placeholder.
|
|
22
|
+
* They are the same strings the request-artifact prerequisite table stores
|
|
23
|
+
* (that table's own relativePath values, e.g. `rd/code-review-<rid>.md`), so a
|
|
24
|
+
* spec can be checked against the table mechanically and one spec can be
|
|
25
|
+
* judged for many slices. The CALLER resolves a path against a session
|
|
26
|
+
* directory and a request id before probing the filesystem; the judge itself
|
|
27
|
+
* never touches a disk.
|
|
28
|
+
*/
|
|
29
|
+
export {};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/compliance-specs/peaks-race-code.spec.ts`
|
|
3
|
+
*
|
|
4
|
+
* The pilot compliance spec for `peaks-race-code`: the steps the fast lane
|
|
5
|
+
* says it performs, and the artifact that proves each one ran.
|
|
6
|
+
*
|
|
7
|
+
* WHY THE FAST LANE IS THE SECOND PILOT. The fast lane writes no Gate A–G
|
|
8
|
+
* file chain, no planning document and no review envelope — it says so itself
|
|
9
|
+
* in the section "What this lane does not write". So its steps are exactly the
|
|
10
|
+
* two obligations it does keep: the session workspace the driver binds to, and
|
|
11
|
+
* the one short record that carries the need, the files changed, the four
|
|
12
|
+
* completion proofs and any risk row. The lane is the cheapest place to see
|
|
13
|
+
* the metric work on a skill whose obligations are small enough to read.
|
|
14
|
+
*
|
|
15
|
+
* THE PATHS ARE SESSION-RELATIVE, the same vocabulary the artifact-
|
|
16
|
+
* prerequisite table stores, so a caller resolves one shape for every spec.
|
|
17
|
+
*
|
|
18
|
+
* AND BOTH PATHS ARE CONTRACT GAPS, WHICH IS THE FINDING. The prerequisite
|
|
19
|
+
* table names no artifact of the fast lane at all, so every path this spec
|
|
20
|
+
* verifies is one the contract does not yet read. That is declared below
|
|
21
|
+
* rather than assumed away: a gate that never asked for the lane's evidence
|
|
22
|
+
* cannot report the lane skipping it.
|
|
23
|
+
*/
|
|
24
|
+
import type { ComplianceContractGap, ComplianceSpec } from '../compliance-spec.js';
|
|
25
|
+
/** The steps `peaks-race-code` declares, as the compliance spec sees them. */
|
|
26
|
+
export declare const PEAKS_RACE_CODE_SPEC: ComplianceSpec;
|
|
27
|
+
/**
|
|
28
|
+
* The paths this spec verifies that the artifact-prerequisite table does not
|
|
29
|
+
* name.
|
|
30
|
+
*
|
|
31
|
+
* BOTH of the lane's obligations are here, and the reason is the lane's own
|
|
32
|
+
* design: it writes no request artifacts, so the transition table — which
|
|
33
|
+
* gates request artifacts — has no row for anything it produces. The lane's
|
|
34
|
+
* evidence is checked by nothing today, which is exactly the gap this metric
|
|
35
|
+
* is built to make visible.
|
|
36
|
+
*/
|
|
37
|
+
export declare const PEAKS_RACE_CODE_CONTRACT_GAPS: readonly ComplianceContractGap[];
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/compliance-specs/peaks-race-code.spec.ts`
|
|
3
|
+
*
|
|
4
|
+
* The pilot compliance spec for `peaks-race-code`: the steps the fast lane
|
|
5
|
+
* says it performs, and the artifact that proves each one ran.
|
|
6
|
+
*
|
|
7
|
+
* WHY THE FAST LANE IS THE SECOND PILOT. The fast lane writes no Gate A–G
|
|
8
|
+
* file chain, no planning document and no review envelope — it says so itself
|
|
9
|
+
* in the section "What this lane does not write". So its steps are exactly the
|
|
10
|
+
* two obligations it does keep: the session workspace the driver binds to, and
|
|
11
|
+
* the one short record that carries the need, the files changed, the four
|
|
12
|
+
* completion proofs and any risk row. The lane is the cheapest place to see
|
|
13
|
+
* the metric work on a skill whose obligations are small enough to read.
|
|
14
|
+
*
|
|
15
|
+
* THE PATHS ARE SESSION-RELATIVE, the same vocabulary the artifact-
|
|
16
|
+
* prerequisite table stores, so a caller resolves one shape for every spec.
|
|
17
|
+
*
|
|
18
|
+
* AND BOTH PATHS ARE CONTRACT GAPS, WHICH IS THE FINDING. The prerequisite
|
|
19
|
+
* table names no artifact of the fast lane at all, so every path this spec
|
|
20
|
+
* verifies is one the contract does not yet read. That is declared below
|
|
21
|
+
* rather than assumed away: a gate that never asked for the lane's evidence
|
|
22
|
+
* cannot report the lane skipping it.
|
|
23
|
+
*/
|
|
24
|
+
/** The steps `peaks-race-code` declares, as the compliance spec sees them. */
|
|
25
|
+
export const PEAKS_RACE_CODE_SPEC = {
|
|
26
|
+
id: 'peaks-race-code',
|
|
27
|
+
source: 'skills/peaks-race-code/SKILL.md',
|
|
28
|
+
owner: 'peaks-race-code',
|
|
29
|
+
steps: [
|
|
30
|
+
{
|
|
31
|
+
id: 'race-session-workspace',
|
|
32
|
+
description: 'The session workspace exists: `peaks workspace init --project .` ran once, and the driver binds to it.',
|
|
33
|
+
required: true,
|
|
34
|
+
ownerSkill: 'peaks-race-code',
|
|
35
|
+
verify: { artifactExists: 'session.json' }
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
id: 'race-trace-record',
|
|
39
|
+
description: 'The one short record exists: the need in one sentence, the lane and who chose it, the files changed, the four completion proofs with their actual results, and any risk row hit.',
|
|
40
|
+
required: true,
|
|
41
|
+
ownerSkill: 'peaks-race-code',
|
|
42
|
+
verify: { artifactExists: 'race/<rid>.md' },
|
|
43
|
+
afterStep: 'race-session-workspace'
|
|
44
|
+
}
|
|
45
|
+
]
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* The paths this spec verifies that the artifact-prerequisite table does not
|
|
49
|
+
* name.
|
|
50
|
+
*
|
|
51
|
+
* BOTH of the lane's obligations are here, and the reason is the lane's own
|
|
52
|
+
* design: it writes no request artifacts, so the transition table — which
|
|
53
|
+
* gates request artifacts — has no row for anything it produces. The lane's
|
|
54
|
+
* evidence is checked by nothing today, which is exactly the gap this metric
|
|
55
|
+
* is built to make visible.
|
|
56
|
+
*/
|
|
57
|
+
export const PEAKS_RACE_CODE_CONTRACT_GAPS = [
|
|
58
|
+
{
|
|
59
|
+
path: 'session.json',
|
|
60
|
+
reason: "the artifact-prerequisite table names no session-workspace file, so nothing in the transition contract reads the fast lane's own workspace binding"
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
path: 'race/<rid>.md',
|
|
64
|
+
reason: "the artifact-prerequisite table names no race-lane record, so the trace that carries the lane's four completion proofs is read by no gate"
|
|
65
|
+
}
|
|
66
|
+
];
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/compliance-specs/peaks-rd.spec.ts`
|
|
3
|
+
*
|
|
4
|
+
* The pilot compliance spec for `peaks-rd`: the steps the RD skill declares,
|
|
5
|
+
* and the artifact that proves each one ran.
|
|
6
|
+
*
|
|
7
|
+
* WHY THESE STEPS. The near-miss question is about checks that can be SKIPPED
|
|
8
|
+
* and still leave a green gate, so the steps here are the ones the skill's own
|
|
9
|
+
* text makes obligatory: the per-request artifact every RD invocation must
|
|
10
|
+
* write, the handoff capsule the RD side hands to QA, the parallel review
|
|
11
|
+
* fan-out's code review and Karpathy review, and the two independent audits
|
|
12
|
+
* the RD gate demands — the security audit and its sibling, the perf audit.
|
|
13
|
+
* The RD-side perf baseline is the conditional one — the skill demands it only
|
|
14
|
+
* for a slice with a user-visible performance surface — so its absence is not
|
|
15
|
+
* a violation and it stays out of the denominator.
|
|
16
|
+
*
|
|
17
|
+
* THE TWO AUDITS ARE SIBLINGS, AND BOTH ARE HERE. The v2.12.0 fan-out
|
|
18
|
+
* collapse moved the security-reviewer and perf-baseline-reviewer slots out of
|
|
19
|
+
* the review fan-out into two standalone audit skills that write
|
|
20
|
+
* `audit/security-<rid>.md` and `audit/perf-<rid>.md`. A spec that modelled
|
|
21
|
+
* one audit and not the other would let the compliance rate read "peaks-rd
|
|
22
|
+
* does its job" while half the parallel audit obligation went unmeasured.
|
|
23
|
+
*
|
|
24
|
+
* THE PATHS ARE SESSION-RELATIVE, the same vocabulary the artifact-
|
|
25
|
+
* prerequisite table stores, so this spec and that table cannot disagree about
|
|
26
|
+
* where an artifact lives. The one path the table does NOT name is declared
|
|
27
|
+
* as a contract gap below rather than quietly used.
|
|
28
|
+
*
|
|
29
|
+
* THE ORDERING IS REAL. The independent audit skills read the immutable
|
|
30
|
+
* peaks-prd handoff and refuse without it, so an audit artifact on disk while
|
|
31
|
+
* the handoff is missing is not evidence of an audit — which is what the
|
|
32
|
+
* `afterStep` edge encodes.
|
|
33
|
+
*
|
|
34
|
+
* `source` names the FILE the steps come from, because the steps come from
|
|
35
|
+
* several sections of it; each step's description names its own section.
|
|
36
|
+
*/
|
|
37
|
+
import type { ComplianceContractGap, ComplianceSpec } from '../compliance-spec.js';
|
|
38
|
+
/** The steps `peaks-rd` declares, as the compliance spec sees them. */
|
|
39
|
+
export declare const PEAKS_RD_SPEC: ComplianceSpec;
|
|
40
|
+
/**
|
|
41
|
+
* The paths this spec verifies that the artifact-prerequisite table does not
|
|
42
|
+
* name.
|
|
43
|
+
*
|
|
44
|
+
* RD's per-request artifact is the one: the transition table gates the handoff
|
|
45
|
+
* and the fan-out evidence, while the per-request artifact is read by the RD
|
|
46
|
+
* handoff-coverage audit instead. That is a real difference between the two
|
|
47
|
+
* mechanisms, and it is why the path is declared here rather than assumed to be
|
|
48
|
+
* in the table.
|
|
49
|
+
*/
|
|
50
|
+
export declare const PEAKS_RD_CONTRACT_GAPS: readonly ComplianceContractGap[];
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/compliance-specs/peaks-rd.spec.ts`
|
|
3
|
+
*
|
|
4
|
+
* The pilot compliance spec for `peaks-rd`: the steps the RD skill declares,
|
|
5
|
+
* and the artifact that proves each one ran.
|
|
6
|
+
*
|
|
7
|
+
* WHY THESE STEPS. The near-miss question is about checks that can be SKIPPED
|
|
8
|
+
* and still leave a green gate, so the steps here are the ones the skill's own
|
|
9
|
+
* text makes obligatory: the per-request artifact every RD invocation must
|
|
10
|
+
* write, the handoff capsule the RD side hands to QA, the parallel review
|
|
11
|
+
* fan-out's code review and Karpathy review, and the two independent audits
|
|
12
|
+
* the RD gate demands — the security audit and its sibling, the perf audit.
|
|
13
|
+
* The RD-side perf baseline is the conditional one — the skill demands it only
|
|
14
|
+
* for a slice with a user-visible performance surface — so its absence is not
|
|
15
|
+
* a violation and it stays out of the denominator.
|
|
16
|
+
*
|
|
17
|
+
* THE TWO AUDITS ARE SIBLINGS, AND BOTH ARE HERE. The v2.12.0 fan-out
|
|
18
|
+
* collapse moved the security-reviewer and perf-baseline-reviewer slots out of
|
|
19
|
+
* the review fan-out into two standalone audit skills that write
|
|
20
|
+
* `audit/security-<rid>.md` and `audit/perf-<rid>.md`. A spec that modelled
|
|
21
|
+
* one audit and not the other would let the compliance rate read "peaks-rd
|
|
22
|
+
* does its job" while half the parallel audit obligation went unmeasured.
|
|
23
|
+
*
|
|
24
|
+
* THE PATHS ARE SESSION-RELATIVE, the same vocabulary the artifact-
|
|
25
|
+
* prerequisite table stores, so this spec and that table cannot disagree about
|
|
26
|
+
* where an artifact lives. The one path the table does NOT name is declared
|
|
27
|
+
* as a contract gap below rather than quietly used.
|
|
28
|
+
*
|
|
29
|
+
* THE ORDERING IS REAL. The independent audit skills read the immutable
|
|
30
|
+
* peaks-prd handoff and refuse without it, so an audit artifact on disk while
|
|
31
|
+
* the handoff is missing is not evidence of an audit — which is what the
|
|
32
|
+
* `afterStep` edge encodes.
|
|
33
|
+
*
|
|
34
|
+
* `source` names the FILE the steps come from, because the steps come from
|
|
35
|
+
* several sections of it; each step's description names its own section.
|
|
36
|
+
*/
|
|
37
|
+
/** The steps `peaks-rd` declares, as the compliance spec sees them. */
|
|
38
|
+
export const PEAKS_RD_SPEC = {
|
|
39
|
+
id: 'peaks-rd',
|
|
40
|
+
source: 'skills/bee/peaks-rd/SKILL.md',
|
|
41
|
+
owner: 'peaks-rd',
|
|
42
|
+
steps: [
|
|
43
|
+
{
|
|
44
|
+
id: 'rd-request-artifact',
|
|
45
|
+
description: 'Mandatory per-request artifact: the durable RD record of the request (red-line scope, coverage status, slice contract, implementation evidence, handoff).',
|
|
46
|
+
required: true,
|
|
47
|
+
ownerSkill: 'peaks-rd',
|
|
48
|
+
verify: { artifactExists: 'rd/requests/<rid>.md' }
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
id: 'rd-handoff',
|
|
52
|
+
description: 'The handoff capsule the RD side hands to peaks-qa: mode, scope, coverage status, validated decisions, current slice, artifact paths, blockers and next action.',
|
|
53
|
+
required: true,
|
|
54
|
+
ownerSkill: 'peaks-prd',
|
|
55
|
+
verify: { artifactExists: 'prd/handoff-<rid>.md' }
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
id: 'rd-code-review',
|
|
59
|
+
description: 'Parallel review fan-out: the code-reviewer wrote the per-slice code review, which is the Gate B3 evidence.',
|
|
60
|
+
required: true,
|
|
61
|
+
ownerSkill: 'peaks-rd',
|
|
62
|
+
verify: { artifactExists: 'rd/code-review-<rid>.md' }
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
id: 'rd-karpathy-review',
|
|
66
|
+
description: 'Parallel review fan-out: the karpathy-reviewer wrote the per-slice Karpathy review, which is the hard Karpathy-Gate.',
|
|
67
|
+
required: true,
|
|
68
|
+
ownerSkill: 'peaks-rd',
|
|
69
|
+
verify: { artifactExists: 'rd/karpathy-review-<rid>.md' }
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
id: 'rd-security-audit',
|
|
73
|
+
description: 'The independent security audit wrote its per-slice verdict; it reads the immutable handoff, so it cannot have run without one.',
|
|
74
|
+
required: true,
|
|
75
|
+
ownerSkill: 'peaks-security-audit',
|
|
76
|
+
verify: { artifactExists: 'audit/security-<rid>.md' },
|
|
77
|
+
afterStep: 'rd-handoff'
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
id: 'rd-perf-audit',
|
|
81
|
+
description: 'The independent perf audit wrote its per-slice verdict; like the security audit it reads the immutable handoff, so it cannot have run without one.',
|
|
82
|
+
required: true,
|
|
83
|
+
ownerSkill: 'peaks-perf-audit',
|
|
84
|
+
verify: { artifactExists: 'audit/perf-<rid>.md' },
|
|
85
|
+
afterStep: 'rd-handoff'
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
id: 'rd-perf-baseline',
|
|
89
|
+
description: 'The RD-side perf baseline. OPTIONAL: the skill demands it only for a slice with a user-visible performance surface, so its absence on any other slice is not a violation.',
|
|
90
|
+
required: false,
|
|
91
|
+
ownerSkill: 'peaks-rd',
|
|
92
|
+
verify: { artifactExists: 'rd/perf-baseline.md' }
|
|
93
|
+
}
|
|
94
|
+
]
|
|
95
|
+
};
|
|
96
|
+
/**
|
|
97
|
+
* The paths this spec verifies that the artifact-prerequisite table does not
|
|
98
|
+
* name.
|
|
99
|
+
*
|
|
100
|
+
* RD's per-request artifact is the one: the transition table gates the handoff
|
|
101
|
+
* and the fan-out evidence, while the per-request artifact is read by the RD
|
|
102
|
+
* handoff-coverage audit instead. That is a real difference between the two
|
|
103
|
+
* mechanisms, and it is why the path is declared here rather than assumed to be
|
|
104
|
+
* in the table.
|
|
105
|
+
*/
|
|
106
|
+
export const PEAKS_RD_CONTRACT_GAPS = [
|
|
107
|
+
{
|
|
108
|
+
path: 'rd/requests/<rid>.md',
|
|
109
|
+
reason: "the artifact-prerequisite table does not name RD's per-request artifact; the RD handoff-coverage audit reads it instead, so the table cannot be the only reader of the contract"
|
|
110
|
+
}
|
|
111
|
+
];
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/hotspot-attribution-summary.ts`
|
|
3
|
+
*
|
|
4
|
+
* The retro-attribution's own accounting: how many historical failures were
|
|
5
|
+
* attributed, how confidently, and on what basis.
|
|
6
|
+
*
|
|
7
|
+
* WHY THIS IS A SEPARATE MODULE FROM THE RANKING. The ranking answers
|
|
8
|
+
* "which artifact does a skill keep failing to deliver"; this answers "how
|
|
9
|
+
* much of that rests on a fact nobody wrote down". The two share the
|
|
10
|
+
* attribution gate (`hotspot-sources.ts`) so they cannot disagree about
|
|
11
|
+
* WHICH failures were attributed — but they report different things, and a
|
|
12
|
+
* reader asking "how many of these attributions rested on a type nobody
|
|
13
|
+
* wrote down?" needs this one.
|
|
14
|
+
*
|
|
15
|
+
* WHY THE PROVENANCE IS COUNTED HERE AND NOT IN THE RANKING. The ranking's
|
|
16
|
+
* rows are per ARTIFACT, and one failure fans out into many candidate rows.
|
|
17
|
+
* Counting there would count one failure many times and answer a question
|
|
18
|
+
* nobody asked. This counts failures.
|
|
19
|
+
*
|
|
20
|
+
* WHY A DEFAULTED TYPE IS NOT A MISSING TYPE. An artifact with no usable
|
|
21
|
+
* `- type:` line still has an EFFECTIVE type — the system's default — and
|
|
22
|
+
* that is the type the prerequisite table was indexed by when the transition
|
|
23
|
+
* ran, so the attribution uses it. The attribution is real; only its
|
|
24
|
+
* PROVENANCE is weak. The two facts are reported apart: `fromDeclaredType`
|
|
25
|
+
* and `fromDefaultedType` are separate numbers, and `unresolved` counts the
|
|
26
|
+
* failures whose request artifact could not be read at all — a fact that is
|
|
27
|
+
* NOT the same as "nobody wrote the type down".
|
|
28
|
+
*/
|
|
29
|
+
import type { RequestTypeReading } from './hotspot-shared.js';
|
|
30
|
+
import type { ObservabilityEvent } from './observability-service.js';
|
|
31
|
+
/**
|
|
32
|
+
* What the CLI's reader found while it read the request artifacts.
|
|
33
|
+
*
|
|
34
|
+
* WHY THESE ARE COUNTED APART FROM EACH OTHER AND FROM `unresolved`. The
|
|
35
|
+
* reader used to count "a type this command does not recognise" — a THIRD
|
|
36
|
+
* fact, and one that never fired, because the canonical reader substitutes
|
|
37
|
+
* the default rather than refusing. So a request whose type line was
|
|
38
|
+
* absent, one whose line named something unknown, and one whose artifact
|
|
39
|
+
* could not be read all looked alike. They do not.
|
|
40
|
+
*/
|
|
41
|
+
export type RequestArtifactCounts = {
|
|
42
|
+
/** Artifacts that could not be read at all — their slice's type is unknown. */
|
|
43
|
+
readonly unreadable: number;
|
|
44
|
+
/** Artifacts that read fine but carried no usable `- type:` line. */
|
|
45
|
+
readonly withoutTypeLine: number;
|
|
46
|
+
};
|
|
47
|
+
/** How the historical failures in a scope were attributed. */
|
|
48
|
+
export type HotspotAttributionSummary = {
|
|
49
|
+
/**
|
|
50
|
+
* How many historical failures were attributed at all — the population the
|
|
51
|
+
* confidence counts divide.
|
|
52
|
+
*/
|
|
53
|
+
readonly attributed: number;
|
|
54
|
+
/** How many the count matched the whole prerequisite list to. */
|
|
55
|
+
readonly inferred: number;
|
|
56
|
+
/** How many the count could not narrow to one entry. */
|
|
57
|
+
readonly ambiguous: number;
|
|
58
|
+
/** How many rested on a type the request artifact DECLARED. */
|
|
59
|
+
readonly fromDeclaredType: number;
|
|
60
|
+
/** How many rested on the DEFAULT type — a type nobody wrote down. */
|
|
61
|
+
readonly fromDefaultedType: number;
|
|
62
|
+
/**
|
|
63
|
+
* How many historical failures had NO usable request type — their artifact
|
|
64
|
+
* could not be read, so nothing could be attributed to them.
|
|
65
|
+
*
|
|
66
|
+
* A DIFFERENT FACT FROM `fromDefaultedType`. A defaulted type is a real
|
|
67
|
+
* type that was not written down; an unresolved one is a type nobody can
|
|
68
|
+
* supply at all. Conflating them would report a missing artifact as a
|
|
69
|
+
* missing line.
|
|
70
|
+
*/
|
|
71
|
+
readonly unresolved: number;
|
|
72
|
+
/** The request artifacts the reader could not read. */
|
|
73
|
+
readonly artifactsUnreadable: number;
|
|
74
|
+
/** The request artifacts that read fine but carried no usable type line. */
|
|
75
|
+
readonly artifactsWithoutTypeLine: number;
|
|
76
|
+
};
|
|
77
|
+
/**
|
|
78
|
+
* The attribution summary of a scope.
|
|
79
|
+
*
|
|
80
|
+
* `requestTypes` is keyed by slice rid and carries each type WITH its
|
|
81
|
+
* provenance, so a failure whose type was defaulted is counted as defaulted
|
|
82
|
+
* rather than being indistinguishable from a declared one.
|
|
83
|
+
*/
|
|
84
|
+
export declare function summarizeAttribution(events: readonly ObservabilityEvent[], requestTypes: ReadonlyMap<string, RequestTypeReading>, artifacts: RequestArtifactCounts): HotspotAttributionSummary;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/hotspot-attribution-summary.ts`
|
|
3
|
+
*
|
|
4
|
+
* The retro-attribution's own accounting: how many historical failures were
|
|
5
|
+
* attributed, how confidently, and on what basis.
|
|
6
|
+
*
|
|
7
|
+
* WHY THIS IS A SEPARATE MODULE FROM THE RANKING. The ranking answers
|
|
8
|
+
* "which artifact does a skill keep failing to deliver"; this answers "how
|
|
9
|
+
* much of that rests on a fact nobody wrote down". The two share the
|
|
10
|
+
* attribution gate (`hotspot-sources.ts`) so they cannot disagree about
|
|
11
|
+
* WHICH failures were attributed — but they report different things, and a
|
|
12
|
+
* reader asking "how many of these attributions rested on a type nobody
|
|
13
|
+
* wrote down?" needs this one.
|
|
14
|
+
*
|
|
15
|
+
* WHY THE PROVENANCE IS COUNTED HERE AND NOT IN THE RANKING. The ranking's
|
|
16
|
+
* rows are per ARTIFACT, and one failure fans out into many candidate rows.
|
|
17
|
+
* Counting there would count one failure many times and answer a question
|
|
18
|
+
* nobody asked. This counts failures.
|
|
19
|
+
*
|
|
20
|
+
* WHY A DEFAULTED TYPE IS NOT A MISSING TYPE. An artifact with no usable
|
|
21
|
+
* `- type:` line still has an EFFECTIVE type — the system's default — and
|
|
22
|
+
* that is the type the prerequisite table was indexed by when the transition
|
|
23
|
+
* ran, so the attribution uses it. The attribution is real; only its
|
|
24
|
+
* PROVENANCE is weak. The two facts are reported apart: `fromDeclaredType`
|
|
25
|
+
* and `fromDefaultedType` are separate numbers, and `unresolved` counts the
|
|
26
|
+
* failures whose request artifact could not be read at all — a fact that is
|
|
27
|
+
* NOT the same as "nobody wrote the type down".
|
|
28
|
+
*/
|
|
29
|
+
import { attributeFailure, isHistoricalFailure } from './hotspot-sources.js';
|
|
30
|
+
/**
|
|
31
|
+
* The attribution summary of a scope.
|
|
32
|
+
*
|
|
33
|
+
* `requestTypes` is keyed by slice rid and carries each type WITH its
|
|
34
|
+
* provenance, so a failure whose type was defaulted is counted as defaulted
|
|
35
|
+
* rather than being indistinguishable from a declared one.
|
|
36
|
+
*/
|
|
37
|
+
export function summarizeAttribution(events, requestTypes, artifacts) {
|
|
38
|
+
let attributed = 0;
|
|
39
|
+
let inferred = 0;
|
|
40
|
+
let ambiguous = 0;
|
|
41
|
+
let fromDeclaredType = 0;
|
|
42
|
+
let fromDefaultedType = 0;
|
|
43
|
+
let unresolved = 0;
|
|
44
|
+
for (const event of events) {
|
|
45
|
+
if (!isHistoricalFailure(event))
|
|
46
|
+
continue;
|
|
47
|
+
const rid = event.sliceRid;
|
|
48
|
+
const reading = rid === undefined ? undefined : requestTypes.get(rid);
|
|
49
|
+
const resolved = attributeFailure(event, reading);
|
|
50
|
+
if (resolved === null) {
|
|
51
|
+
unresolved += 1;
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
attributed += 1;
|
|
55
|
+
if (resolved.attribution.confidence === 'inferred')
|
|
56
|
+
inferred += 1;
|
|
57
|
+
else
|
|
58
|
+
ambiguous += 1;
|
|
59
|
+
if (resolved.attribution.typeSource === 'declared')
|
|
60
|
+
fromDeclaredType += 1;
|
|
61
|
+
else
|
|
62
|
+
fromDefaultedType += 1;
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
attributed,
|
|
66
|
+
inferred,
|
|
67
|
+
ambiguous,
|
|
68
|
+
fromDeclaredType,
|
|
69
|
+
fromDefaultedType,
|
|
70
|
+
unresolved,
|
|
71
|
+
artifactsUnreadable: artifacts.unreadable,
|
|
72
|
+
artifactsWithoutTypeLine: artifacts.withoutTypeLine
|
|
73
|
+
};
|
|
74
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `src/services/observability/hotspot-rows.ts`
|
|
3
|
+
*
|
|
4
|
+
* The hotspot RANKING: the rows, in the order a reader should read them.
|
|
5
|
+
*
|
|
6
|
+
* WHY THIS EXISTS. The programme's premise is that the QA verdict says
|
|
7
|
+
* "pass" without saying WHICH check was skipped, so a reader cannot go and
|
|
8
|
+
* fix the skill that is actually failing. This ranking names the artifact and
|
|
9
|
+
* its owner, so the next step ("raise this to a hook") has a target.
|
|
10
|
+
*
|
|
11
|
+
* TWO SOURCES, AND THE CONFIDENCE COLUMN SAYS WHICH. A FORWARD event (v2)
|
|
12
|
+
* carries the missing list as it was MEASURED: the path, the mode and the
|
|
13
|
+
* owner are all facts, and the row's confidence is `exact`. A HISTORICAL
|
|
14
|
+
* event (v1) carries only a COUNT, so the artifact is a CANDIDATE recovered
|
|
15
|
+
* by the retro-attribution, and the row's confidence is `inferred` (the count
|
|
16
|
+
* matched the whole prerequisite list) or `ambiguous` (the count did not
|
|
17
|
+
* localise to one entry). The spec (§2.2 S2.2) puts all three tiers in one
|
|
18
|
+
* ranking and requires them to be LABELLED, because an `inferred` row says
|
|
19
|
+
* "this artifact was in the missing set", NOT "this artifact was proven
|
|
20
|
+
* missing". A reader who sorts by count must be able to see which rows are
|
|
21
|
+
* measurements and which are reconstructions.
|
|
22
|
+
*
|
|
23
|
+
* A ROW WITH NO OWNER IS STILL A ROW. An artifact whose directory the
|
|
24
|
+
* owner map does not know is a NEW artifact type, and a new artifact type is
|
|
25
|
+
* exactly what a reader needs to see — the gate is that it is COUNTED, not
|
|
26
|
+
* that it is attributed. Dropping it would hide a whole class of failures
|
|
27
|
+
* behind a mapping gap, which is the failure mode the owner map exists to
|
|
28
|
+
* prevent.
|
|
29
|
+
*
|
|
30
|
+
* WHY THE READERS LIVE NEXT DOOR. `hotspot-sources.ts` holds the reading of
|
|
31
|
+
* the event stream, so this file states what a row MEANS and what the
|
|
32
|
+
* ranking guarantees; `hotspot-shared.ts` holds the vocabulary the two
|
|
33
|
+
* share. The same split `aggregation.ts` and `aggregation-helpers.ts` make in
|
|
34
|
+
* this directory.
|
|
35
|
+
*/
|
|
36
|
+
import { type HotspotRequestTypes, type HotspotRow } from './hotspot-shared.js';
|
|
37
|
+
import type { ObservabilityEvent } from './observability-service.js';
|
|
38
|
+
/**
|
|
39
|
+
* The hotspot ranking over a set of events.
|
|
40
|
+
*
|
|
41
|
+
* The request types are the DECLARED type of each slice, resolved by the
|
|
42
|
+
* caller from the request artifact: the retro-attribution needs them to
|
|
43
|
+
* re-derive a transition's prerequisite list, and the event stream does
|
|
44
|
+
* not carry them. Without them the historical failures cannot be
|
|
45
|
+
* attributed, and the ranking would silently show only the forward events.
|
|
46
|
+
*/
|
|
47
|
+
export declare function aggregateHotspots(events: readonly ObservabilityEvent[], requestTypes?: HotspotRequestTypes): HotspotRow[];
|