staysfixed 0.3.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +159 -3
- package/README.md +611 -402
- package/package.json +8 -3
- package/src/cli/index.js +14 -0
- package/src/v2/adapters/android-driver.js +1705 -0
- package/src/v2/adapters/android.js +1117 -0
- package/src/v2/adapters/contract.js +643 -0
- package/src/v2/adapters/electron.js +1594 -0
- package/src/v2/adapters/http.js +734 -0
- package/src/v2/adapters/ios-driver.js +1551 -0
- package/src/v2/adapters/ios.js +989 -0
- package/src/v2/adapters/isolate.js +739 -0
- package/src/v2/adapters/process.js +931 -0
- package/src/v2/adapters/source.js +1292 -0
- package/src/v2/adapters/web-driver.js +1532 -0
- package/src/v2/adapters/web.js +1009 -0
- package/src/v2/adapters/windows.js +1329 -0
- package/src/v2/browsers.js +1203 -0
- package/src/v2/cause.js +371 -0
- package/src/v2/check.js +1429 -0
- package/src/v2/ci.js +1209 -0
- package/src/v2/cli.js +670 -0
- package/src/v2/cluster.js +372 -0
- package/src/v2/coverage.js +1124 -0
- package/src/v2/detect.js +1199 -0
- package/src/v2/doctor.js +1702 -0
- package/src/v2/escalate.js +679 -0
- package/src/v2/init.js +1394 -0
- package/src/v2/intent.js +659 -0
- package/src/v2/journeys/from-routes.js +500 -0
- package/src/v2/journeys/from-suite.js +988 -0
- package/src/v2/journeys/index.js +651 -0
- package/src/v2/journeys/record.js +516 -0
- package/src/v2/mcp/server.js +374 -0
- package/src/v2/mcp/tools.js +1571 -0
- package/src/v2/normalise.js +783 -0
- package/src/v2/observation.js +938 -0
- package/src/v2/rank.js +672 -0
- package/src/v2/reference.js +1051 -0
- package/src/v2/remote.js +910 -0
- package/src/v2/run.js +1080 -0
- package/src/v2/sealed.js +568 -0
- package/src/v2/selfcheck.js +729 -0
- package/src/v2/ship.js +684 -0
- package/src/v2/store.js +703 -0
- package/src/v2/types.js +509 -0
- package/src/v2/waiver.js +511 -0
- package/src/v2/watch/focus.js +215 -0
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The adapter interface — written once, so the engine never knows what it is driving.
|
|
3
|
+
*
|
|
4
|
+
* The engine's job is arithmetic: run the new build twice, subtract the wobble, compare
|
|
5
|
+
* what is left against the reference. It does that over a flat list of `path -> value`
|
|
6
|
+
* observations and nothing else. An adapter's job is to turn one platform — a CLI, a
|
|
7
|
+
* server, a browser, a phone — into that flat list. Six more adapters will be written
|
|
8
|
+
* against this file, so the shape has to be right the first time.
|
|
9
|
+
*
|
|
10
|
+
* The five methods, in the order the engine calls them:
|
|
11
|
+
*
|
|
12
|
+
* detect(project) Can you drive this project on this machine, right now? Also:
|
|
13
|
+
* what is missing that would let you drive more of it? This is
|
|
14
|
+
* what `doctor` reports, and it is the first thing an agent asks.
|
|
15
|
+
* journeys(project) What steps do you know how to walk? Read out of the code, out
|
|
16
|
+
* of the project's own test suite, out of a recording — never
|
|
17
|
+
* out of a person.
|
|
18
|
+
* prepare(build) Get one build ready to be walked. Unpack it, install it into a
|
|
19
|
+
* scratch copy, boot it. Called once per build, not per journey,
|
|
20
|
+
* because booting is the expensive part.
|
|
21
|
+
* run(journey, build, ctx) Walk one journey against one prepared build and report what
|
|
22
|
+
* you saw, as Observations.
|
|
23
|
+
* teardown() Put everything back. Kill only what you started.
|
|
24
|
+
*
|
|
25
|
+
* Three rules every adapter obeys, and the engine cannot enforce for you:
|
|
26
|
+
*
|
|
27
|
+
* 1. NEVER TOUCH THE REAL PROJECT. Work in a scratch copy. The person running this has
|
|
28
|
+
* the real thing open in an editor.
|
|
29
|
+
* 2. NEVER LET SOMETHING IRREVERSIBLE HAPPEN. Money, messages, deleted data. Watch the
|
|
30
|
+
* call go out, record that it was asked for, and refuse it at the wire. A refusal is
|
|
31
|
+
* reported with `covered: false` — missing coverage, never a pass.
|
|
32
|
+
* 3. NEVER KILL WHAT YOU DID NOT START. Somebody's own app may be running.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { CHANNELS, isChannel, joinPath as joinSegments, makeObservation } from '../observation.js';
|
|
36
|
+
|
|
37
|
+
export { CHANNELS };
|
|
38
|
+
|
|
39
|
+
/** @typedef {import('../types.js').Channel} Channel */
|
|
40
|
+
/** @typedef {import('../types.js').Surface} Surface */
|
|
41
|
+
/** @typedef {import('../types.js').Observation} Observation */
|
|
42
|
+
/** @typedef {import('../types.js').ObservedValue} JsonValue */
|
|
43
|
+
/** @typedef {import('../types.js').Journey} Journey */
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Why a thing was not observed. Short vocabulary on purpose: the engine counts these and
|
|
47
|
+
* reports them as holes, and a free-text reason cannot be counted.
|
|
48
|
+
*
|
|
49
|
+
* @typedef {'irreversible'|'missing tool'|'refused'|'too big'|'timed out'|'not supported here'|'crashed'|'needs a sample'|'measures the machine'} NotCoveredReason
|
|
50
|
+
*/
|
|
51
|
+
|
|
52
|
+
/** @type {Record<NotCoveredReason, string>} */
|
|
53
|
+
export const NOT_COVERED_MEANING = Object.freeze({
|
|
54
|
+
irreversible: 'doing this for real would spend money, send a message, or destroy data',
|
|
55
|
+
'missing tool': 'something this machine does not have would be needed',
|
|
56
|
+
refused: 'the project asked us not to',
|
|
57
|
+
'too big': 'the value was too large to keep, so only a fingerprint of it was kept',
|
|
58
|
+
'timed out': 'it did not finish in the time allowed',
|
|
59
|
+
'not supported here': 'this platform cannot be observed this way, and saying so is the honest answer',
|
|
60
|
+
crashed: 'the thing being observed fell over before it could be read',
|
|
61
|
+
'needs a sample': 'a real value has to be supplied before this can be tried at all',
|
|
62
|
+
'measures the machine':
|
|
63
|
+
'a stopwatch measures how busy this machine was at least as much as it measures the product, so the number is recorded and never compared',
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
// ---------------------------------------------------------------------------
|
|
67
|
+
// The shapes
|
|
68
|
+
// ---------------------------------------------------------------------------
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* One build, as the engine hands it to an adapter.
|
|
72
|
+
*
|
|
73
|
+
* `root` is a directory the adapter may read. It is NOT the person's working copy unless
|
|
74
|
+
* the engine says so, and an adapter must not write into it either way.
|
|
75
|
+
*
|
|
76
|
+
* @typedef {object} Build
|
|
77
|
+
* @property {string} id Content-addressed id of this build.
|
|
78
|
+
* @property {string} label Plain English: 'the build you shipped', 'your change'.
|
|
79
|
+
* @property {'reference'|'candidate'} role
|
|
80
|
+
* @property {string} root Directory holding the source or the unpacked artifact.
|
|
81
|
+
* @property {string} [artifact] Path to a packaged artifact, when there is one.
|
|
82
|
+
* @property {string|null} [gitSha]
|
|
83
|
+
*/
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* A build that has been made ready to walk.
|
|
87
|
+
*
|
|
88
|
+
* @typedef {object} PreparedBuild
|
|
89
|
+
* @property {Build} build
|
|
90
|
+
* @property {string} root Where the walkable copy lives. Scratch, always.
|
|
91
|
+
* @property {boolean} ready False means prepare gave up; `why` says so in English.
|
|
92
|
+
* @property {string} why Plain English, always filled in — including on success.
|
|
93
|
+
* @property {Record<string, string|number|boolean|undefined>} [facts]
|
|
94
|
+
* Anything a journey needs: a port, a binary path, a pid.
|
|
95
|
+
* A fact may be undefined: a web app read at a fixed address
|
|
96
|
+
* has no port of its own, and an adapter should be able to
|
|
97
|
+
* say so rather than invent a value to satisfy a type.
|
|
98
|
+
* @property {() => Promise<void>} dispose Undo just this build's preparation.
|
|
99
|
+
*/
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* One repeatable set of steps.
|
|
103
|
+
*
|
|
104
|
+
* Journeys are never written by a person. They come from the code (free and exact), from
|
|
105
|
+
* the project's own test suite (already written, sitting there unused), from a recorded
|
|
106
|
+
* session, or from an agent that explored one named gap and froze it into a file.
|
|
107
|
+
*
|
|
108
|
+
* @property {string} id Stable, file-safe. Becomes the first path segment.
|
|
109
|
+
* @property {string} name Plain English. 'run the help text', 'GET /api/sessions'.
|
|
110
|
+
* @property {'code'|'suite'|'recording'|'config'|'agent'} from Where these steps came from.
|
|
111
|
+
* @property {string} [why] Plain English: why this is worth walking.
|
|
112
|
+
* @property {string} kind Adapter-specific: 'command', 'import', 'request', ...
|
|
113
|
+
* @property {JsonValue} detail Adapter-specific payload. The adapter that produced
|
|
114
|
+
* the journey is the only thing that reads it.
|
|
115
|
+
* @property {boolean} [irreversible] True means walking this for real would spend money,
|
|
116
|
+
* send a message or destroy data. The adapter must
|
|
117
|
+
* observe it at the call boundary and refuse the effect.
|
|
118
|
+
* @property {number} [timeoutMs]
|
|
119
|
+
*/
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* What an adapter says about a project when asked whether it can drive it.
|
|
123
|
+
*
|
|
124
|
+
* This is the honest-limits channel, and it is the first thing an agent installing the tool
|
|
125
|
+
* reads. `missing` is the important half: not "no", but "no, and here is the one thing that
|
|
126
|
+
* would turn this into a yes".
|
|
127
|
+
*
|
|
128
|
+
* @typedef {object} Detection
|
|
129
|
+
* @property {boolean} applies Can this adapter drive this project at all?
|
|
130
|
+
* @property {number} confidence 0..1. How sure. Below 0.5 the engine asks first.
|
|
131
|
+
* @property {string} why Plain English, always filled in, including for 'no'.
|
|
132
|
+
* @property {Missing[]} missing What would unlock more. Empty when nothing would.
|
|
133
|
+
* @property {string[]} [notes] Anything else worth saying in one line each.
|
|
134
|
+
*/
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* One thing that is not here, and what having it would buy.
|
|
138
|
+
*
|
|
139
|
+
* @typedef {object} Missing
|
|
140
|
+
* @property {string} what 'a Java runtime', 'a database snapshot to restore'.
|
|
141
|
+
* @property {string} unlocks Plain English: what becomes possible once it is there.
|
|
142
|
+
* @property {string} [howToGet] The exact command or link, filled in where detectable.
|
|
143
|
+
* @property {boolean} [blocking] True means nothing works without it.
|
|
144
|
+
*/
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Everything a run is allowed to use, handed in rather than reached for, so a run can be
|
|
148
|
+
* cancelled, redirected to a scratch disk, or replayed.
|
|
149
|
+
*
|
|
150
|
+
* @typedef {object} RunContext
|
|
151
|
+
* @property {AbortSignal} [signal] Cancel. Adapters must honour it.
|
|
152
|
+
* @property {string} scratchDir Somewhere to write. Wiped between runs by the engine.
|
|
153
|
+
* @property {string} evidenceDir Somewhere to keep things too big to inline.
|
|
154
|
+
* @property {number} seed The one seed. Same for both builds, both runs.
|
|
155
|
+
* @property {string} clock ISO time the product should believe it is.
|
|
156
|
+
* @property {(message: string) => void} [log] Progress, in plain English.
|
|
157
|
+
* @property {Record<string, any>} [config] The project's config for THIS adapter — the slice
|
|
158
|
+
* under a key matching the adapter's name. Handed in
|
|
159
|
+
* here as well as to `detect` because `prepare` needs
|
|
160
|
+
* it too, and an adapter that remembered it between
|
|
161
|
+
* calls would be one shared mutable variable away
|
|
162
|
+
* from two builds reading each other's settings.
|
|
163
|
+
* @property {boolean} [allowIrreversible] Default false, and the engine never sets it true.
|
|
164
|
+
* It exists so the refusal is a decision in the code
|
|
165
|
+
* rather than an accident of omission.
|
|
166
|
+
*/
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* A platform, taught to the engine.
|
|
170
|
+
*
|
|
171
|
+
* @typedef {object} Adapter
|
|
172
|
+
* @property {string} name Short id: 'process', 'http', 'source'.
|
|
173
|
+
* @property {string} title Plain English: 'CLI tools and libraries'.
|
|
174
|
+
* @property {string} describe One sentence an agent can read to know what this
|
|
175
|
+
* adapter watches and what it cannot see.
|
|
176
|
+
* @property {Channel[]} channels Which of the seven this adapter can fill. Honest —
|
|
177
|
+
* the coverage ledger is built from these.
|
|
178
|
+
* @property {(project: AdapterProject) => Promise<Detection>} detect
|
|
179
|
+
* @property {(project: AdapterProject) => Promise<Journey[]>} journeys
|
|
180
|
+
* @property {(build: Build, ctx: RunContext) => Promise<PreparedBuild>} prepare
|
|
181
|
+
* @property {(journey: Journey, build: PreparedBuild, ctx: RunContext) => Promise<Observation[]>} run
|
|
182
|
+
* @property {() => Promise<void>} teardown
|
|
183
|
+
*/
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* The slice of a project an adapter is allowed to see. Deliberately small: an adapter that
|
|
187
|
+
* can reach the whole engine ends up depending on it.
|
|
188
|
+
*
|
|
189
|
+
* @typedef {object} AdapterProject
|
|
190
|
+
* @property {string} root The real project root. READ ONLY, always.
|
|
191
|
+
* @property {Record<string, any>} [config] Whatever the project put in its config for this
|
|
192
|
+
* adapter, under a key matching the adapter's name.
|
|
193
|
+
* @property {Observation[]} [contract] The static contract, when the source adapter has
|
|
194
|
+
* already read it. This is how the HTTP adapter learns
|
|
195
|
+
* its routes without crawling.
|
|
196
|
+
*/
|
|
197
|
+
|
|
198
|
+
// ---------------------------------------------------------------------------
|
|
199
|
+
// Building an adapter
|
|
200
|
+
// ---------------------------------------------------------------------------
|
|
201
|
+
|
|
202
|
+
const REQUIRED_METHODS = ['detect', 'journeys', 'prepare', 'run', 'teardown'];
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Check an adapter before the engine trusts it, and say what is wrong in plain English.
|
|
206
|
+
*
|
|
207
|
+
* Separate from {@link defineAdapter} so a test, or `doctor`, can ask the question without
|
|
208
|
+
* building anything.
|
|
209
|
+
*
|
|
210
|
+
* @param {Partial<Adapter>} spec
|
|
211
|
+
* @returns {string[]} problems, empty when it is fine
|
|
212
|
+
*/
|
|
213
|
+
export function checkAdapter(spec) {
|
|
214
|
+
/** @type {string[]} */
|
|
215
|
+
const problems = [];
|
|
216
|
+
if (!spec || typeof spec !== 'object') return ['An adapter has to be an object.'];
|
|
217
|
+
|
|
218
|
+
if (typeof spec.name !== 'string' || !/^[a-z][a-z0-9-]*$/.test(spec.name)) {
|
|
219
|
+
problems.push('An adapter needs a short lowercase name like "process" or "http".');
|
|
220
|
+
}
|
|
221
|
+
if (typeof spec.title !== 'string' || spec.title.trim() === '') {
|
|
222
|
+
problems.push(`Adapter "${spec.name}" needs a title a person can read, like "CLI tools and libraries".`);
|
|
223
|
+
}
|
|
224
|
+
if (typeof spec.describe !== 'string' || spec.describe.trim() === '') {
|
|
225
|
+
problems.push(`Adapter "${spec.name}" needs one sentence saying what it watches and what it cannot see.`);
|
|
226
|
+
}
|
|
227
|
+
if (!Array.isArray(spec.channels) || spec.channels.length === 0) {
|
|
228
|
+
problems.push(`Adapter "${spec.name}" has to say which channels it fills.`);
|
|
229
|
+
} else {
|
|
230
|
+
for (const channel of spec.channels) {
|
|
231
|
+
if (!isChannel(channel)) {
|
|
232
|
+
problems.push(`Adapter "${spec.name}" claims a channel called "${channel}", which is not one of the seven.`);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
for (const method of REQUIRED_METHODS) {
|
|
237
|
+
if (typeof (/** @type {any} */ (spec)[method]) !== 'function') {
|
|
238
|
+
problems.push(`Adapter "${spec.name}" is missing ${method}().`);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
return problems;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Take an adapter spec and hand back something the engine can hold.
|
|
246
|
+
*
|
|
247
|
+
* Frozen, because six adapters sharing one engine is exactly the shape where one of them
|
|
248
|
+
* quietly reaches over and patches another.
|
|
249
|
+
*
|
|
250
|
+
* @param {Adapter} spec
|
|
251
|
+
* @returns {Adapter}
|
|
252
|
+
*/
|
|
253
|
+
export function defineAdapter(spec) {
|
|
254
|
+
const problems = checkAdapter(spec);
|
|
255
|
+
if (problems.length > 0) {
|
|
256
|
+
throw new Error(`This adapter cannot be used yet:\n - ${problems.join('\n - ')}`);
|
|
257
|
+
}
|
|
258
|
+
return Object.freeze({ ...spec });
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// ---------------------------------------------------------------------------
|
|
262
|
+
// Paths
|
|
263
|
+
// ---------------------------------------------------------------------------
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Build an observation path out of parts.
|
|
267
|
+
*
|
|
268
|
+
* A thin variadic wrapper over the engine's own `joinPath`, which owns the rules: dots
|
|
269
|
+
* between the parts, a dot inside a part escaped so it cannot be mistaken for a separator.
|
|
270
|
+
* It is wrapped rather than re-exported only because reading
|
|
271
|
+
* `path(kind, journey, 'status')` at a call site is easier than reading an array literal,
|
|
272
|
+
* and because every adapter going through one function is what keeps six of them agreeing.
|
|
273
|
+
*
|
|
274
|
+
* The FIRST part is the kind of thing being observed — `api`, `cli`, `ipc`, `route`, `file`,
|
|
275
|
+
* `proc`, `net`, `export`, `count` — not the journey. That ordering is what lets the engine
|
|
276
|
+
* cluster and rank by what broke rather than by which journey happened to find it.
|
|
277
|
+
*
|
|
278
|
+
* @param {...(string|number)} segments
|
|
279
|
+
* @returns {string}
|
|
280
|
+
*/
|
|
281
|
+
export function joinPath(...segments) {
|
|
282
|
+
return joinSegments(segments.filter((s) => s !== '' && s !== null && s !== undefined).map(String));
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// ---------------------------------------------------------------------------
|
|
286
|
+
// Values
|
|
287
|
+
// ---------------------------------------------------------------------------
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Put a value into the one shape the comparison engine understands.
|
|
291
|
+
*
|
|
292
|
+
* A pre-pass before the engine's own validator, which rejects anything that is not a
|
|
293
|
+
* string, a number, a boolean, null, or a list or plain object of those. Rather than let an
|
|
294
|
+
* adapter throw because something handed it a Date or a Buffer, everything is turned into
|
|
295
|
+
* something describable first — so a diff can say "this used to be a function" instead of
|
|
296
|
+
* "this used to be nothing".
|
|
297
|
+
*
|
|
298
|
+
* Object keys are sorted, because two runs of the same code can build the same object in a
|
|
299
|
+
* different order and that is not a difference.
|
|
300
|
+
*
|
|
301
|
+
* @param {unknown} value
|
|
302
|
+
* @param {WeakSet<object>} [seen]
|
|
303
|
+
* @returns {JsonValue}
|
|
304
|
+
*/
|
|
305
|
+
export function stableValue(value, seen = new WeakSet()) {
|
|
306
|
+
if (value === null || value === undefined) return null;
|
|
307
|
+
const type = typeof value;
|
|
308
|
+
if (type === 'string' || type === 'boolean') return /** @type {string|boolean} */ (value);
|
|
309
|
+
if (type === 'number') {
|
|
310
|
+
const n = /** @type {number} */ (value);
|
|
311
|
+
if (Number.isNaN(n)) return '(not a number)';
|
|
312
|
+
if (!Number.isFinite(n)) return n > 0 ? '(infinity)' : '(negative infinity)';
|
|
313
|
+
return n;
|
|
314
|
+
}
|
|
315
|
+
if (type === 'bigint') return `${value}n`;
|
|
316
|
+
if (type === 'function') return `(a function called ${/** @type {Function} */ (value).name || 'nothing'})`;
|
|
317
|
+
if (type === 'symbol') return `(the symbol ${String(value)})`;
|
|
318
|
+
|
|
319
|
+
const object = /** @type {object} */ (value);
|
|
320
|
+
if (seen.has(object)) return '(refers back to itself)';
|
|
321
|
+
seen.add(object);
|
|
322
|
+
|
|
323
|
+
if (Array.isArray(object)) return object.map((item) => stableValue(item, seen));
|
|
324
|
+
if (object instanceof Date) return object.toISOString();
|
|
325
|
+
if (object instanceof RegExp) return String(object);
|
|
326
|
+
if (object instanceof Error) return `${object.name}: ${object.message}`;
|
|
327
|
+
if (object instanceof Map) {
|
|
328
|
+
return stableValue(Object.fromEntries([...object.entries()].map(([k, v]) => [String(k), v])), seen);
|
|
329
|
+
}
|
|
330
|
+
if (object instanceof Set) return [...object].map((item) => stableValue(item, seen)).sort(compareJson);
|
|
331
|
+
if (ArrayBuffer.isView(object) || object instanceof ArrayBuffer) {
|
|
332
|
+
const bytes = object instanceof ArrayBuffer ? object.byteLength : /** @type {ArrayBufferView} */ (object).byteLength;
|
|
333
|
+
return `(${bytes} bytes)`;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/** @type {Record<string, JsonValue>} */
|
|
337
|
+
const out = {};
|
|
338
|
+
for (const key of Object.keys(object).sort()) {
|
|
339
|
+
out[key] = stableValue(/** @type {any} */ (object)[key], seen);
|
|
340
|
+
}
|
|
341
|
+
return out;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* A total order over stable values, so lists that have no natural order still come out the
|
|
346
|
+
* same way twice. Cheap and only used for sorting.
|
|
347
|
+
* @param {JsonValue} a
|
|
348
|
+
* @param {JsonValue} b
|
|
349
|
+
*/
|
|
350
|
+
export function compareJson(a, b) {
|
|
351
|
+
const left = typeof a === 'string' ? a : JSON.stringify(a);
|
|
352
|
+
const right = typeof b === 'string' ? b : JSON.stringify(b);
|
|
353
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// ---------------------------------------------------------------------------
|
|
357
|
+
// Making an observation
|
|
358
|
+
// ---------------------------------------------------------------------------
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Make one observation.
|
|
362
|
+
*
|
|
363
|
+
* Everything an adapter sees goes through here, so the shape is identical whatever produced
|
|
364
|
+
* it and so there is exactly one place that knows how an adapter's words map onto the
|
|
365
|
+
* engine's `Observation`. Two rules are enforced here and nowhere else:
|
|
366
|
+
*
|
|
367
|
+
* - `value` is the ONLY thing compared. The sentence, the file it came from and the
|
|
368
|
+
* picture backing it up all go into `meta`, which the engine never compares. If any of
|
|
369
|
+
* them leaked into `value`, a file move or a reworded sentence would report as a
|
|
370
|
+
* regression.
|
|
371
|
+
* - `says` is required. It is not decoration: it is what an agent reads when a difference
|
|
372
|
+
* lands in its lap, and it is the reason nothing about this tool needs documentation.
|
|
373
|
+
*
|
|
374
|
+
* @param {object} spec
|
|
375
|
+
* @param {Channel} spec.channel
|
|
376
|
+
* @param {string|(string|number)[]} spec.path A finished path, or parts to join.
|
|
377
|
+
* @param {unknown} spec.value
|
|
378
|
+
* @param {string} spec.says
|
|
379
|
+
* @param {boolean} [spec.covered] False means we did not really look. See `reason`.
|
|
380
|
+
* @param {NotCoveredReason} [spec.reason]
|
|
381
|
+
* @param {{file?: string, line?: number, url?: string}} [spec.where]
|
|
382
|
+
* @param {string} [spec.evidence]
|
|
383
|
+
* @param {string} [spec.journey]
|
|
384
|
+
* @param {Surface} [spec.surface]
|
|
385
|
+
* @returns {Observation}
|
|
386
|
+
*/
|
|
387
|
+
export function observation(spec) {
|
|
388
|
+
const path = Array.isArray(spec.path) ? joinPath(...spec.path) : spec.path;
|
|
389
|
+
if (!spec.says) throw new Error(`The observation at "${path}" needs one plain sentence saying what it is.`);
|
|
390
|
+
if (!isChannel(spec.channel)) {
|
|
391
|
+
throw new Error(`The observation at "${path}" claims channel "${spec.channel}", which is not one of the seven.`);
|
|
392
|
+
}
|
|
393
|
+
/** @type {import('../types.js').ObservationMeta} */
|
|
394
|
+
const meta = { describe: spec.says };
|
|
395
|
+
if (spec.where?.file) meta.source = spec.where.file;
|
|
396
|
+
else if (spec.where?.url) meta.source = spec.where.url;
|
|
397
|
+
if (spec.where?.line !== undefined) meta.line = spec.where.line;
|
|
398
|
+
if (spec.evidence) meta.evidence = spec.evidence;
|
|
399
|
+
if (spec.journey) meta.journey = spec.journey;
|
|
400
|
+
if (spec.surface) meta.surface = spec.surface;
|
|
401
|
+
if (spec.covered === false) {
|
|
402
|
+
// The engine reads `refused` when it builds the coverage ledger. Everything an adapter
|
|
403
|
+
// could not look at lands here, whichever of the reasons it was — a payment it would not
|
|
404
|
+
// make, a runtime this machine does not have, a parameter nobody supplied. They are all
|
|
405
|
+
// the same thing to a reader: a hole, with the reason attached, and never a pass.
|
|
406
|
+
meta.refused = true;
|
|
407
|
+
meta.refusedWhy = `${NOT_COVERED_MEANING[spec.reason ?? 'refused']} (${spec.reason ?? 'refused'})`;
|
|
408
|
+
}
|
|
409
|
+
return makeObservation(path, spec.channel, stableValue(spec.value), meta);
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/**
|
|
413
|
+
* A hole, recorded honestly.
|
|
414
|
+
*
|
|
415
|
+
* The rule the whole tool rests on: a refusal is reported as missing coverage, never as a
|
|
416
|
+
* pass. A run that quietly skipped the payment path and said "nothing changed" is worse
|
|
417
|
+
* than no run at all, because somebody believed it.
|
|
418
|
+
*
|
|
419
|
+
* @param {object} spec
|
|
420
|
+
* @param {Channel} spec.channel
|
|
421
|
+
* @param {string|(string|number)[]} spec.path
|
|
422
|
+
* @param {NotCoveredReason} spec.reason
|
|
423
|
+
* @param {string} spec.says What we would have looked at, and why we did not.
|
|
424
|
+
* @param {{file?: string, line?: number, url?: string}} [spec.where]
|
|
425
|
+
* @returns {Observation}
|
|
426
|
+
*/
|
|
427
|
+
export function notCovered(spec) {
|
|
428
|
+
return observation({
|
|
429
|
+
channel: spec.channel,
|
|
430
|
+
path: spec.path,
|
|
431
|
+
value: `not checked — ${NOT_COVERED_MEANING[spec.reason]}`,
|
|
432
|
+
says: spec.says,
|
|
433
|
+
covered: false,
|
|
434
|
+
reason: spec.reason,
|
|
435
|
+
where: spec.where,
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
// ---------------------------------------------------------------------------
|
|
440
|
+
// Coarse measures
|
|
441
|
+
// ---------------------------------------------------------------------------
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* The time ladder. Roughly three times apart, so ordinary variance stays inside one rung
|
|
445
|
+
* and a real slowdown crosses one. Exact milliseconds are never observed: they differ on
|
|
446
|
+
* every run, they would swamp every diff, and nobody has ever fixed a bug because a command
|
|
447
|
+
* took 412ms instead of 389ms.
|
|
448
|
+
*/
|
|
449
|
+
const TIME_LADDER = /** @type {const} */ ([
|
|
450
|
+
[100, 'instant'],
|
|
451
|
+
[300, 'quick'],
|
|
452
|
+
[1000, 'under a second'],
|
|
453
|
+
[3000, 'a few seconds'],
|
|
454
|
+
[10000, 'several seconds'],
|
|
455
|
+
[30000, 'half a minute'],
|
|
456
|
+
[90000, 'a minute or so'],
|
|
457
|
+
[300000, 'a few minutes'],
|
|
458
|
+
]);
|
|
459
|
+
|
|
460
|
+
/**
|
|
461
|
+
* @param {number} ms
|
|
462
|
+
* @returns {string} a plain-English bucket
|
|
463
|
+
*/
|
|
464
|
+
export function timeBucket(ms) {
|
|
465
|
+
if (!Number.isFinite(ms) || ms < 0) return 'unknown';
|
|
466
|
+
for (const [limit, label] of TIME_LADDER) if (ms < limit) return label;
|
|
467
|
+
return 'over five minutes';
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* How long something took, recorded and DELIBERATELY NOT COMPARED.
|
|
472
|
+
*
|
|
473
|
+
* This used to be an ordinary observation whose value was the bucket the run landed in, and
|
|
474
|
+
* it was the single worst thing in the tool, for a reason that is arithmetic rather than
|
|
475
|
+
* theoretical. A wall clock on a shared machine measures how busy the machine is at least as
|
|
476
|
+
* much as it measures the product. Two runs of identical code, one while a test suite is
|
|
477
|
+
* running and one on a quiet laptop, land on different rungs of any ladder you care to draw
|
|
478
|
+
* — and the tool then reported a difference nobody caused, or worse, reported the address as
|
|
479
|
+
* "newly unpredictable", which is its sharpest accusation.
|
|
480
|
+
*
|
|
481
|
+
* Measured on this Mac on 2026-08-30, on the self-check corpus's own fixture: thirty runs of
|
|
482
|
+
* the same one-line program, machine idle, ran 48ms to 96ms — with the first rung boundary at
|
|
483
|
+
* 100ms. Four milliseconds of headroom. Anything at all happening on the machine crosses it,
|
|
484
|
+
* and that is exactly what happened the night the self-check came back "1 of 9 wrong" while
|
|
485
|
+
* the test suite ran alongside it, and passed five times in a row afterwards.
|
|
486
|
+
*
|
|
487
|
+
* The fix is not a wider bucket — every ladder has a boundary and every boundary has this
|
|
488
|
+
* problem — and it is certainly not a tolerance, which this tool does not have and will not
|
|
489
|
+
* grow. It is to stop claiming something a stopwatch cannot tell you. The number is still
|
|
490
|
+
* recorded, in the sentence, where a person can read it. It is never differenced.
|
|
491
|
+
*
|
|
492
|
+
* WHAT THIS GIVES UP, said plainly: Stays Fixed will not tell you your product got slower.
|
|
493
|
+
* WHAT IT DOES NOT GIVE UP: a build that hangs is still caught, because it gets killed for
|
|
494
|
+
* taking too long and how it finished IS compared; and every counter that comes from the
|
|
495
|
+
* product rather than from the clock — files written, calls made, doors answered — is still
|
|
496
|
+
* compared exactly.
|
|
497
|
+
*
|
|
498
|
+
* @param {object} spec
|
|
499
|
+
* @param {Channel} spec.channel
|
|
500
|
+
* @param {string|(string|number)[]} spec.path
|
|
501
|
+
* @param {number} spec.ms What it actually took, for the sentence.
|
|
502
|
+
* @param {string} spec.what What was being timed, in the reader's words.
|
|
503
|
+
* @param {string} [spec.andAlso] Anything else worth saying in the same breath.
|
|
504
|
+
* @param {string} [spec.journey]
|
|
505
|
+
* @returns {Observation}
|
|
506
|
+
*/
|
|
507
|
+
export function howLongItTook(spec) {
|
|
508
|
+
return observation({
|
|
509
|
+
channel: spec.channel,
|
|
510
|
+
path: spec.path,
|
|
511
|
+
// One fixed string, so this address is identical in every capture of every build and can
|
|
512
|
+
// never become a difference. The measurement lives in the sentence, which is never compared.
|
|
513
|
+
value: `not compared — ${NOT_COVERED_MEANING['measures the machine']}`,
|
|
514
|
+
says:
|
|
515
|
+
`${spec.what} took ${timeBucket(spec.ms)}. That is recorded and NOT compared: a stopwatch on a shared machine ` +
|
|
516
|
+
`measures the machine as much as the product, so a busy laptop would otherwise invent a slowdown that nobody caused. ` +
|
|
517
|
+
`A build that hangs is still caught — it gets stopped for taking too long, and how it finished is compared.` +
|
|
518
|
+
(spec.andAlso ? ` ${spec.andAlso}` : ''),
|
|
519
|
+
covered: false,
|
|
520
|
+
reason: 'measures the machine',
|
|
521
|
+
journey: spec.journey,
|
|
522
|
+
});
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/**
|
|
526
|
+
* Sizes, on the same principle as time. A response body that grew by two bytes is not news;
|
|
527
|
+
* one that doubled is.
|
|
528
|
+
* @param {number} bytes
|
|
529
|
+
* @returns {string}
|
|
530
|
+
*/
|
|
531
|
+
export function sizeBucket(bytes) {
|
|
532
|
+
if (!Number.isFinite(bytes) || bytes < 0) return 'unknown';
|
|
533
|
+
if (bytes === 0) return 'empty';
|
|
534
|
+
const ladder = /** @type {const} */ ([
|
|
535
|
+
[128, 'a line or two'],
|
|
536
|
+
[1024, 'under a kilobyte'],
|
|
537
|
+
[10240, 'a few kilobytes'],
|
|
538
|
+
[102400, 'tens of kilobytes'],
|
|
539
|
+
[1048576, 'hundreds of kilobytes'],
|
|
540
|
+
[10485760, 'a few megabytes'],
|
|
541
|
+
[104857600, 'tens of megabytes'],
|
|
542
|
+
]);
|
|
543
|
+
for (const [limit, label] of ladder) if (bytes < limit) return label;
|
|
544
|
+
return 'over a hundred megabytes';
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/**
|
|
548
|
+
* Counts, bucketed once they get big enough that the exact number is noise. Small counts
|
|
549
|
+
* stay exact, because going from three files to four IS the finding.
|
|
550
|
+
* @param {number} n
|
|
551
|
+
* @returns {number|string}
|
|
552
|
+
*/
|
|
553
|
+
export function countBucket(n) {
|
|
554
|
+
if (!Number.isFinite(n) || n < 0) return 'unknown';
|
|
555
|
+
if (n <= 20) return n;
|
|
556
|
+
if (n < 50) return 'between 21 and 50';
|
|
557
|
+
if (n < 100) return 'between 51 and 100';
|
|
558
|
+
if (n < 500) return 'in the hundreds';
|
|
559
|
+
if (n < 1000) return 'many hundreds';
|
|
560
|
+
return 'thousands';
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
// ---------------------------------------------------------------------------
|
|
564
|
+
// Text that has to be the same twice
|
|
565
|
+
// ---------------------------------------------------------------------------
|
|
566
|
+
|
|
567
|
+
/**
|
|
568
|
+
* Rub out the things THIS TOOL varied, and nothing else.
|
|
569
|
+
*
|
|
570
|
+
* This is a deliberately short list, and keeping it short is the point. Every product has
|
|
571
|
+
* volatile output — version strings, ids, dates — and it is tempting to scrub all of it
|
|
572
|
+
* here. Do not. That is the noise-control layer's job, its rules live in the project's git
|
|
573
|
+
* so a person can see and argue with them, and the wobble measurement catches most of it
|
|
574
|
+
* for free. What belongs HERE is only the variation the harness itself introduced: the
|
|
575
|
+
* scratch directory it chose, the port it picked, the temp folder the operating system
|
|
576
|
+
* handed it. Rubbing those out is not judgement, it is undoing our own footprint.
|
|
577
|
+
*
|
|
578
|
+
* @param {string} text
|
|
579
|
+
* @param {object} footprint
|
|
580
|
+
* @param {string[]} [footprint.dirs] Absolute directories we created for this run.
|
|
581
|
+
* @param {number[]} [footprint.ports] Ports we picked.
|
|
582
|
+
* @param {string} [footprint.projectRoot] The real project root, when it appears in output.
|
|
583
|
+
* @returns {string}
|
|
584
|
+
*/
|
|
585
|
+
export function undoOurFootprint(text, footprint) {
|
|
586
|
+
let out = String(text).replace(/\r\n/g, '\n');
|
|
587
|
+
for (const dir of (footprint.dirs ?? []).slice().sort((a, b) => b.length - a.length)) {
|
|
588
|
+
if (!dir) continue;
|
|
589
|
+
out = out.split(dir).join('<the scratch folder>');
|
|
590
|
+
}
|
|
591
|
+
if (footprint.projectRoot) out = out.split(footprint.projectRoot).join('<the project>');
|
|
592
|
+
for (const port of footprint.ports ?? []) {
|
|
593
|
+
if (!port) continue;
|
|
594
|
+
out = out.split(`:${port}`).join(':<the port we picked>');
|
|
595
|
+
}
|
|
596
|
+
return out;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/**
|
|
600
|
+
* Keep a piece of text at a size worth storing.
|
|
601
|
+
*
|
|
602
|
+
* Anything longer gets its head and tail kept — the two ends are where the interesting lines
|
|
603
|
+
* are — plus the EXACT number of bytes left out, so a middle that grew or shrank still shows
|
|
604
|
+
* as a difference. A middle that changed without changing length does NOT, and that hole is
|
|
605
|
+
* stated rather than hidden: the caller marks the observation as not fully covered and writes
|
|
606
|
+
* the whole text to the evidence folder. See the comment in the body for why a digest of the
|
|
607
|
+
* whole text cannot be used here.
|
|
608
|
+
*
|
|
609
|
+
* @param {string} text
|
|
610
|
+
* @param {number} [limit] bytes
|
|
611
|
+
* @returns {{text: string, truncated: boolean, bytes: number}}
|
|
612
|
+
*/
|
|
613
|
+
export function trimForStorage(text, limit = 64 * 1024) {
|
|
614
|
+
const bytes = Buffer.byteLength(text, 'utf8');
|
|
615
|
+
if (bytes <= limit) return { text, truncated: false, bytes };
|
|
616
|
+
const keep = Math.floor(limit / 2);
|
|
617
|
+
const head = text.slice(0, keep);
|
|
618
|
+
const tail = text.slice(-keep);
|
|
619
|
+
// The marker used to carry a COARSE size bucket, and the doc above it claimed a fingerprint
|
|
620
|
+
// of the whole that was never actually computed. Both halves of that were wrong, and the
|
|
621
|
+
// result was the worst thing this tool can produce: a change that happened entirely in the
|
|
622
|
+
// discarded middle of a large output left a byte-identical stored value, so the comparison
|
|
623
|
+
// saw nothing and the run reported that nothing had changed. A silence that reads like an
|
|
624
|
+
// all-clear.
|
|
625
|
+
//
|
|
626
|
+
// The exact byte count goes in instead. A digest of the whole text would be strictly
|
|
627
|
+
// better AND IT CANNOT GO HERE: normalisation runs after the adapter, on the head and the
|
|
628
|
+
// tail, so a digest taken now would include every timestamp and every id the rules exist to
|
|
629
|
+
// rub out — the address would then disagree with itself on every run, be measured as wobble,
|
|
630
|
+
// and get subtracted, which would switch off the comparison of large outputs altogether.
|
|
631
|
+
// An exact length survives normalisation, because almost everything volatile (a timestamp,
|
|
632
|
+
// a uuid, a hex id) has a fixed width.
|
|
633
|
+
//
|
|
634
|
+
// What is left uncovered is real and it is named rather than hidden: a change confined to
|
|
635
|
+
// the middle that keeps the length identical is not seen. The caller marks the observation
|
|
636
|
+
// as not fully covered, the coverage ledger states the hole, and the whole text is written
|
|
637
|
+
// to the evidence folder so anybody can look.
|
|
638
|
+
return {
|
|
639
|
+
text: `${head}\n... exactly ${bytes - keep * 2} bytes left out of the middle of ${bytes} ...\n${tail}`,
|
|
640
|
+
truncated: true,
|
|
641
|
+
bytes,
|
|
642
|
+
};
|
|
643
|
+
}
|