staysfixed 0.7.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +429 -0
- package/README.md +193 -57
- package/docs/design-v2.md +24 -4
- package/docs/getting-started.md +19 -6
- package/docs/guards.md +2 -2
- package/docs/how-v2-works.md +12 -11
- package/docs/mcp.md +17 -8
- package/docs/settings.md +564 -0
- package/docs/watching.md +10 -4
- package/examples/staysfixed.config.electron.js +17 -6
- package/examples/staysfixed.config.web.js +22 -5
- package/package.json +2 -1
- package/src/cli/index.js +55 -46
- package/src/cli/status.js +45 -1
- package/src/cli/watch-flags.js +54 -0
- package/src/core/config.js +54 -3
- package/src/core/paths.js +15 -0
- package/src/guard/run.js +70 -3
- package/src/report/console.js +50 -6
- package/src/run.js +11 -0
- package/src/types.js +3 -0
- package/src/v2/adapters/android-driver.js +6 -1
- package/src/v2/adapters/android.js +97 -2
- package/src/v2/adapters/child.js +101 -0
- package/src/v2/adapters/contract.js +42 -5
- package/src/v2/adapters/electron.js +72 -6
- package/src/v2/adapters/http.js +18 -11
- package/src/v2/adapters/ios-driver.js +64 -14
- package/src/v2/adapters/ios.js +247 -25
- package/src/v2/adapters/process.js +783 -71
- package/src/v2/adapters/python.js +495 -0
- package/src/v2/adapters/source.js +373 -18
- package/src/v2/adapters/web-driver.js +134 -24
- package/src/v2/adapters/web.js +149 -18
- package/src/v2/adapters/windows.js +18 -1
- package/src/v2/browsers.js +66 -3
- package/src/v2/cause.js +61 -17
- package/src/v2/check.js +653 -69
- package/src/v2/ci.js +130 -35
- package/src/v2/cli.js +65 -42
- package/src/v2/cluster.js +220 -14
- package/src/v2/coverage.js +43 -176
- package/src/v2/detect.js +308 -60
- package/src/v2/doctor.js +353 -54
- package/src/v2/escalate.js +5 -1
- package/src/v2/init.js +183 -66
- package/src/v2/intent.js +9 -23
- package/src/v2/journeys/from-suite.js +336 -30
- package/src/v2/journeys/index.js +99 -6
- package/src/v2/mcp/tools.js +90 -16
- package/src/v2/normalise.js +169 -23
- package/src/v2/observation.js +19 -33
- package/src/v2/rank.js +216 -23
- package/src/v2/reference.js +160 -24
- package/src/v2/remote.js +113 -18
- package/src/v2/run.js +103 -14
- package/src/v2/sealed.js +0 -20
- package/src/v2/selfcheck.js +190 -13
- package/src/v2/ship.js +55 -5
- package/src/v2/store.js +67 -1
- package/src/v2/types.js +12 -2
- package/src/v2/waiver.js +64 -54
- package/src/v2/watch/events.js +60 -215
- package/src/v2/watch/focus.js +14 -4
- package/src/v2/watch/panel.js +167 -17
package/src/v2/mcp/tools.js
CHANGED
|
@@ -388,7 +388,7 @@ export function toolDefinitions() {
|
|
|
388
388
|
journeys: {
|
|
389
389
|
type: 'string',
|
|
390
390
|
description:
|
|
391
|
-
"Where the steps come from. 'code' is the default and needs nothing: each adapter reads your source and offers what it finds - routes, commands, screens, message channels.
|
|
391
|
+
"Where the steps come from. 'code' is the default and needs nothing: each adapter reads your source and offers what it finds - routes, commands, screens, message channels. 'suite' walks the project's own test suite as well: each test file runs twice inside the scratch copy, every check is reported by name, and it stops after 90 seconds naming each file it did not reach. It catches breaks nothing else can - a rounding change the product's own output never shows. It is opt-in because running a stranger's whole suite twice on every check is not something to do by default. You can also pass a path to a journeys file naming steps by hand. 'recorded' (replay a recorded session) is written and not yet wired into a run: ask for it and it says so rather than checking something else.",
|
|
392
392
|
},
|
|
393
393
|
surface: {
|
|
394
394
|
type: 'string',
|
|
@@ -756,18 +756,17 @@ async function toolCheck(ctx, input) {
|
|
|
756
756
|
const limit = positive(input.limit) ?? DEFAULT_LIMIT;
|
|
757
757
|
const offset = positive(input.offset) ?? 0;
|
|
758
758
|
|
|
759
|
-
// A value the engine does not understand must be refused BY NAME
|
|
760
|
-
//
|
|
761
|
-
//
|
|
762
|
-
// the engine as the name of a
|
|
763
|
-
// .../
|
|
764
|
-
//
|
|
759
|
+
// A value the engine does not understand must be refused BY NAME, never passed down.
|
|
760
|
+
//
|
|
761
|
+
// `suite` is now wired and reaches the harvest. `recorded` is still written and called by
|
|
762
|
+
// nothing, so passing it down would reach the engine as the name of a FILE and come back as
|
|
763
|
+
// "there is no journeys file at .../recorded" — an error that sends an agent looking for a
|
|
764
|
+
// file it never asked for. Refusing it by name and saying why is the honest answer, and a
|
|
765
|
+
// clean result about the wrong steps would be worse than no result.
|
|
765
766
|
const wantedJourneys = text(input.journeys);
|
|
766
|
-
if (wantedJourneys === '
|
|
767
|
+
if (wantedJourneys === 'recorded') {
|
|
767
768
|
return problem(
|
|
768
|
-
|
|
769
|
-
? 'Harvesting your own test suite as journeys is written and not wired into a run yet, so nothing was checked. Leave journeys out to use the steps each adapter reads from your source, or pass the path to a journeys file. Saying this rather than quietly checking something else is deliberate: a clean result about the wrong steps is worse than no result.'
|
|
770
|
-
: 'Replaying a recorded session is written and not wired into a run yet, so nothing was checked. Leave journeys out to use the steps each adapter reads from your source, or pass the path to a journeys file.'
|
|
769
|
+
'Replaying a recorded session is written and not wired into a run yet, so nothing was checked. Leave journeys out to use the steps each adapter reads from your source, pass "suite" to walk your own test suite, or pass the path to a journeys file.'
|
|
771
770
|
);
|
|
772
771
|
}
|
|
773
772
|
|
|
@@ -824,7 +823,12 @@ async function toolCheck(ctx, input) {
|
|
|
824
823
|
);
|
|
825
824
|
|
|
826
825
|
const page = unaccounted.slice(offset, offset + limit);
|
|
827
|
-
|
|
826
|
+
// A run that compared NOTHING is not a clean run, and this is the surface where saying so
|
|
827
|
+
// matters most. The engine had already worked it out and set `ok: false`; this line only
|
|
828
|
+
// ever counted differences, so a project with nothing on record produced zero differences,
|
|
829
|
+
// counted as clean, and the agent was told everything still works.
|
|
830
|
+
const comparedNothing = typeof result?.comparedNothing === 'string' && result.comparedNothing.length > 0;
|
|
831
|
+
const clean = cleanForAgent(result, unaccounted.length, newlyUnstable.length);
|
|
828
832
|
|
|
829
833
|
// What this run did not look at, in the engine's own words. It rides in every reply,
|
|
830
834
|
// clean ones included: a green result on a product with three hundred unopened doors is
|
|
@@ -844,7 +848,16 @@ async function toolCheck(ctx, input) {
|
|
|
844
848
|
if (input.format === 'json') {
|
|
845
849
|
const payload = {
|
|
846
850
|
ok: clean,
|
|
847
|
-
|
|
851
|
+
// "differences found" would be the wrong word for a run that found none because it
|
|
852
|
+
// compared none. There are three outcomes here, not two, and the third is the one
|
|
853
|
+
// that must never be mistaken for either.
|
|
854
|
+
verdict: result?.blocked
|
|
855
|
+
? 'blocked'
|
|
856
|
+
: comparedNothing
|
|
857
|
+
? 'nothing was compared'
|
|
858
|
+
: clean
|
|
859
|
+
? 'nothing unaccounted for'
|
|
860
|
+
: 'differences found',
|
|
848
861
|
mode: result?.mode ?? null,
|
|
849
862
|
note: result?.summary ?? null,
|
|
850
863
|
noiseRemoved: result?.differencesNoise ?? null,
|
|
@@ -867,7 +880,14 @@ async function toolCheck(ctx, input) {
|
|
|
867
880
|
waiversLeft: accounting?.left ?? null,
|
|
868
881
|
note: accounting?.note ?? null,
|
|
869
882
|
},
|
|
870
|
-
|
|
883
|
+
// The class an agent reads has to be the class that DECIDES things, not the engine's
|
|
884
|
+
// first guess. A 20% markup on a price came back as `class: "ordinary"` here while the
|
|
885
|
+
// human text on the same run said "1 of them sealed and not yours to waive" and
|
|
886
|
+
// `staysfixed_waive` refused it outright because it touches money. An agent reading
|
|
887
|
+
// "ordinary" would reasonably believe it may wave a price change through, and would
|
|
888
|
+
// tell somebody so. `waivable` is spelled out beside it so nothing has to be inferred
|
|
889
|
+
// from a word at all.
|
|
890
|
+
findings: page.map(findingForAgent),
|
|
871
891
|
aimedAt: aimed ? { surface: surface ?? 'auto', at: at ?? null, confirmed: missedTheTarget === null } : null,
|
|
872
892
|
aimingWarning: missedTheTarget,
|
|
873
893
|
};
|
|
@@ -914,6 +934,53 @@ async function toolCheck(ctx, input) {
|
|
|
914
934
|
return { content: [{ type: 'text', text: body + tail }], isError: !clean };
|
|
915
935
|
}
|
|
916
936
|
|
|
937
|
+
/**
|
|
938
|
+
* Is this a clean run, as far as the machine asking is concerned?
|
|
939
|
+
*
|
|
940
|
+
* The last line is the one that matters and it is the one that was missing. This surface
|
|
941
|
+
* used to work "clean" out from the difference count ALONE, and a product with nothing on
|
|
942
|
+
* record produces no differences — so zero differences counted as a pass, and an agent was
|
|
943
|
+
* told "everything that worked before still works" about a run that compared nothing at all.
|
|
944
|
+
* The engine had already decided; nobody asked it.
|
|
945
|
+
*
|
|
946
|
+
* So the engine's own verdict is the floor. Whatever else is true, this can never answer
|
|
947
|
+
* clean about a run the engine called not-ok. Counting reasons here will always be a list
|
|
948
|
+
* somebody forgets to add to; deferring to the decision already made cannot be.
|
|
949
|
+
*
|
|
950
|
+
* @param {any} result What `check` returned.
|
|
951
|
+
* @param {number} unaccounted Differences nobody has accounted for.
|
|
952
|
+
* @param {number} newlyUnstable Addresses that were steady and are not any more.
|
|
953
|
+
* @returns {boolean}
|
|
954
|
+
*/
|
|
955
|
+
export function cleanForAgent(result, unaccounted, newlyUnstable) {
|
|
956
|
+
if (result && result.ok === false) return false;
|
|
957
|
+
if (result && result.blocked === true) return false;
|
|
958
|
+
if (result && typeof result.comparedNothing === 'string' && result.comparedNothing.length > 0) return false;
|
|
959
|
+
return unaccounted === 0 && newlyUnstable === 0;
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
/**
|
|
963
|
+
* One finding, shaped for the machine that reads it.
|
|
964
|
+
*
|
|
965
|
+
* Split out and exported because it is a DECISION, not formatting, and a decision only a
|
|
966
|
+
* running MCP server can reach is one nobody notices breaking. The class an agent reads has
|
|
967
|
+
* to be the class that decides things: a 20% markup on a price came back over MCP as
|
|
968
|
+
* `class: "ordinary"` while the human text on the same run said "1 of them sealed and not
|
|
969
|
+
* yours to waive" and `staysfixed_waive` refused it because it touches money. An agent
|
|
970
|
+
* reading "ordinary" would reasonably believe it may wave a price change through, and would
|
|
971
|
+
* say so to a person. `waivable` is spelled out beside it so nothing has to be inferred
|
|
972
|
+
* from a word at all.
|
|
973
|
+
*
|
|
974
|
+
* @param {any} f
|
|
975
|
+
* @returns {any}
|
|
976
|
+
*/
|
|
977
|
+
export function findingForAgent(f) {
|
|
978
|
+
const sealed = classify(f);
|
|
979
|
+
return sealed
|
|
980
|
+
? { ...f, class: sealed.class, sealed: true, waivable: false, sealedBecause: sealed.why, sealedBy: sealed.matched }
|
|
981
|
+
: { ...f, sealed: false, waivable: true };
|
|
982
|
+
}
|
|
983
|
+
|
|
917
984
|
/**
|
|
918
985
|
* @param {object} a
|
|
919
986
|
* @param {CheckResult} a.result
|
|
@@ -932,7 +999,7 @@ async function toolCheck(ctx, input) {
|
|
|
932
999
|
* @param {string|null} a.covers
|
|
933
1000
|
* @returns {string}
|
|
934
1001
|
*/
|
|
935
|
-
function renderCheck({ result, unaccounted, page, offset, limit, waived, expired, waiversLeft, newlyUnstable, intent, clean, missedTheTarget, notChecked, covers }) {
|
|
1002
|
+
export function renderCheck({ result, unaccounted, page, offset, limit, waived, expired, waiversLeft, newlyUnstable, intent, clean, missedTheTarget, notChecked, covers }) {
|
|
936
1003
|
/** @type {string[]} */
|
|
937
1004
|
const out = [];
|
|
938
1005
|
|
|
@@ -943,7 +1010,14 @@ function renderCheck({ result, unaccounted, page, offset, limit, waived, expired
|
|
|
943
1010
|
return out.join('\n');
|
|
944
1011
|
}
|
|
945
1012
|
|
|
946
|
-
if (
|
|
1013
|
+
if (result?.comparedNothing) {
|
|
1014
|
+
out.push(
|
|
1015
|
+
result.comparedNothing === 'no reference'
|
|
1016
|
+
? 'NOTHING WAS ACTUALLY COMPARED. There is no build of this product on record as working, so this run had nothing whatever to hold today\'s behaviour against. This is not a pass and not a failure - it is no answer.'
|
|
1017
|
+
: 'NOTHING WAS ACTUALLY COMPARED. Every journey was walked, and not one of them had anything on record from the build you were happy with. This is not a pass and not a failure - it is no answer.',
|
|
1018
|
+
);
|
|
1019
|
+
out.push('Do not report this as a clean run. Only shipping records what "working" means, and no agent may cut that reference.');
|
|
1020
|
+
} else if (clean) {
|
|
947
1021
|
out.push('NOTHING UNACCOUNTED FOR. Everything that worked before still works, as far as this run could see.');
|
|
948
1022
|
} else if (unaccounted.length) {
|
|
949
1023
|
const sealed = unaccounted.filter((f) => classify(f) !== null).length;
|
package/src/v2/normalise.js
CHANGED
|
@@ -264,9 +264,22 @@ export const OPTIONAL_RULES = [
|
|
|
264
264
|
];
|
|
265
265
|
|
|
266
266
|
/**
|
|
267
|
-
* A version stamp for the shipped set
|
|
268
|
-
*
|
|
269
|
-
*
|
|
267
|
+
* A version stamp for the shipped set, and a thing to be careful with.
|
|
268
|
+
*
|
|
269
|
+
* It is the prefix on every fingerprint, so bumping it does not mark one rule as changed — it
|
|
270
|
+
* marks EVERY stored capture in the world as normalised under different rules, all at once, and
|
|
271
|
+
* every project's next check reports "compared across a change to the normalisation rules" until
|
|
272
|
+
* somebody ships again. That is a large, silent cost for a number nobody thinks twice about.
|
|
273
|
+
*
|
|
274
|
+
* So bump it only when a change alters what a comparison SHOWS. There is already a way to add a
|
|
275
|
+
* rule without paying it, and it should be the first thing tried: ship the new rule `off: true`.
|
|
276
|
+
* The fingerprint hashes `activeRules` only, so a rule nobody has switched on does not move it,
|
|
277
|
+
* the rule is documented and copyable from the day it ships, and each project turns it on when
|
|
278
|
+
* it wants it — which is one project's stored record going stale rather than everyone's.
|
|
279
|
+
*
|
|
280
|
+
* A genuinely new `kind` is the case that cannot dodge it: an old copy of the tool meets a rule
|
|
281
|
+
* it does not know how to apply, and pretending the two are comparable would be worse than the
|
|
282
|
+
* churn. Bump it there, and say so in the changelog.
|
|
270
283
|
*/
|
|
271
284
|
export const RULES_VERSION = 1;
|
|
272
285
|
|
|
@@ -325,10 +338,17 @@ export function assertRules(rules) {
|
|
|
325
338
|
* Merge a project's rules over the shipped ones.
|
|
326
339
|
*
|
|
327
340
|
* Same id wins, so a project turns a default off by writing `{id, off: true}` and narrows one
|
|
328
|
-
* by writing `{id, paths: [...]}` — no need to restate a rule to change one field of it.
|
|
341
|
+
* by writing `{id, paths: [...]}` — no need to restate a rule to change one field of it. The
|
|
342
|
+
* type says so too now; it used to promise whole rules only, which made the one shape the
|
|
343
|
+
* paragraph above recommends a type error.
|
|
344
|
+
*
|
|
345
|
+
* A part-rule whose id matches nothing is REFUSED rather than inserted. Inserted, it became a
|
|
346
|
+
* rule with no `kind`, which every branch of the walker skips — so a project that misspelled
|
|
347
|
+
* `clock.iso` got exactly the silence of a rule that was working perfectly, on the one file
|
|
348
|
+
* whose whole job is to be auditable.
|
|
329
349
|
*
|
|
330
350
|
* @param {NormaliseRule[]} base
|
|
331
|
-
* @param {NormaliseRule[]} extra
|
|
351
|
+
* @param {(NormaliseRule | (Partial<NormaliseRule> & {id: string}))[]} extra
|
|
332
352
|
* @returns {NormaliseRule[]}
|
|
333
353
|
*/
|
|
334
354
|
export function mergeRules(base, extra) {
|
|
@@ -337,7 +357,14 @@ export function mergeRules(base, extra) {
|
|
|
337
357
|
for (const rule of base) byId.set(rule.id, rule);
|
|
338
358
|
for (const rule of extra) {
|
|
339
359
|
const existing = byId.get(rule.id);
|
|
340
|
-
|
|
360
|
+
if (!existing && ruleProblem(rule)) {
|
|
361
|
+
throw new StaysFixedError(`There is no normalisation rule called "${rule.id}" to change.`, {
|
|
362
|
+
hint: 'Check the spelling against the rule list, or write the whole rule out if you meant to add a new one.',
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
// Cast, because the refusal above is what proves it: an entry with no rule to merge
|
|
366
|
+
// into has already been through ruleProblem and come back clean, so it is a whole rule.
|
|
367
|
+
byId.set(rule.id, existing ? { ...existing, ...rule } : /** @type {NormaliseRule} */ (rule));
|
|
341
368
|
}
|
|
342
369
|
return [...byId.values()];
|
|
343
370
|
}
|
|
@@ -352,34 +379,145 @@ export function activeRules(rules) {
|
|
|
352
379
|
}
|
|
353
380
|
|
|
354
381
|
/**
|
|
355
|
-
* A short fingerprint of a rule set.
|
|
382
|
+
* A short fingerprint of what a rule set DOES.
|
|
356
383
|
*
|
|
357
384
|
* Stored on every capture. Comparing a capture normalised under one set of rules against one
|
|
358
|
-
* normalised under another is meaningless — the differences you see are the rules changing,
|
|
359
|
-
*
|
|
385
|
+
* normalised under another is meaningless — the differences you see are the rules changing, not
|
|
386
|
+
* the product — and this is what lets the run notice and say so.
|
|
387
|
+
*
|
|
388
|
+
* Two things are deliberately NOT in it, and both were bugs before they were decisions.
|
|
389
|
+
*
|
|
390
|
+
* WHERE A RULE APPLIES IS NOT WHAT IT DOES. `paths`, `channels` and `at` are scope. A rule that
|
|
391
|
+
* learns one more address goes on tidying everything it already tidied in exactly the same way,
|
|
392
|
+
* so a stored comparison is not suspect and saying it is spends the reader's attention on
|
|
393
|
+
* nothing. Scope is stamped separately by `rulesScope` and reported in its own quieter words,
|
|
394
|
+
* because a rule reaching a NEW address genuinely can start hiding something there and the
|
|
395
|
+
* honest thing is to name the address rather than to shrug or to shout.
|
|
396
|
+
*
|
|
397
|
+
* A FACT ABOUT THIS MACHINE IS NOT A RULE. `machineRules` builds patterns out of where the
|
|
398
|
+
* project is checked out, where home is, and what temp folder this machine handed out — and one
|
|
399
|
+
* of those is a fresh mkdtemp on every single run. Hashing those patterns made this fingerprint
|
|
400
|
+
* change on every run of the same binary, so the "the rules changed" caveat fired forever, on
|
|
401
|
+
* runs where nothing had changed at all, and never healed. It also emptied the caveat of
|
|
402
|
+
* meaning, which is worse: a warning that is always on is a warning nobody reads on the day it
|
|
403
|
+
* is true. Those rules go in by `id` and `kind` alone. The test is not "is this rule identical"
|
|
404
|
+
* but "would a change to it change what a comparison shows", and two machines both rewriting
|
|
405
|
+
* their own checkout to `<project>` produce the same normalised value.
|
|
360
406
|
*
|
|
361
407
|
* @param {NormaliseRule[]} rules
|
|
362
408
|
* @returns {string}
|
|
363
409
|
*/
|
|
364
410
|
export function rulesFingerprint(rules) {
|
|
365
411
|
const active = activeRules(rules)
|
|
366
|
-
.map((r) =>
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
412
|
+
.map((r) =>
|
|
413
|
+
r.machine
|
|
414
|
+
? { id: r.id, kind: r.kind, machine: true }
|
|
415
|
+
: {
|
|
416
|
+
id: r.id,
|
|
417
|
+
kind: r.kind,
|
|
418
|
+
pattern: r.pattern ?? '',
|
|
419
|
+
flags: r.flags ?? '',
|
|
420
|
+
with: r.with ?? '',
|
|
421
|
+
digits: r.digits ?? 0,
|
|
422
|
+
keys: r.keys ?? false,
|
|
423
|
+
numbers: r.numbers ?? false,
|
|
424
|
+
})
|
|
379
425
|
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
380
426
|
return `v${RULES_VERSION}-${sha256(JSON.stringify(active)).slice(0, 12)}`;
|
|
381
427
|
}
|
|
382
428
|
|
|
429
|
+
/**
|
|
430
|
+
* Where each switched-on rule applies, as data rather than as a hash.
|
|
431
|
+
*
|
|
432
|
+
* Kept readable on purpose. The fingerprint can be a hash because the only question ever asked
|
|
433
|
+
* of it is "same or not"; scope has to answer "what is covered now that was not covered before",
|
|
434
|
+
* and no hash can. It is small — a handful of globs on a handful of rules — and it is what turns
|
|
435
|
+
* "the rules changed" into "this rule now also covers screen.checkout.total".
|
|
436
|
+
*
|
|
437
|
+
* @param {NormaliseRule[]} rules
|
|
438
|
+
* @returns {Record<string, string[]>} Rule id to its globs, sorted. Rules that apply everywhere
|
|
439
|
+
* are left out: an unscoped rule has nothing to say here.
|
|
440
|
+
*/
|
|
441
|
+
export function rulesScope(rules) {
|
|
442
|
+
/** @type {Record<string, string[]>} */
|
|
443
|
+
const scope = {};
|
|
444
|
+
for (const rule of activeRules(rules)) {
|
|
445
|
+
const where = [
|
|
446
|
+
...(rule.paths ?? []),
|
|
447
|
+
...(rule.channels ?? []).map((c) => `channel:${c}`),
|
|
448
|
+
...(rule.at ?? []).map((a) => `inside:${a}`),
|
|
449
|
+
].sort();
|
|
450
|
+
if (where.length > 0) scope[rule.id] = where;
|
|
451
|
+
}
|
|
452
|
+
return scope;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/**
|
|
456
|
+
* What actually changed between the rules a stored capture was tidied by and today's.
|
|
457
|
+
*
|
|
458
|
+
* This exists because the caveat it feeds used to print two hashes. An agent reading
|
|
459
|
+
* "v1-fcf4b8000217 versus v1-29141a9ec069" can do nothing with either of them — it cannot tell
|
|
460
|
+
* whether to distrust the comparison, re-run it, or ignore the line, so it ignores the line.
|
|
461
|
+
*
|
|
462
|
+
* The three answers are different sizes of news and are said as such:
|
|
463
|
+
* behaviour changed A rule now rewrites something differently. The comparison IS suspect.
|
|
464
|
+
* scope changed A rule reaches somewhere it did not. Only that address is suspect, and
|
|
465
|
+
* it is named, because "somewhere" is not something anybody can act on.
|
|
466
|
+
* scope unknown The stored capture predates the scope stamp. Say that, rather than
|
|
467
|
+
* diffing against an empty object and reporting every glob as new.
|
|
468
|
+
*
|
|
469
|
+
* @param {{fingerprint?: string, scope?: Record<string, string[]>}} before From the stored capture.
|
|
470
|
+
* @param {{fingerprint?: string, scope?: Record<string, string[]>}} after From this run.
|
|
471
|
+
* @returns {{same: boolean, behaviourChanged: boolean, scopeChanged: boolean, addressesNewlyCovered: string[], say: string}}
|
|
472
|
+
*/
|
|
473
|
+
export function describeRuleChange(before, after) {
|
|
474
|
+
const none = { same: true, behaviourChanged: false, scopeChanged: false, addressesNewlyCovered: [], say: '' };
|
|
475
|
+
if (!before.fingerprint || !after.fingerprint) return none;
|
|
476
|
+
|
|
477
|
+
const behaviourChanged = before.fingerprint !== after.fingerprint;
|
|
478
|
+
const knowScope = before.scope !== undefined && after.scope !== undefined;
|
|
479
|
+
|
|
480
|
+
/** @type {string[]} */
|
|
481
|
+
const newlyCovered = [];
|
|
482
|
+
if (knowScope) {
|
|
483
|
+
const was = /** @type {Record<string, string[]>} */ (before.scope);
|
|
484
|
+
const now = /** @type {Record<string, string[]>} */ (after.scope);
|
|
485
|
+
for (const [id, globs] of Object.entries(now)) {
|
|
486
|
+
const had = new Set(was[id] ?? []);
|
|
487
|
+
for (const glob of globs) if (!had.has(glob)) newlyCovered.push(`${id} now also covers ${glob}`);
|
|
488
|
+
}
|
|
489
|
+
// A rule that STOPPED covering an address can only show differences that were being hidden,
|
|
490
|
+
// which is the safe direction and not worth a line of anybody's attention.
|
|
491
|
+
}
|
|
492
|
+
const scopeChanged = newlyCovered.length > 0;
|
|
493
|
+
|
|
494
|
+
if (!behaviourChanged && !scopeChanged) {
|
|
495
|
+
return { ...none, same: !behaviourChanged && !scopeChanged };
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/** @type {string[]} */
|
|
499
|
+
const parts = [];
|
|
500
|
+
if (behaviourChanged) {
|
|
501
|
+
parts.push(
|
|
502
|
+
'A normalisation rule rewrites something differently now than when the old build was recorded, '
|
|
503
|
+
+ 'so some of what you see below may be the rules changing rather than the product, and a rule '
|
|
504
|
+
+ 'that grew stricter since could be covering something up.',
|
|
505
|
+
);
|
|
506
|
+
}
|
|
507
|
+
if (scopeChanged) {
|
|
508
|
+
const named = newlyCovered.slice(0, 4).join('; ');
|
|
509
|
+
const more = newlyCovered.length > 4 ? `, and ${newlyCovered.length - 4} more` : '';
|
|
510
|
+
parts.push(
|
|
511
|
+
`A rule reaches somewhere it did not reach before: ${named}${more}. Everything else compares `
|
|
512
|
+
+ 'normally; only what those cover could be quieter than it should be.',
|
|
513
|
+
);
|
|
514
|
+
} else if (behaviourChanged && !knowScope) {
|
|
515
|
+
parts.push('The old record predates the scope stamp, so which addresses each rule covered then cannot be compared.');
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
return { same: false, behaviourChanged, scopeChanged, addressesNewlyCovered: newlyCovered, say: parts.join(' ') };
|
|
519
|
+
}
|
|
520
|
+
|
|
383
521
|
/**
|
|
384
522
|
* The rule set in plain English, for `doctor` and for any agent asking the tool what it does.
|
|
385
523
|
* Says both what is on and what is deliberately off, because the second list is the more
|
|
@@ -427,6 +565,7 @@ export function machineRules(where) {
|
|
|
427
565
|
wouldHide: 'Nothing about the product. It only ever replaces a prefix that is a fact about the machine.',
|
|
428
566
|
pattern: literal(where.root),
|
|
429
567
|
with: '<project>',
|
|
568
|
+
machine: true,
|
|
430
569
|
});
|
|
431
570
|
}
|
|
432
571
|
if (where.home) {
|
|
@@ -438,6 +577,7 @@ export function machineRules(where) {
|
|
|
438
577
|
wouldHide: 'The username, where a product shows it deliberately.',
|
|
439
578
|
pattern: literal(where.home),
|
|
440
579
|
with: '<home>',
|
|
580
|
+
machine: true,
|
|
441
581
|
});
|
|
442
582
|
}
|
|
443
583
|
if (where.tmp) {
|
|
@@ -449,6 +589,7 @@ export function machineRules(where) {
|
|
|
449
589
|
wouldHide: 'Nothing the product decides.',
|
|
450
590
|
pattern: literal(where.tmp),
|
|
451
591
|
with: '<temp>',
|
|
592
|
+
machine: true,
|
|
452
593
|
});
|
|
453
594
|
}
|
|
454
595
|
return rules;
|
|
@@ -501,7 +642,11 @@ const rxCache = new Map();
|
|
|
501
642
|
* @returns {RegExp}
|
|
502
643
|
*/
|
|
503
644
|
function regexFor(pattern, flags) {
|
|
504
|
-
|
|
645
|
+
// Written as the escape, never as the byte itself. A raw NUL in a source file makes grep,
|
|
646
|
+
// ripgrep and file(1) call the whole file binary and skip it in silence, so every audit that
|
|
647
|
+
// searches this tree stops seeing this module. That is how a dead export in here survived a
|
|
648
|
+
// sweep that was looking straight at it.
|
|
649
|
+
const key = `${flags}\u0000${pattern}`;
|
|
505
650
|
let rx = rxCache.get(key);
|
|
506
651
|
if (!rx) {
|
|
507
652
|
rx = new RegExp(pattern, flags);
|
|
@@ -625,6 +770,7 @@ export function normaliseCapture(capture, rules) {
|
|
|
625
770
|
...capture,
|
|
626
771
|
observations: capture.observations.map((o) => normaliseObservation(o, rules)),
|
|
627
772
|
rules: rulesFingerprint(rules),
|
|
773
|
+
rulesScope: rulesScope(rules),
|
|
628
774
|
};
|
|
629
775
|
}
|
|
630
776
|
|
package/src/v2/observation.js
CHANGED
|
@@ -45,21 +45,6 @@ export const CHANNELS = [
|
|
|
45
45
|
'pixels',
|
|
46
46
|
];
|
|
47
47
|
|
|
48
|
-
/**
|
|
49
|
-
* What each channel is, in the words we would use to a person. Printed by `doctor` and handed
|
|
50
|
-
* to any agent that asks the tool to describe itself.
|
|
51
|
-
* @type {Record<Channel, string>}
|
|
52
|
-
*/
|
|
53
|
-
export const CHANNEL_NOTES = {
|
|
54
|
-
meaning: 'What the interface says a control is and does — its role, its name, whether it is on, off or disabled. Not the underlying markup.',
|
|
55
|
-
effects: 'What the product sent out into the world: calls made, files written, processes started, things saved.',
|
|
56
|
-
complaints: 'What the product complained about: console messages, errors, crashes, the code it exited with.',
|
|
57
|
-
results: 'What the product gave back: what it printed, what it answered, what it offers other code.',
|
|
58
|
-
contract: 'The doors the source code says exist: routes, exported functions, message channels. Read without running anything.',
|
|
59
|
-
counters: 'Rough counts and rough timings. Deliberately rough — precise timing is noise, not information.',
|
|
60
|
-
pixels: 'What it looked like. Used to show a person a problem another channel already found.',
|
|
61
|
-
};
|
|
62
|
-
|
|
63
48
|
/**
|
|
64
49
|
* @param {unknown} value
|
|
65
50
|
* @returns {value is Channel}
|
|
@@ -78,17 +63,6 @@ const MAX_PATH_LENGTH = 512;
|
|
|
78
63
|
/** Deepest value we will store. Past this something is recursing, not observing. */
|
|
79
64
|
const MAX_VALUE_DEPTH = 64;
|
|
80
65
|
|
|
81
|
-
/**
|
|
82
|
-
* The share of its own addresses a build may disagree with itself about before the run stops
|
|
83
|
-
* counting as a measurement at all. Half is not a tuned number and nothing depends on its
|
|
84
|
-
* exact value: it is the point past which more of the comparison has been thrown away than
|
|
85
|
-
* kept, and no answer computed from what is left deserves to be called clean.
|
|
86
|
-
*/
|
|
87
|
-
const STORM_SHARE = 0.5;
|
|
88
|
-
|
|
89
|
-
/** Below this many addresses the share means nothing — three out of four is not a storm. */
|
|
90
|
-
const STORM_FLOOR = 12;
|
|
91
|
-
|
|
92
66
|
/** Control characters and newlines, which would break the store and every log line. */
|
|
93
67
|
const CONTROL_CHARS = /[\u0000-\u001f\u007f]/;
|
|
94
68
|
|
|
@@ -863,13 +837,25 @@ export function wobbleStorm(wobble) {
|
|
|
863
837
|
const looked = unstable + wobble.steady;
|
|
864
838
|
const vanished = wobble.entries.filter((e) => e.kind === 'vanished').length;
|
|
865
839
|
const share = looked === 0 ? 0 : unstable / looked;
|
|
866
|
-
//
|
|
867
|
-
//
|
|
868
|
-
//
|
|
869
|
-
//
|
|
870
|
-
//
|
|
871
|
-
|
|
872
|
-
|
|
840
|
+
// MORE OF IT WOBBLED THAN HELD STILL, and that one comparison is the whole rule. There is
|
|
841
|
+
// no threshold here to tune and no number to defend: half is the point past which more of
|
|
842
|
+
// the comparison has been thrown away than kept, and no answer computed from what is left
|
|
843
|
+
// deserves to be called clean.
|
|
844
|
+
//
|
|
845
|
+
// There used to be a floor under it: below twelve addresses the share was ignored
|
|
846
|
+
// altogether, with one exception carved out for the case where nothing at all held still.
|
|
847
|
+
// The floor was put there so that three-out-of-four would not be called a storm — and
|
|
848
|
+
// three of four IS one. It leaves a single address standing, and a run that compared one
|
|
849
|
+
// address is not entitled to the word clean. What the floor actually let through was worse
|
|
850
|
+
// than the thing it was guarding against: ten of eleven addresses wobbling was read as too
|
|
851
|
+
// small a sample to judge, so all ten differences were subtracted and the run came back
|
|
852
|
+
// saying nothing that already worked had changed. Confident, clean, and about one address.
|
|
853
|
+
//
|
|
854
|
+
// Counting rather than thresholding also swallows the zero-steady case whole. Nothing held
|
|
855
|
+
// still means everything wobbled, which is already more than half, so the exception that
|
|
856
|
+
// used to be written out separately is now just the arithmetic. One rule, no floor, no
|
|
857
|
+
// exception, and no number anybody has to defend.
|
|
858
|
+
if (!wobble.measured || unstable <= wobble.steady) {
|
|
873
859
|
return { stormy: false, share, looked, vanished, why: '' };
|
|
874
860
|
}
|
|
875
861
|
const percent = Math.round(share * 100);
|