proof-harness 0.1.0-next.3 → 0.1.0-next.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/COMPATIBILITY.md +41 -1
- package/PROOF_SDK_CONTRACT.md +34 -0
- package/README.md +29 -1
- package/cli/config.mjs +1 -0
- package/cli/engines/proof_module_check.mjs +166 -11
- package/cli/engines/proof_mutation_check.mjs +46 -1
- package/cli/engines/proof_plant_verification.mjs +88 -0
- package/dist/{fixture-B-dmnG5G.d.ts → fixture-DwDOegDK.d.ts} +9 -1
- package/dist/playwright.d.ts +8 -1
- package/dist/playwright.js +39 -6
- package/dist/playwright.js.map +1 -1
- package/dist/server.d.ts +19 -2
- package/dist/server.js +40 -4
- package/dist/server.js.map +1 -1
- package/package.json +1 -1
package/COMPATIBILITY.md
CHANGED
|
@@ -11,7 +11,7 @@ Consumers must pin an exact prerelease version, for example:
|
|
|
11
11
|
```json
|
|
12
12
|
{
|
|
13
13
|
"devDependencies": {
|
|
14
|
-
"proof-harness": "0.1.0-next.
|
|
14
|
+
"proof-harness": "0.1.0-next.5"
|
|
15
15
|
}
|
|
16
16
|
}
|
|
17
17
|
```
|
|
@@ -24,3 +24,43 @@ tighten validation or packaging contracts even when its base version remains
|
|
|
24
24
|
The trace, mission, and health protocol versions are independent from the npm
|
|
25
25
|
package version. Consumers should reject unsupported artifact or protocol
|
|
26
26
|
versions rather than guessing how to interpret them.
|
|
27
|
+
|
|
28
|
+
## Prerelease migration notes
|
|
29
|
+
|
|
30
|
+
### 0.1.0-next.5
|
|
31
|
+
|
|
32
|
+
No artifact schema or protocol changes. Three behavioral changes:
|
|
33
|
+
|
|
34
|
+
- **`seed.workspace` no longer writes `type`.** The insert carries only
|
|
35
|
+
`name` plus caller-supplied values — the harness never invents a column
|
|
36
|
+
value it wasn't given. A schema whose `workspaces` table has required
|
|
37
|
+
columns without defaults (the conventional template's `type` included)
|
|
38
|
+
must supply them via `seed.workspace(name, { columns: { … } })` or the new
|
|
39
|
+
`workspaceColumns` option on `assert.tenantIsolation`, or add a database
|
|
40
|
+
default.
|
|
41
|
+
- **`createProofServiceClient` refuses non-local targets.** Any
|
|
42
|
+
`NEXT_PUBLIC_SUPABASE_URL` whose parsed hostname is not
|
|
43
|
+
`localhost`/`127.0.0.1`/`[::1]` aborts as `unsafe_database` before a
|
|
44
|
+
client is constructed, because the suite deletes the users and workspaces
|
|
45
|
+
it seeds. CI setups reaching Supabase through a docker-network hostname
|
|
46
|
+
call `allowProofServiceHosts(["<host>"])` in code; there is no environment
|
|
47
|
+
override.
|
|
48
|
+
- **`modules-check` gains an accept-list.** `.proof/module-policy.json`
|
|
49
|
+
(`schemaVersion: 1`, `acceptedUndescribed: [{ module, reason }]`) lets an
|
|
50
|
+
adopting consumer record undescribed required-root modules as visible debt
|
|
51
|
+
instead of turning the check off. Entries that stop applying fail as
|
|
52
|
+
`module_policy_stale`. Absent file = no acceptances; existing consumers
|
|
53
|
+
are unaffected.
|
|
54
|
+
|
|
55
|
+
### 0.1.0-next.4
|
|
56
|
+
|
|
57
|
+
No entry-point, artifact schema, or protocol changes. The `mutate` command
|
|
58
|
+
tightens validation: each mutation's subject is now read back after apply and
|
|
59
|
+
again after its proof, so a defect that never took effect fails as
|
|
60
|
+
`NOT PLANTED` and one that was externally undone mid-run fails as
|
|
61
|
+
`UN-PLANTED` — both distinct from a missed proof. A catalog mutation whose
|
|
62
|
+
`apply` intentionally leaves its subject unchanged (it plants a side object
|
|
63
|
+
that `cleanup` removes) must now declare `applyDoesNotChangeSubject: true`;
|
|
64
|
+
runs with such undeclared mutations fail until the flag is added. The shared
|
|
65
|
+
dev server is also started before the first defect is planted rather than
|
|
66
|
+
inside the first mutation's window.
|
package/PROOF_SDK_CONTRACT.md
CHANGED
|
@@ -941,6 +941,40 @@ operation }` assertion has neither a matching mutation nor a valid entry in
|
|
|
941
941
|
`.proof/mutation-policy.json`, pinned to the current spec hash and a passing
|
|
942
942
|
positive control. A changed/deleted claim, changed spec, missing control, or
|
|
943
943
|
newly added mutation makes the acceptance stale and fails inventory.
|
|
944
|
+
- **A proof run cannot target a hosted database.** `createProofServiceClient`
|
|
945
|
+
parses `NEXT_PUBLIC_SUPABASE_URL` and hard-aborts as `unsafe_database` on
|
|
946
|
+
any hostname that is not local — matched on the parsed hostname, never by
|
|
947
|
+
substring, so `localhost.example.com` does not pass. The suite deletes the
|
|
948
|
+
auth users and workspaces it seeds; against a hosted project that failure
|
|
949
|
+
is silent and unrecoverable. There is deliberately no environment-variable
|
|
950
|
+
override: a genuinely disposable non-local host (a docker-network CI
|
|
951
|
+
hostname) is allowed in code via `allowProofServiceHosts`, where a reviewer
|
|
952
|
+
sees it. The mutation runner applies the same refusal to its psql target.
|
|
953
|
+
- **The harness never invents a column value it wasn't given.** Anywhere the
|
|
954
|
+
package writes to a consumer-owned table, the values are caller-supplied or
|
|
955
|
+
the column is omitted — tenancy shape is a consumer decision.
|
|
956
|
+
`seed.workspace` writes only `name`; a schema that requires more states it
|
|
957
|
+
via `seed.workspace(name, { columns })` or `workspaceColumns` on
|
|
958
|
+
`assert.tenantIsolation`.
|
|
959
|
+
- **Accepted descriptor gaps expire.** `proof:modules:check` accepts an
|
|
960
|
+
undescribed required-root module only through `.proof/module-policy.json`
|
|
961
|
+
(`schemaVersion: 1`, `acceptedUndescribed: [{ module, reason }]`), each
|
|
962
|
+
entry carrying a written reason, and prints the accepted backlog on every
|
|
963
|
+
run. An acceptance whose module gains a descriptor or no longer exists
|
|
964
|
+
fails as `module_policy_stale`, so paid-down debt leaves the file rather
|
|
965
|
+
than lingering as invisible permission.
|
|
966
|
+
- **A mutation that is not demonstrably live cannot judge a proof.** The
|
|
967
|
+
mutation runner never (re)applies migrations; it starts (or adopts) the
|
|
968
|
+
shared dev server before the first defect is planted, then reads each
|
|
969
|
+
mutation's subject back after apply and again after the proof. An apply that
|
|
970
|
+
left the subject unchanged fails as NOT PLANTED (a `REVOKE` no-op, or a
|
|
971
|
+
privilege still reachable via `PUBLIC` or an inherited role); a subject that
|
|
972
|
+
reverted mid-run — for example consumer tooling re-applying a wholesale
|
|
973
|
+
`REVOKE ALL; GRANT …` migration — fails as UN-PLANTED. Neither is reported
|
|
974
|
+
as a missed proof, because the proof never saw the defect. A mutation whose
|
|
975
|
+
apply intentionally leaves its subject untouched (it plants a side object
|
|
976
|
+
that `cleanup` removes) must declare `applyDoesNotChangeSubject: true` in
|
|
977
|
+
the catalog to opt out of the two subject comparisons explicitly.
|
|
944
978
|
|
|
945
979
|
`pnpm proof:mutate` is the regression test for these guarantees: it re-opens known
|
|
946
980
|
vulnerabilities in the local database one at a time and fails if the corresponding
|
package/README.md
CHANGED
|
@@ -17,7 +17,7 @@ Public prereleases are published under the npm `next` tag. Pin an exact
|
|
|
17
17
|
prerelease version while the API is stabilizing:
|
|
18
18
|
|
|
19
19
|
```sh
|
|
20
|
-
pnpm add proof-harness@0.1.0-next.
|
|
20
|
+
pnpm add proof-harness@0.1.0-next.5
|
|
21
21
|
```
|
|
22
22
|
|
|
23
23
|
The package exposes environment-specific entry points:
|
|
@@ -32,6 +32,34 @@ See [PROOF_SDK_CONTRACT.md](./PROOF_SDK_CONTRACT.md) for the consumer contract
|
|
|
32
32
|
and [COMPATIBILITY.md](./COMPATIBILITY.md) for protocol/schema compatibility
|
|
33
33
|
and consumer pinning requirements.
|
|
34
34
|
|
|
35
|
+
## Portability map
|
|
36
|
+
|
|
37
|
+
The package has two layers; consumers should know which one they are leaning
|
|
38
|
+
on:
|
|
39
|
+
|
|
40
|
+
- **Assumes nothing about your schema** — trace recording and the assertion
|
|
41
|
+
vocabulary, mission validation, coverage and its ratchet, drift, mutation
|
|
42
|
+
testing, module checks, and the CLI. These read your generated schema and
|
|
43
|
+
config, or take what you give them.
|
|
44
|
+
- **Assumes the conventional Supabase/workspace layout** — `seed.*` (a
|
|
45
|
+
`workspaces` table with a `name` column, a `workspace_members` join table,
|
|
46
|
+
and a `public.users` mirror maintained by a `handle_new_user` trigger) and
|
|
47
|
+
`assert.tenantIsolation`, which seeds through those helpers. The seed
|
|
48
|
+
helpers never invent column values: a schema that requires more than the
|
|
49
|
+
helpers are given states it via `seed.workspace(name, { columns })` or
|
|
50
|
+
`workspaceColumns` on `assert.tenantIsolation`.
|
|
51
|
+
|
|
52
|
+
The `/api/proof/*` routes and the `NEXT_PUBLIC_SUPABASE_URL` /
|
|
53
|
+
`SUPABASE_SECRET_KEY` environment names are documented protocol rather than
|
|
54
|
+
template leakage.
|
|
55
|
+
|
|
56
|
+
`createProofServiceClient` refuses every non-local Supabase target
|
|
57
|
+
(`unsafe_database`): the proof suite deletes the auth users and workspaces it
|
|
58
|
+
seeds, so a hosted project must never be reachable. A genuinely disposable
|
|
59
|
+
non-local host — a docker-network hostname in CI — is allowed explicitly in
|
|
60
|
+
code via `allowProofServiceHosts`; there is deliberately no environment
|
|
61
|
+
override.
|
|
62
|
+
|
|
35
63
|
The package also provides one `proof-harness` executable with `scan`, `parse`,
|
|
36
64
|
`registry`, `build`, `verify`, `coverage`, `inventory`, `drift`, `modules`,
|
|
37
65
|
`modules-check`, and `mutate` subcommands. Commands read repository paths from
|
package/cli/config.mjs
CHANGED
|
@@ -17,7 +17,12 @@
|
|
|
17
17
|
// module_missing every core/extension module HAS a descriptor. The one
|
|
18
18
|
// that matters most — without it, deleting a descriptor
|
|
19
19
|
// makes every other check on that module vanish and CI
|
|
20
|
-
// stays green.
|
|
20
|
+
// stays green. A module may instead be accepted as
|
|
21
|
+
// undescribed in .proof/module-policy.json with a
|
|
22
|
+
// written reason (module_policy / module_policy_stale
|
|
23
|
+
// guard that file), so an adopting consumer pays the
|
|
24
|
+
// descriptor debt down module by module instead of
|
|
25
|
+
// switching the check off.
|
|
21
26
|
// module_identity id matches its directory; kind matches its root
|
|
22
27
|
// module_shape the descriptor is actually the shape it claims
|
|
23
28
|
// module_question_shape ids unique and snake_case, options real and distinct,
|
|
@@ -58,6 +63,7 @@ process.on("warning", (warning) => {
|
|
|
58
63
|
|
|
59
64
|
const MODULES_PATH = CONFIG.artifacts.modules;
|
|
60
65
|
const ENV_EXAMPLE = CONFIG.repository.envExample;
|
|
66
|
+
const MODULE_POLICY_PATH = CONFIG.policies.module;
|
|
61
67
|
const DESCRIPTOR = "module.meta.ts";
|
|
62
68
|
|
|
63
69
|
/** Roots that must carry a descriptor, and the kind each implies. */
|
|
@@ -193,22 +199,148 @@ function findCycles(modules) {
|
|
|
193
199
|
);
|
|
194
200
|
}
|
|
195
201
|
|
|
202
|
+
// ---------------------------------------------------------------------------
|
|
203
|
+
// Module policy
|
|
204
|
+
//
|
|
205
|
+
// A consumer adopting the package with existing undescribed modules would
|
|
206
|
+
// otherwise face an all-or-nothing gate: write every considered descriptor
|
|
207
|
+
// before the check can pass once, or switch the check off — and a check that
|
|
208
|
+
// is off finds nothing at all. `.proof/module-policy.json` turns the wall
|
|
209
|
+
// into the same ratchet coverage uses: a module may be accepted as
|
|
210
|
+
// undescribed WITH a written reason, the listing is the visible backlog, and
|
|
211
|
+
// an entry that no longer applies fails as stale so paid-down debt leaves
|
|
212
|
+
// the file.
|
|
213
|
+
// ---------------------------------------------------------------------------
|
|
214
|
+
|
|
215
|
+
function readModulePolicy(policyPath) {
|
|
216
|
+
const empty = { acceptedUndescribed: [] };
|
|
217
|
+
if (!fs.existsSync(policyPath)) return { policy: empty, problems: [] };
|
|
218
|
+
try {
|
|
219
|
+
const policy = JSON.parse(fs.readFileSync(policyPath, "utf8"));
|
|
220
|
+
if (policy.schemaVersion !== 1) {
|
|
221
|
+
return {
|
|
222
|
+
policy: empty,
|
|
223
|
+
problems: [
|
|
224
|
+
{
|
|
225
|
+
category: "module_policy",
|
|
226
|
+
message: `${policyPath} has unsupported schemaVersion ${policy.schemaVersion ?? "<missing>"}; expected 1.`,
|
|
227
|
+
},
|
|
228
|
+
],
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
return { policy, problems: [] };
|
|
232
|
+
} catch (error) {
|
|
233
|
+
return {
|
|
234
|
+
policy: empty,
|
|
235
|
+
problems: [
|
|
236
|
+
{
|
|
237
|
+
category: "module_policy",
|
|
238
|
+
message: `cannot read ${policyPath}: ${error instanceof Error ? error.message : String(error)}`,
|
|
239
|
+
},
|
|
240
|
+
],
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Judge every undescribed module against the accepted-undescribed policy.
|
|
247
|
+
*
|
|
248
|
+
* `missing` is `[{ root, name, kind }]` for required-root modules without a
|
|
249
|
+
* descriptor; `described` is the set of module ids that have one. Returns the
|
|
250
|
+
* problems to report and the acceptances that applied, so the caller can make
|
|
251
|
+
* the surviving debt visible without opening the file.
|
|
252
|
+
*/
|
|
253
|
+
export function evaluateModulePolicy({
|
|
254
|
+
missing,
|
|
255
|
+
described,
|
|
256
|
+
policy,
|
|
257
|
+
policyPath,
|
|
258
|
+
descriptor = DESCRIPTOR,
|
|
259
|
+
}) {
|
|
260
|
+
const problems = [];
|
|
261
|
+
const entries = Array.isArray(policy?.acceptedUndescribed)
|
|
262
|
+
? policy.acceptedUndescribed
|
|
263
|
+
: [];
|
|
264
|
+
|
|
265
|
+
const byModule = new Map();
|
|
266
|
+
for (const entry of entries) {
|
|
267
|
+
if (
|
|
268
|
+
typeof entry?.module !== "string" ||
|
|
269
|
+
typeof entry?.reason !== "string" ||
|
|
270
|
+
entry.reason.trim() === ""
|
|
271
|
+
) {
|
|
272
|
+
problems.push({
|
|
273
|
+
category: "module_policy",
|
|
274
|
+
message: `${policyPath}: every acceptedUndescribed entry needs a "module" path and a non-empty "reason", found ${JSON.stringify(entry)}. An acceptance without a written reason is not a decision anybody can revisit.`,
|
|
275
|
+
});
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
if (byModule.has(entry.module)) {
|
|
279
|
+
problems.push({
|
|
280
|
+
category: "module_policy",
|
|
281
|
+
message: `${policyPath}: duplicate acceptedUndescribed entry for "${entry.module}".`,
|
|
282
|
+
});
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
byModule.set(entry.module, entry);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
const accepted = [];
|
|
289
|
+
const usedModules = new Set();
|
|
290
|
+
for (const { root, name, kind } of missing) {
|
|
291
|
+
const key = `${root}/${name}`;
|
|
292
|
+
const acceptance = byModule.get(key);
|
|
293
|
+
if (acceptance) {
|
|
294
|
+
usedModules.add(key);
|
|
295
|
+
accepted.push(acceptance);
|
|
296
|
+
continue;
|
|
297
|
+
}
|
|
298
|
+
problems.push({
|
|
299
|
+
category: "module_missing",
|
|
300
|
+
message: `expected ${root}/${name}/${descriptor} to exist, found nothing. Without a descriptor this module is invisible to a planner, and every other check on it silently does not run.`,
|
|
301
|
+
suggestion:
|
|
302
|
+
`Add ${root}/${name}/${descriptor} declaring at least id, kind and purpose (kind: "${kind}"), ` +
|
|
303
|
+
`or record the module in ${policyPath} under acceptedUndescribed with a durable reason — the listing is the backlog.`,
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
const describedIds = new Set(described);
|
|
308
|
+
for (const [key, entry] of byModule) {
|
|
309
|
+
if (usedModules.has(key)) continue;
|
|
310
|
+
const basename = key.split("/").at(-1);
|
|
311
|
+
const why = describedIds.has(basename)
|
|
312
|
+
? "the module now has a descriptor — delete the acceptance"
|
|
313
|
+
: "no required-root module directory matches that path — delete or fix the acceptance";
|
|
314
|
+
problems.push({
|
|
315
|
+
category: "module_policy_stale",
|
|
316
|
+
message: `${policyPath}: acceptance for "${entry.module}" no longer applies: ${why}.`,
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
return { problems, accepted };
|
|
321
|
+
}
|
|
322
|
+
|
|
196
323
|
// ---------------------------------------------------------------------------
|
|
197
324
|
// Checks
|
|
198
325
|
// ---------------------------------------------------------------------------
|
|
199
326
|
|
|
200
|
-
function checkCoverage(modules, problems) {
|
|
327
|
+
function checkCoverage(modules, policyResult, problems) {
|
|
201
328
|
const described = new Set(modules.map((m) => m.id));
|
|
329
|
+
const missing = [];
|
|
202
330
|
for (const { root, kind } of REQUIRED_ROOTS) {
|
|
203
331
|
for (const name of directoriesIn(root)) {
|
|
204
332
|
if (described.has(name)) continue;
|
|
205
|
-
|
|
206
|
-
category: "module_missing",
|
|
207
|
-
message: `expected ${root}/${name}/${DESCRIPTOR} to exist, found nothing. Without a descriptor this module is invisible to a planner, and every other check on it silently does not run.`,
|
|
208
|
-
suggestion: `Add ${root}/${name}/${DESCRIPTOR} declaring at least id, kind and purpose (kind: "${kind}").`,
|
|
209
|
-
});
|
|
333
|
+
missing.push({ root, name, kind });
|
|
210
334
|
}
|
|
211
335
|
}
|
|
336
|
+
const outcome = evaluateModulePolicy({
|
|
337
|
+
missing,
|
|
338
|
+
described,
|
|
339
|
+
policy: policyResult.policy,
|
|
340
|
+
policyPath: MODULE_POLICY_PATH,
|
|
341
|
+
});
|
|
342
|
+
problems.push(...policyResult.problems, ...outcome.problems);
|
|
343
|
+
return outcome.accepted;
|
|
212
344
|
}
|
|
213
345
|
|
|
214
346
|
function checkIdentity(mod, problems) {
|
|
@@ -429,7 +561,11 @@ async function main() {
|
|
|
429
561
|
const seenIds = new Map();
|
|
430
562
|
const problems = [];
|
|
431
563
|
|
|
432
|
-
checkCoverage(
|
|
564
|
+
const acceptedUndescribed = checkCoverage(
|
|
565
|
+
modules,
|
|
566
|
+
readModulePolicy(MODULE_POLICY_PATH),
|
|
567
|
+
problems,
|
|
568
|
+
);
|
|
433
569
|
for (const mod of modules) {
|
|
434
570
|
checkIdentity(mod, problems);
|
|
435
571
|
checkShape(mod, MODULE_KINDS, problems);
|
|
@@ -442,7 +578,16 @@ async function main() {
|
|
|
442
578
|
|
|
443
579
|
if (asJson) {
|
|
444
580
|
console.log(
|
|
445
|
-
JSON.stringify(
|
|
581
|
+
JSON.stringify(
|
|
582
|
+
{
|
|
583
|
+
modules: modules.length,
|
|
584
|
+
acceptedUndescribed,
|
|
585
|
+
problems,
|
|
586
|
+
cycles,
|
|
587
|
+
},
|
|
588
|
+
null,
|
|
589
|
+
2,
|
|
590
|
+
),
|
|
446
591
|
);
|
|
447
592
|
}
|
|
448
593
|
|
|
@@ -475,10 +620,20 @@ async function main() {
|
|
|
475
620
|
);
|
|
476
621
|
}
|
|
477
622
|
|
|
623
|
+
if (acceptedUndescribed.length > 0) {
|
|
624
|
+
console.log(
|
|
625
|
+
`[proof:modules:check] ${acceptedUndescribed.length} module(s) accepted as undescribed by ${MODULE_POLICY_PATH} — that listing is the backlog:`,
|
|
626
|
+
);
|
|
627
|
+
for (const entry of acceptedUndescribed) {
|
|
628
|
+
console.log(` - ${entry.module}: ${entry.reason}`);
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
|
|
478
632
|
console.log(
|
|
479
633
|
`[proof:modules:check] ${modules.length} module descriptor(s) verified against source: ` +
|
|
480
634
|
`${decisions} decision(s), ${inputs} input(s), ${defaults} default(s), ` +
|
|
481
|
-
`${artifact.unowned_tables?.length ?? 0} unowned table(s)
|
|
635
|
+
`${artifact.unowned_tables?.length ?? 0} unowned table(s), ` +
|
|
636
|
+
`${acceptedUndescribed.length} accepted undescribed module(s).`,
|
|
482
637
|
);
|
|
483
638
|
}
|
|
484
639
|
|
|
@@ -495,4 +650,4 @@ if (invokedDirectly) {
|
|
|
495
650
|
});
|
|
496
651
|
}
|
|
497
652
|
|
|
498
|
-
export { main, seamProblem, findCycles };
|
|
653
|
+
export { main, seamProblem, findCycles, readModulePolicy };
|
|
@@ -12,7 +12,10 @@
|
|
|
12
12
|
//
|
|
13
13
|
// Mutations are applied to the DATABASE, not the source tree: they are precise,
|
|
14
14
|
// instant and reversible, where patching SQL files would require a full
|
|
15
|
-
// migration cycle per mutation.
|
|
15
|
+
// migration cycle per mutation. Migrations are never (re)applied by this
|
|
16
|
+
// script; each planted defect is read back after apply and again after the
|
|
17
|
+
// proof, so a plant that never took effect — or was externally undone
|
|
18
|
+
// mid-run — fails as its own finding instead of being blamed on the proof.
|
|
16
19
|
//
|
|
17
20
|
// The mutation list is also the honest answer to "what does the baseline suite
|
|
18
21
|
// actually detect?" — each entry names a finding, the hole it re-opens, and the
|
|
@@ -39,6 +42,10 @@ import {
|
|
|
39
42
|
evaluateMutationClaimCoverage,
|
|
40
43
|
mutationCoversClaim,
|
|
41
44
|
} from "./proof_mutation_inventory.mjs";
|
|
45
|
+
import {
|
|
46
|
+
assessPlantedSubject,
|
|
47
|
+
assessSubjectAfterProof,
|
|
48
|
+
} from "./proof_plant_verification.mjs";
|
|
42
49
|
import {
|
|
43
50
|
applySourceMutation,
|
|
44
51
|
restoreSourceMutation,
|
|
@@ -913,6 +920,15 @@ Exits non-zero if any proof failed to notice its vulnerability.`);
|
|
|
913
920
|
let devServer = null;
|
|
914
921
|
|
|
915
922
|
try {
|
|
923
|
+
// Start (or adopt) the shared server BEFORE the first defect is planted.
|
|
924
|
+
// Consumer `pnpm dev` bootstrap hooks — migration re-application, seeding,
|
|
925
|
+
// schema reconciliation — must run against the healthy database, not
|
|
926
|
+
// inside the first mutation's window, where a wholesale re-GRANT would
|
|
927
|
+
// silently un-plant it.
|
|
928
|
+
if (selected.some((mutation) => !isSourceMutation(mutation))) {
|
|
929
|
+
devServer = await sharedDevServer();
|
|
930
|
+
}
|
|
931
|
+
|
|
916
932
|
for (const m of selected) {
|
|
917
933
|
console.log(`── ${m.id} ──────────────────────────────────────────`);
|
|
918
934
|
console.log(` planting: ${m.breaks}`);
|
|
@@ -933,6 +949,19 @@ Exits non-zero if any proof failed to notice its vulnerability.`);
|
|
|
933
949
|
applyMutation(container, m, snapshot);
|
|
934
950
|
applied = true;
|
|
935
951
|
|
|
952
|
+
// A defect that is not demonstrably live proves nothing: running the
|
|
953
|
+
// proof anyway would blame the proof ("MISSED") for a plant that never
|
|
954
|
+
// happened. Read the subject back and refuse to continue on mismatch.
|
|
955
|
+
const plantedSnapshot = handler.snapshot(container, m.subject);
|
|
956
|
+
const plantProblem = assessPlantedSubject({
|
|
957
|
+
id: m.id,
|
|
958
|
+
description: handler.describe(m.subject),
|
|
959
|
+
before: snapshot,
|
|
960
|
+
afterApply: plantedSnapshot,
|
|
961
|
+
applyDoesNotChangeSubject: m.applyDoesNotChangeSubject === true,
|
|
962
|
+
});
|
|
963
|
+
if (plantProblem) throw new Error(plantProblem);
|
|
964
|
+
|
|
936
965
|
if (isSourceMutation(m)) {
|
|
937
966
|
if (devServer) {
|
|
938
967
|
await devServer.stop();
|
|
@@ -947,6 +976,22 @@ Exits non-zero if any proof failed to notice its vulnerability.`);
|
|
|
947
976
|
const { code, output } = await runProof(m.spec, m.id);
|
|
948
977
|
const playwrightFailed = code !== 0;
|
|
949
978
|
|
|
979
|
+
// The defect must still be live now that the proof has finished. If
|
|
980
|
+
// the subject reverted mid-run (issue #7: a re-applied migration
|
|
981
|
+
// re-granting a revoked privilege), the proof's verdict — green or
|
|
982
|
+
// red — says nothing about the proof, and archiving it as MISSED
|
|
983
|
+
// would misdirect the operator toward a working proof.
|
|
984
|
+
const afterProofSnapshot = handler.snapshot(container, m.subject);
|
|
985
|
+
const liveProblem = assessSubjectAfterProof({
|
|
986
|
+
id: m.id,
|
|
987
|
+
description: handler.describe(m.subject),
|
|
988
|
+
before: snapshot,
|
|
989
|
+
afterApply: plantedSnapshot,
|
|
990
|
+
afterProof: afterProofSnapshot,
|
|
991
|
+
applyDoesNotChangeSubject: m.applyDoesNotChangeSubject === true,
|
|
992
|
+
});
|
|
993
|
+
if (liveProblem) throw new Error(liveProblem);
|
|
994
|
+
|
|
950
995
|
// A proof can fail for the wrong reason (a crash, a timeout, a missing
|
|
951
996
|
// browser). Where the reason matters — as with the deny-all mutation that
|
|
952
997
|
// only a positive control can catch — require it in the output.
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// proof_plant_verification.mjs
|
|
3
|
+
//
|
|
4
|
+
// A mutation run is only evidence if the planted defect was actually live for
|
|
5
|
+
// the entire proof. These checks compare the subject's snapshot at three
|
|
6
|
+
// moments — before apply, after apply, and after the proof — and turn every
|
|
7
|
+
// silent mismatch into a structured, correctly-attributed failure instead of
|
|
8
|
+
// letting it masquerade as "MISSED — the proof cannot fail":
|
|
9
|
+
//
|
|
10
|
+
// - apply changed nothing → the defect never went live (NOT PLANTED)
|
|
11
|
+
// - subject reverted mid-proof → something un-planted it (UN-PLANTED)
|
|
12
|
+
// - subject drifted mid-proof → the run cannot be attributed
|
|
13
|
+
//
|
|
14
|
+
// Without them, an ineffective REVOKE (the privilege still reachable via
|
|
15
|
+
// PUBLIC or an inherited role, a grantor no-op, or tooling that re-applies
|
|
16
|
+
// migrations mid-run) leaves the proof green against a healthy database, and
|
|
17
|
+
// the summary blames the proof for a defect that was never live.
|
|
18
|
+
//
|
|
19
|
+
// A mutation whose apply intentionally leaves its subject untouched — it
|
|
20
|
+
// plants a side object that `cleanup` removes, and the subject only anchors
|
|
21
|
+
// restore — must declare `applyDoesNotChangeSubject: true` in the catalog.
|
|
22
|
+
// Such mutations keep the historical blind spot by explicit, reviewable
|
|
23
|
+
// choice rather than by accident.
|
|
24
|
+
// ---------------------------------------------------------------------------
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Judge the subject immediately after `apply`. Returns an error message when
|
|
28
|
+
* the run must stop, or null when the plant is coherent.
|
|
29
|
+
*/
|
|
30
|
+
export function assessPlantedSubject({
|
|
31
|
+
id,
|
|
32
|
+
description,
|
|
33
|
+
before,
|
|
34
|
+
afterApply,
|
|
35
|
+
applyDoesNotChangeSubject = false,
|
|
36
|
+
}) {
|
|
37
|
+
if (applyDoesNotChangeSubject) {
|
|
38
|
+
if (afterApply === before) return null;
|
|
39
|
+
return (
|
|
40
|
+
`mutation ${id} declares applyDoesNotChangeSubject, but applying it changed ` +
|
|
41
|
+
`${description}. The declaration is wrong; remove it so the subject is verified.\n` +
|
|
42
|
+
` before apply: ${before}\n after apply: ${afterApply}`
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
if (afterApply !== before) return null;
|
|
46
|
+
return (
|
|
47
|
+
`mutation ${id} is NOT PLANTED: ${description} reads exactly the same before and after ` +
|
|
48
|
+
`apply, so the defect is not live and its proof would run against a healthy database. ` +
|
|
49
|
+
`Common causes: the privilege is still held via PUBLIC or an inherited role, the ` +
|
|
50
|
+
`GRANT/REVOKE was a silent no-op, or concurrent tooling immediately restored the subject. ` +
|
|
51
|
+
`If the apply intentionally leaves the subject untouched (it plants a side object that ` +
|
|
52
|
+
`cleanup removes), declare applyDoesNotChangeSubject: true on the mutation.\n` +
|
|
53
|
+
` subject snapshot: ${before}`
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Judge the subject after the proof ran, before restore. Returns an error
|
|
59
|
+
* message when the defect was no longer (or differently) planted, or null
|
|
60
|
+
* when the proof demonstrably ran against the planted defect.
|
|
61
|
+
*/
|
|
62
|
+
export function assessSubjectAfterProof({
|
|
63
|
+
id,
|
|
64
|
+
description,
|
|
65
|
+
before,
|
|
66
|
+
afterApply,
|
|
67
|
+
afterProof,
|
|
68
|
+
applyDoesNotChangeSubject = false,
|
|
69
|
+
}) {
|
|
70
|
+
const expected = applyDoesNotChangeSubject ? before : afterApply;
|
|
71
|
+
if (afterProof === expected) return null;
|
|
72
|
+
if (afterProof === before) {
|
|
73
|
+
return (
|
|
74
|
+
`mutation ${id} was UN-PLANTED while its proof ran: ${description} returned to its ` +
|
|
75
|
+
`pre-mutation state before the proof finished, so the proof never had a chance to detect ` +
|
|
76
|
+
`the defect and its verdict is meaningless. Something outside the harness restored the ` +
|
|
77
|
+
`subject mid-run — typically dev-server bootstrap or migration tooling re-applying grants ` +
|
|
78
|
+
`(for example a wholesale "REVOKE ALL; GRANT ..." migration). This is a ` +
|
|
79
|
+
`harness/environment failure, not a missed proof.`
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
return (
|
|
83
|
+
`mutation ${id} lost its planted state while its proof ran: ${description} changed to a ` +
|
|
84
|
+
`state matching neither the planted defect nor the original, so the run cannot be ` +
|
|
85
|
+
`attributed to the mutation.\n` +
|
|
86
|
+
` before apply: ${before}\n after apply: ${afterApply}\n after proof: ${afterProof}`
|
|
87
|
+
);
|
|
88
|
+
}
|
|
@@ -25,8 +25,16 @@ declare const seed: {
|
|
|
25
25
|
*
|
|
26
26
|
* Bypasses RLS via the service-role client, so it works even when no user
|
|
27
27
|
* is authenticated.
|
|
28
|
+
*
|
|
29
|
+
* The insert writes only `name` plus whatever `opts.columns` supplies: the
|
|
30
|
+
* harness never invents a column value it wasn't given, because tenancy
|
|
31
|
+
* shape is a consumer decision. A schema whose `workspaces` table requires
|
|
32
|
+
* more than `name` (a NOT NULL column without a default) states those
|
|
33
|
+
* values here — or via `workspaceColumns` on `assert.tenantIsolation`.
|
|
28
34
|
*/
|
|
29
|
-
workspace(name: string
|
|
35
|
+
workspace(name: string, opts?: {
|
|
36
|
+
columns?: Record<string, unknown>;
|
|
37
|
+
}): Promise<SeedWorkspace>;
|
|
30
38
|
/**
|
|
31
39
|
* Create an auth user with the given credentials and attach them to the
|
|
32
40
|
* provided workspace with the given role. Relies on the `handle_new_user`
|
package/dist/playwright.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Page, APIResponse } from '@playwright/test';
|
|
2
2
|
import { SupabaseClient } from '@supabase/supabase-js';
|
|
3
3
|
import { A as ActionResult } from './action-types-C3OmHRyH.js';
|
|
4
|
-
import { P as ProofFixtureContext, a as ProofFixtureFactory } from './fixture-
|
|
4
|
+
import { P as ProofFixtureContext, a as ProofFixtureFactory } from './fixture-DwDOegDK.js';
|
|
5
5
|
import { h as ProofKind, d as AssertionRole, a as TraceAssertion, j as ProofOptions, q as TraceStep, m as StepResult, S as StepOptions } from './trace-types-DkIzq-l9.js';
|
|
6
6
|
|
|
7
7
|
declare const actAsUser: {
|
|
@@ -146,6 +146,13 @@ interface TenantIsolationBaseOptions {
|
|
|
146
146
|
* runs don't collide on error paths where teardown is skipped.
|
|
147
147
|
*/
|
|
148
148
|
tag?: string;
|
|
149
|
+
/**
|
|
150
|
+
* Extra column values for the two seeded workspaces, forwarded to
|
|
151
|
+
* `seed.workspace`. The harness itself writes only `name`; a schema whose
|
|
152
|
+
* `workspaces` table requires more (a NOT NULL column without a default)
|
|
153
|
+
* states those values here.
|
|
154
|
+
*/
|
|
155
|
+
workspaceColumns?: Record<string, unknown>;
|
|
149
156
|
}
|
|
150
157
|
/**
|
|
151
158
|
* Tenant fixture source.
|
package/dist/playwright.js
CHANGED
|
@@ -599,6 +599,27 @@ var trace = {
|
|
|
599
599
|
|
|
600
600
|
// src/server/service-client.ts
|
|
601
601
|
import { createClient as createClient2 } from "@supabase/supabase-js";
|
|
602
|
+
var LOCAL_HOSTS = /* @__PURE__ */ new Set(["localhost", "127.0.0.1", "[::1]", "::1"]);
|
|
603
|
+
var extraAllowedHosts = /* @__PURE__ */ new Set();
|
|
604
|
+
function proofServiceHostProblem(rawUrl, allowedHosts = extraAllowedHosts) {
|
|
605
|
+
let hostname;
|
|
606
|
+
try {
|
|
607
|
+
hostname = new URL(rawUrl).hostname;
|
|
608
|
+
} catch {
|
|
609
|
+
return `NEXT_PUBLIC_SUPABASE_URL is not a parseable URL (${JSON.stringify(rawUrl)}), so it cannot be verified as local`;
|
|
610
|
+
}
|
|
611
|
+
if (LOCAL_HOSTS.has(hostname) || allowedHosts.has(hostname)) return null;
|
|
612
|
+
return `NEXT_PUBLIC_SUPABASE_URL points at a non-local Supabase (host: ${hostname})`;
|
|
613
|
+
}
|
|
614
|
+
function assertProofServiceUrlIsSafe(rawUrl) {
|
|
615
|
+
const problem = proofServiceHostProblem(rawUrl);
|
|
616
|
+
if (!problem) return;
|
|
617
|
+
throw new Error(
|
|
618
|
+
`[PROOF_FAIL] unsafe_database: expected a local Supabase target, found ${problem}
|
|
619
|
+
file: src/server/service-client.ts
|
|
620
|
+
suggestion: The proof suite seeds and DELETES auth users and workspaces, so it refuses every non-local database. Point NEXT_PUBLIC_SUPABASE_URL at a local stack (pnpm exec supabase start). A genuinely disposable non-local host (for example a docker-network hostname in CI) must be allowed in code via allowProofServiceHosts([...]); there is deliberately no environment-variable override.`
|
|
621
|
+
);
|
|
622
|
+
}
|
|
602
623
|
function requireProofEnv(name) {
|
|
603
624
|
const value = process.env[name];
|
|
604
625
|
if (!value) {
|
|
@@ -611,8 +632,10 @@ function requireProofEnv(name) {
|
|
|
611
632
|
return value;
|
|
612
633
|
}
|
|
613
634
|
function createProofServiceClient() {
|
|
635
|
+
const url = requireProofEnv("NEXT_PUBLIC_SUPABASE_URL");
|
|
636
|
+
assertProofServiceUrlIsSafe(url);
|
|
614
637
|
return createClient2(
|
|
615
|
-
|
|
638
|
+
url,
|
|
616
639
|
requireProofEnv("SUPABASE_SECRET_KEY"),
|
|
617
640
|
{
|
|
618
641
|
auth: {
|
|
@@ -668,12 +691,18 @@ var seed = {
|
|
|
668
691
|
*
|
|
669
692
|
* Bypasses RLS via the service-role client, so it works even when no user
|
|
670
693
|
* is authenticated.
|
|
694
|
+
*
|
|
695
|
+
* The insert writes only `name` plus whatever `opts.columns` supplies: the
|
|
696
|
+
* harness never invents a column value it wasn't given, because tenancy
|
|
697
|
+
* shape is a consumer decision. A schema whose `workspaces` table requires
|
|
698
|
+
* more than `name` (a NOT NULL column without a default) states those
|
|
699
|
+
* values here — or via `workspaceColumns` on `assert.tenantIsolation`.
|
|
671
700
|
*/
|
|
672
|
-
async workspace(name) {
|
|
701
|
+
async workspace(name, opts) {
|
|
673
702
|
const sb = createProofServiceClient();
|
|
674
703
|
const { data, error, status } = await sb.from("workspaces").insert({
|
|
675
|
-
|
|
676
|
-
|
|
704
|
+
...opts?.columns,
|
|
705
|
+
name
|
|
677
706
|
}).select("id, name, owner_id").single();
|
|
678
707
|
if (error || !data) {
|
|
679
708
|
failSeed(
|
|
@@ -1469,8 +1498,12 @@ var assertMethods = {
|
|
|
1469
1498
|
const scope = opts.scope ?? "workspace";
|
|
1470
1499
|
const scopeColumn = opts.scopeColumn ?? (scope === "user" ? "user_id" : "workspace_id");
|
|
1471
1500
|
validateTenantIsolationCriterion(table, scopeColumn, opts.criterion);
|
|
1472
|
-
const orgA = await seed.workspace(`Proof OrgA ${tag}
|
|
1473
|
-
|
|
1501
|
+
const orgA = await seed.workspace(`Proof OrgA ${tag}`, {
|
|
1502
|
+
columns: opts.workspaceColumns
|
|
1503
|
+
});
|
|
1504
|
+
const orgB = await seed.workspace(`Proof OrgB ${tag}`, {
|
|
1505
|
+
columns: opts.workspaceColumns
|
|
1506
|
+
});
|
|
1474
1507
|
const userA = await seed.user({
|
|
1475
1508
|
email: `proof-a-${tag}@proof.test`,
|
|
1476
1509
|
password: `ProofPass!${tag}1`,
|