watertight 0.2.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +133 -81
- package/SKILL.md +2 -0
- package/dist/cli.js +39 -6
- package/dist/compile.js +24 -2
- package/dist/init.js +65 -0
- package/dist/ir.js +40 -4
- package/dist/refresh.js +58 -23
- package/dist/render.js +24 -6
- package/dist/renderMd.js +16 -6
- package/dist/scan.js +60 -22
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,29 +1,43 @@
|
|
|
1
1
|
# watertight
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/watertight)
|
|
4
|
+
[](https://github.com/camosss/watertight/actions/workflows/ci.yml)
|
|
5
|
+

|
|
6
|
+

|
|
7
|
+
[](LICENSE)
|
|
5
8
|
|
|
9
|
+
Reports that hold water — every number carries its **receipt**, and ungrounded claims **fail the build**.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
npx watertight report.md metrics.json
|
|
6
13
|
```
|
|
7
|
-
$ watertight report.md metrics.json
|
|
8
14
|
|
|
9
|
-
|
|
15
|
+
```
|
|
16
|
+
watertight v0.2.0 · 20 grounded metrics · 4 claims · 5 identifiers
|
|
10
17
|
holds water → report.html
|
|
11
18
|
```
|
|
12
19
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
20
|
+
An ungrounded number is a **leak**. A report with zero leaks **holds water**:
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
2 leak(s) — the report does not hold water:
|
|
24
|
+
|
|
25
|
+
✗ [naked-number] report.md:31 — "15%" appears in the narrative without a receipt
|
|
26
|
+
✗ [derived-mismatch] metric "revenue_total" is 1440, but its parts sum to 1428
|
|
27
|
+
```
|
|
19
28
|
|
|
20
|
-
|
|
29
|
+
<br>
|
|
21
30
|
|
|
22
31
|
## Why this exists
|
|
23
32
|
|
|
33
|
+
A number typed into prose has no memory of where it came from. Two weeks later
|
|
34
|
+
nobody can say which query produced it, whether it was rounded, or whether it
|
|
35
|
+
was ever true — and when an LLM helps write the report, "never true" is a live
|
|
36
|
+
possibility.
|
|
37
|
+
|
|
24
38
|
This came out of writing a real ad-revenue verification report with an AI
|
|
25
|
-
assistant. Compiling the draft surfaced three distinct failure classes in
|
|
26
|
-
|
|
39
|
+
assistant. Compiling the draft surfaced three distinct failure classes in one
|
|
40
|
+
document:
|
|
27
41
|
|
|
28
42
|
- a human had written **+15%** for a delta that computed to **+15.5%** —
|
|
29
43
|
a rounding transcription error nobody had caught;
|
|
@@ -31,10 +45,14 @@ one document:
|
|
|
31
45
|
- the assistant confidently "remembered" a figure that appeared nowhere in
|
|
32
46
|
any export.
|
|
33
47
|
|
|
34
|
-
All three are the same disease — a number with no receipt — and all three
|
|
35
|
-
|
|
48
|
+
All three are the same disease — a number with no receipt — and all three are
|
|
49
|
+
mechanically detectable. So: detect them, every build. watertight treats a
|
|
50
|
+
report like source code: the narrative references metrics, the metrics carry
|
|
51
|
+
provenance, and a compiler refuses to build anything it cannot trace.
|
|
36
52
|
|
|
37
|
-
|
|
53
|
+
<br>
|
|
54
|
+
|
|
55
|
+
## Quick start
|
|
38
56
|
|
|
39
57
|
```bash
|
|
40
58
|
npm install -g watertight
|
|
@@ -42,25 +60,12 @@ npm install -g watertight
|
|
|
42
60
|
npx watertight report.md metrics.json
|
|
43
61
|
```
|
|
44
62
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
## How it works
|
|
48
|
-
|
|
49
|
-
You write two files.
|
|
50
|
-
|
|
51
|
-
**`metrics.json` — the IR.** Every value, with its receipt:
|
|
63
|
+
You write two files. **`metrics.json`** holds every value with its receipt —
|
|
64
|
+
a measured metric says where it was fetched, over what window, and when:
|
|
52
65
|
|
|
53
66
|
```json
|
|
54
67
|
{
|
|
55
68
|
"metrics": {
|
|
56
|
-
"conversion_before": {
|
|
57
|
-
"value": 0.031,
|
|
58
|
-
"unit": "ratio",
|
|
59
|
-
"definition": "purchases / sessions entering checkout",
|
|
60
|
-
"source": { "type": "sql", "query": "reports/checkout.sql" },
|
|
61
|
-
"window": "2026-08-15 ~ 2026-08-31",
|
|
62
|
-
"fetched_at": "2026-09-15"
|
|
63
|
-
},
|
|
64
69
|
"conversion_after": {
|
|
65
70
|
"value": 0.036,
|
|
66
71
|
"unit": "ratio",
|
|
@@ -68,34 +73,42 @@ You write two files.
|
|
|
68
73
|
"source": { "type": "sql", "query": "reports/checkout.sql" },
|
|
69
74
|
"window": "2026-09-01 ~ 2026-09-14",
|
|
70
75
|
"fetched_at": "2026-09-15"
|
|
71
|
-
},
|
|
72
|
-
"lift": {
|
|
73
|
-
"value": 0.161,
|
|
74
|
-
"unit": "ratio-point",
|
|
75
|
-
"definition": "relative change in conversion",
|
|
76
|
-
"derived": { "op": "pct_change", "before": "conversion_before", "after": "conversion_after" }
|
|
77
|
-
},
|
|
78
|
-
"revenue_total": {
|
|
79
|
-
"value": 1428,
|
|
80
|
-
"unit": "USD",
|
|
81
|
-
"derived": { "op": "sum", "of": ["revenue_ios", "revenue_android"] }
|
|
82
|
-
},
|
|
83
|
-
"revenue_target": {
|
|
84
|
-
"value": [1000, 5000],
|
|
85
|
-
"unit": "USD",
|
|
86
|
-
"source": { "type": "hypothesis", "doc": "PLAN-42" },
|
|
87
|
-
"window": "planning estimate",
|
|
88
|
-
"fetched_at": "-"
|
|
89
76
|
}
|
|
90
77
|
},
|
|
91
|
-
"identifiers": {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
78
|
+
"identifiers": { "app_version": "3.2.0", "flag": "checkout_v2" }
|
|
79
|
+
}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
A **derived** metric is recomputed on every compile — a total that doesn't add
|
|
83
|
+
up, or a delta that doesn't recompute, is a build failure:
|
|
84
|
+
|
|
85
|
+
```json
|
|
86
|
+
"lift": {
|
|
87
|
+
"value": 0.161,
|
|
88
|
+
"unit": "ratio-point",
|
|
89
|
+
"definition": "relative change in conversion",
|
|
90
|
+
"derived": { "op": "pct_change", "before": "conversion_before", "after": "conversion_after" }
|
|
91
|
+
},
|
|
92
|
+
"revenue_total": {
|
|
93
|
+
"value": 1428,
|
|
94
|
+
"unit": "USD",
|
|
95
|
+
"derived": { "op": "sum", "of": ["revenue_ios", "revenue_android"] }
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
A **range** states a hypothesis honestly — plans are receipts too:
|
|
100
|
+
|
|
101
|
+
```json
|
|
102
|
+
"revenue_target": {
|
|
103
|
+
"value": [1000, 5000],
|
|
104
|
+
"unit": "USD",
|
|
105
|
+
"source": { "type": "hypothesis", "doc": "PLAN-42" },
|
|
106
|
+
"window": "planning estimate",
|
|
107
|
+
"fetched_at": "-"
|
|
95
108
|
}
|
|
96
109
|
```
|
|
97
110
|
|
|
98
|
-
**`report.md
|
|
111
|
+
**`report.md`** is the narrative. No literal figures, only references:
|
|
99
112
|
|
|
100
113
|
```markdown
|
|
101
114
|
Rolled out in `{{id:app_version}}` behind `{{id:flag}}`.
|
|
@@ -113,32 +126,56 @@ Compile:
|
|
|
113
126
|
|
|
114
127
|
```bash
|
|
115
128
|
watertight report.md metrics.json # → report.html (self-contained, hover for receipts)
|
|
116
|
-
watertight . --format md # → grounded markdown
|
|
129
|
+
watertight . --format md # → grounded markdown (below)
|
|
117
130
|
watertight . --check # verify only, write nothing (CI)
|
|
131
|
+
watertight . --check --max-age 30 # also fail receipts older than 30 days
|
|
118
132
|
watertight . --json # machine-readable result
|
|
119
133
|
```
|
|
120
134
|
|
|
121
|
-
|
|
135
|
+
<br>
|
|
122
136
|
|
|
123
|
-
|
|
124
|
-
2 leak(s) — the report does not hold water:
|
|
137
|
+
## What the output looks like
|
|
125
138
|
|
|
126
|
-
|
|
127
|
-
|
|
139
|
+
`--format md` produces grounded markdown that pastes into Notion, a PR body,
|
|
140
|
+
or Slack with its provenance intact — every figure bold with a superscript
|
|
141
|
+
into a receipts appendix:
|
|
142
|
+
|
|
143
|
+
```markdown
|
|
144
|
+
Conversion moved from **3.1%** ⁽¹⁾ to **3.6%** ⁽²⁾, a lift of **+16.1%p** ⁽³⁾.
|
|
145
|
+
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
### Receipts (3 metrics)
|
|
149
|
+
|
|
150
|
+
1. **conversion_before** = 3.1% — purchases / sessions entering checkout
|
|
151
|
+
sql · reports/checkout.sql · 2026-08-15 ~ 2026-08-31 · fetched 2026-09-15
|
|
152
|
+
2. **conversion_after** = 3.6% — purchases / sessions entering checkout
|
|
153
|
+
sql · reports/checkout.sql · 2026-09-01 ~ 2026-09-14 · fetched 2026-09-15
|
|
154
|
+
3. **lift** = +16.1%p — relative change in conversion
|
|
155
|
+
= conversion_before → conversion_after (recomputed)
|
|
128
156
|
```
|
|
129
157
|
|
|
158
|
+
The default HTML render is a single self-contained file where hovering any
|
|
159
|
+
figure shows its receipt.
|
|
160
|
+
|
|
161
|
+
<br>
|
|
162
|
+
|
|
130
163
|
## What it checks
|
|
131
164
|
|
|
132
165
|
| check | catches |
|
|
133
166
|
|---|---|
|
|
134
|
-
| `naked-number` | any digit in prose not covered by a reference
|
|
167
|
+
| `naked-number` | any digit in prose not covered by a reference — heading text included; dates, URLs, code spans and `{{raw:}}` are exempt |
|
|
135
168
|
| `unknown-ref` | `{{m:...}}` / `{{id:...}}` pointing at nothing |
|
|
136
169
|
| `missing-field` | a measured metric without `source`, `window`, or `fetched_at` |
|
|
137
170
|
| `definition-required` | a `ratio` / `ratio-point` metric with no stated basis — this is how a fill rate of 107% stays honest |
|
|
138
171
|
| `derived-mismatch` | a `sum` that doesn't add up; a `pct_change` that doesn't recompute — to the stored value's own precision, so `0.161` passes as 16.1% but `0.15` for 15.5% fails |
|
|
139
172
|
| `claim-without-evidence` | a `{{claim:}}` with no `evidence:` keys, or keys that don't exist |
|
|
173
|
+
| `malformed-marker` | anything still marker-shaped after every recognised form — a typo'd `{{claim:…\|evidnce:…}}` must fail, not render verbatim |
|
|
140
174
|
| `bad-derived` | derived ops referencing missing or non-numeric inputs |
|
|
141
175
|
| `empty-ir` | a report "grounded" in nothing |
|
|
176
|
+
| `stale-metric` | with `--max-age <days>`: a receipt whose `fetched_at` is older than the budget — numbers age |
|
|
177
|
+
|
|
178
|
+
<br>
|
|
142
179
|
|
|
143
180
|
## Re-verification
|
|
144
181
|
|
|
@@ -150,25 +187,6 @@ can reach, rewrites `value` and `fetched_at`, and recomputes derived values:
|
|
|
150
187
|
- `command` sources — run **only** with the explicit `--allow-commands` flag,
|
|
151
188
|
because refreshing an IR you didn't author must never execute its shell
|
|
152
189
|
commands
|
|
153
|
-
- any other source type can be covered by a **custom fetcher** — a JS module
|
|
154
|
-
you point at explicitly:
|
|
155
|
-
|
|
156
|
-
```bash
|
|
157
|
-
watertight refresh . --fetchers ./my-fetchers.mjs
|
|
158
|
-
```
|
|
159
|
-
|
|
160
|
-
```js
|
|
161
|
-
// my-fetchers.mjs — export one function per source type
|
|
162
|
-
export async function mixpanel(source) {
|
|
163
|
-
// credentials from the environment, receipt fields from the IR
|
|
164
|
-
return valueFetchedFrom(source.project, source.bookmark)
|
|
165
|
-
}
|
|
166
|
-
```
|
|
167
|
-
|
|
168
|
-
[`examples/fetchers/mixpanel.mjs`](./examples/fetchers/mixpanel.mjs) is a
|
|
169
|
-
working example. Loading a module runs its code, so only pass files you
|
|
170
|
-
wrote or trust — the IR itself can never name a fetcher. Built-in csv/json/
|
|
171
|
-
command adapters always win over a custom one of the same name.
|
|
172
190
|
- everything else (dashboards, vendor reports, hypotheses) is **named as
|
|
173
191
|
skipped** — never silently assumed fresh
|
|
174
192
|
|
|
@@ -176,6 +194,34 @@ Derived values follow their inputs: sums are recomputed exactly, percentage
|
|
|
176
194
|
changes at the precision the author stated. `--dry-run` previews changes
|
|
177
195
|
without writing.
|
|
178
196
|
|
|
197
|
+
<br>
|
|
198
|
+
|
|
199
|
+
## Custom fetchers
|
|
200
|
+
|
|
201
|
+
Any other source type can be refreshed through a JS module you point at
|
|
202
|
+
explicitly:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
watertight refresh . --fetchers ./my-fetchers.mjs
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
```js
|
|
209
|
+
// my-fetchers.mjs — export one function per source type
|
|
210
|
+
export async function mixpanel(source) {
|
|
211
|
+
// credentials from the environment, receipt fields from the IR
|
|
212
|
+
return valueFetchedFrom(source.project, source.bookmark)
|
|
213
|
+
}
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
[`examples/fetchers/mixpanel.mjs`](./examples/fetchers/mixpanel.mjs) is a
|
|
217
|
+
working reference. The trust boundary is explicit: a fetcher module loads only
|
|
218
|
+
when the person running refresh names it on the command line — the IR itself
|
|
219
|
+
can never designate one, so an untrusted `metrics.json` still cannot execute
|
|
220
|
+
anything. Built-in `csv`/`json`/`command` adapters always win over a custom
|
|
221
|
+
one of the same name.
|
|
222
|
+
|
|
223
|
+
<br>
|
|
224
|
+
|
|
179
225
|
## For AI-authored reports
|
|
180
226
|
|
|
181
227
|
[`SKILL.md`](./SKILL.md) is an authoring contract for coding agents: gather
|
|
@@ -183,6 +229,8 @@ receipts first, build the IR from real sources only, reference — never type
|
|
|
183
229
|
figures, and fix leaks by re-fetching, not by weakening the text. The compile
|
|
184
230
|
step turns "please don't hallucinate numbers" from a request into a gate.
|
|
185
231
|
|
|
232
|
+
<br>
|
|
233
|
+
|
|
186
234
|
## What it deliberately does not do
|
|
187
235
|
|
|
188
236
|
- **Judge whether evidence supports a claim.** `{{claim:}}` proves evidence
|
|
@@ -194,6 +242,8 @@ step turns "please don't hallucinate numbers" from a request into a gate.
|
|
|
194
242
|
- **Use an LLM.** Every check is deterministic. The point is to be the fixed
|
|
195
243
|
ground an LLM-assisted workflow can push against.
|
|
196
244
|
|
|
245
|
+
<br>
|
|
246
|
+
|
|
197
247
|
## Related work
|
|
198
248
|
|
|
199
249
|
- [Proof-Carrying Numbers (arXiv:2509.06902)](https://arxiv.org/abs/2509.06902)
|
|
@@ -204,6 +254,8 @@ step turns "please don't hallucinate numbers" from a request into a gate.
|
|
|
204
254
|
proves the "reports as source code" mechanism — but doesn't enforce that
|
|
205
255
|
prose figures stay grounded.
|
|
206
256
|
|
|
257
|
+
<br>
|
|
258
|
+
|
|
207
259
|
## License
|
|
208
260
|
|
|
209
|
-
MIT
|
|
261
|
+
`watertight` is released under an MIT license. See [License](LICENSE) for more information.
|
package/SKILL.md
CHANGED
|
@@ -77,8 +77,10 @@ Never type a figure into prose. Reference it:
|
|
|
77
77
|
watertight report.md metrics.json # writes report.html
|
|
78
78
|
watertight . --format md # grounded markdown (Notion / PR / Slack)
|
|
79
79
|
watertight . --check # CI mode: verify, write nothing
|
|
80
|
+
watertight . --check --max-age 30 # also fail receipts older than 30 days
|
|
80
81
|
```
|
|
81
82
|
|
|
83
|
+
Every leak names its line (`report.md:31`), so fix them where they live.
|
|
82
84
|
Fix leaks by *going and getting the receipt* — running the query, opening
|
|
83
85
|
the export — never by deleting the number, weakening the claim, or wrapping
|
|
84
86
|
a measurement in `{{raw:}}` to silence the checker. A `derived-mismatch` is
|
package/dist/cli.js
CHANGED
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
import { access, readFile, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { join, resolve } from 'node:path';
|
|
4
4
|
import { compile } from './compile.js';
|
|
5
|
+
import { basename } from 'node:path';
|
|
5
6
|
import { refresh } from './refresh.js';
|
|
7
|
+
import { init } from './init.js';
|
|
6
8
|
import { pathToFileURL } from 'node:url';
|
|
7
9
|
const USAGE = `watertight — reports that hold water. Every number carries its receipt;
|
|
8
10
|
an ungrounded claim is a leak, and a report with leaks does not build.
|
|
@@ -11,8 +13,11 @@ Usage
|
|
|
11
13
|
watertight <dir> compile <dir>/report.md + <dir>/metrics.json → <dir>/report.html
|
|
12
14
|
watertight <report> <ir> explicit file paths
|
|
13
15
|
watertight refresh <dir> re-fetch metric values from their sources, update metrics.json
|
|
16
|
+
watertight init [dir] scaffold a report.md + metrics.json pair that already holds water
|
|
14
17
|
|
|
15
18
|
Options
|
|
19
|
+
--max-age <days> compile only: fail any metric whose fetched_at is older —
|
|
20
|
+
numbers age, and a stale receipt is quietly becoming a leak
|
|
16
21
|
--format <html|md> output format (default: html). md is grounded markdown with a
|
|
17
22
|
receipts appendix — pastes into Notion, PR bodies or Slack intact
|
|
18
23
|
--out <file> where to write the output (default: report.html / report.grounded.md)
|
|
@@ -39,6 +44,7 @@ function parseArgs(argv) {
|
|
|
39
44
|
let allowCommands = false;
|
|
40
45
|
let fetchersPath;
|
|
41
46
|
let format = 'html';
|
|
47
|
+
let maxAgeDays;
|
|
42
48
|
let help = false;
|
|
43
49
|
let version = false;
|
|
44
50
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -55,19 +61,31 @@ function parseArgs(argv) {
|
|
|
55
61
|
fetchersPath = args[++i];
|
|
56
62
|
else if (arg === '--format')
|
|
57
63
|
format = args[++i] === 'md' ? 'md' : 'html';
|
|
64
|
+
else if (arg === '--max-age') {
|
|
65
|
+
maxAgeDays = Number(args[++i]);
|
|
66
|
+
if (!Number.isFinite(maxAgeDays) || maxAgeDays < 0) {
|
|
67
|
+
console.error(`error: --max-age needs a number of days, got "${args[i]}"`);
|
|
68
|
+
process.exit(2);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
58
71
|
else if (arg === '--json')
|
|
59
72
|
json = true;
|
|
60
73
|
else if (arg === '-h' || arg === '--help')
|
|
61
74
|
help = true;
|
|
62
75
|
else if (arg === '-v' || arg === '--version')
|
|
63
76
|
version = true;
|
|
64
|
-
else if (
|
|
77
|
+
else if (arg.startsWith('-')) {
|
|
78
|
+
// a strictness tool must not silently ignore a typo'd flag — --chekc writing a file is a betrayal
|
|
79
|
+
console.error(`error: unknown flag "${arg}" — see --help`);
|
|
80
|
+
process.exit(2);
|
|
81
|
+
}
|
|
82
|
+
else
|
|
65
83
|
positional.push(arg);
|
|
66
84
|
}
|
|
67
|
-
const command = positional[0] === 'refresh' ? 'refresh' : 'compile';
|
|
68
|
-
if (command
|
|
85
|
+
const command = positional[0] === 'refresh' ? 'refresh' : positional[0] === 'init' ? 'init' : 'compile';
|
|
86
|
+
if (command !== 'compile')
|
|
69
87
|
positional.shift();
|
|
70
|
-
return { command, positional, out, check, json, help, version, dryRun, allowCommands, format, fetchersPath };
|
|
88
|
+
return { command, positional, out, check, json, help, version, dryRun, allowCommands, format, fetchersPath, maxAgeDays };
|
|
71
89
|
}
|
|
72
90
|
async function exists(path) {
|
|
73
91
|
try {
|
|
@@ -85,6 +103,20 @@ async function main() {
|
|
|
85
103
|
return console.log(pkg.version);
|
|
86
104
|
if (opts.help)
|
|
87
105
|
return console.log(USAGE);
|
|
106
|
+
if (opts.command === 'init') {
|
|
107
|
+
const dir = resolve(opts.positional[0] ?? '.');
|
|
108
|
+
try {
|
|
109
|
+
const { written } = await init(dir);
|
|
110
|
+
for (const f of written)
|
|
111
|
+
console.log(` + ${f}`);
|
|
112
|
+
console.log('\nCompile it: watertight ' + (opts.positional[0] ?? '.'));
|
|
113
|
+
process.exit(0);
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
console.error(`error: ${err instanceof Error ? err.message : String(err)}`);
|
|
117
|
+
process.exit(2);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
88
120
|
// a mistyped path must fail loudly, never read as an empty report
|
|
89
121
|
let reportPath;
|
|
90
122
|
let irPath;
|
|
@@ -132,7 +164,7 @@ async function main() {
|
|
|
132
164
|
: `\n${r.changes.length} change(s)${opts.dryRun ? ' (dry run — nothing written)' : r.wrote ? ` — updated ${irPath}` : ''}`);
|
|
133
165
|
process.exit(r.errors.length > 0 ? 1 : 0);
|
|
134
166
|
}
|
|
135
|
-
const result = await compile(reportPath, irPath, opts.format);
|
|
167
|
+
const result = await compile(reportPath, irPath, { format: opts.format, maxAgeDays: opts.maxAgeDays });
|
|
136
168
|
const defaultName = opts.format === 'md' ? 'report.grounded.md' : 'report.html';
|
|
137
169
|
const outPath = resolve(opts.out ?? join(reportPath, '..', defaultName));
|
|
138
170
|
if (opts.json) {
|
|
@@ -143,7 +175,8 @@ async function main() {
|
|
|
143
175
|
if (result.leaks.length > 0) {
|
|
144
176
|
console.log(`\n${result.leaks.length} leak(s) — the report does not hold water:\n`);
|
|
145
177
|
for (const leak of result.leaks) {
|
|
146
|
-
|
|
178
|
+
const where = leak.line !== undefined ? `${basename(reportPath)}:${leak.line} — ` : '';
|
|
179
|
+
console.log(` ✗ [${leak.rule}] ${where}${leak.message}`);
|
|
147
180
|
if (leak.detail)
|
|
148
181
|
console.log(` ${leak.detail}`);
|
|
149
182
|
}
|
package/dist/compile.js
CHANGED
|
@@ -2,16 +2,38 @@ import { readFile } from 'node:fs/promises';
|
|
|
2
2
|
import { parseIr } from './ir.js';
|
|
3
3
|
import { render } from './render.js';
|
|
4
4
|
import { renderMarkdown } from './renderMd.js';
|
|
5
|
-
import { scanNakedNumbers, scanRefs } from './scan.js';
|
|
6
|
-
export async function compile(reportPath, irPath,
|
|
5
|
+
import { scanMarkers, scanNakedNumbers, scanRefs } from './scan.js';
|
|
6
|
+
export async function compile(reportPath, irPath, options = 'html') {
|
|
7
|
+
const opts = typeof options === 'string' ? { format: options } : options;
|
|
8
|
+
const format = opts.format ?? 'html';
|
|
7
9
|
const report = await readFile(reportPath, 'utf8');
|
|
8
10
|
const { ir, leaks } = parseIr(JSON.parse(await readFile(irPath, 'utf8')));
|
|
11
|
+
if (ir && opts.maxAgeDays !== undefined) {
|
|
12
|
+
const now = opts.now ?? new Date();
|
|
13
|
+
for (const [key, m] of Object.entries(ir.metrics)) {
|
|
14
|
+
if (!m.fetched_at)
|
|
15
|
+
continue;
|
|
16
|
+
const fetched = new Date(m.fetched_at);
|
|
17
|
+
if (Number.isNaN(fetched.getTime()))
|
|
18
|
+
continue; // "-" and friends: no date to age
|
|
19
|
+
const days = Math.floor((now.getTime() - fetched.getTime()) / 86_400_000);
|
|
20
|
+
if (days > opts.maxAgeDays) {
|
|
21
|
+
leaks.push({
|
|
22
|
+
severity: 'error',
|
|
23
|
+
rule: 'stale-metric',
|
|
24
|
+
message: `metric "${key}" was fetched ${m.fetched_at} — ${days} days ago, older than max-age ${opts.maxAgeDays}`,
|
|
25
|
+
detail: 'Run watertight refresh, or re-fetch by hand and update fetched_at.',
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
9
30
|
const grounded = {
|
|
10
31
|
metrics: [...report.matchAll(/\{\{m:/g)].length,
|
|
11
32
|
identifiers: [...report.matchAll(/\{\{id:/g)].length,
|
|
12
33
|
claims: [...report.matchAll(/\{\{claim:/g)].length,
|
|
13
34
|
};
|
|
14
35
|
leaks.push(...scanNakedNumbers(report));
|
|
36
|
+
leaks.push(...scanMarkers(report));
|
|
15
37
|
if (ir)
|
|
16
38
|
leaks.push(...scanRefs(report, new Set(Object.keys(ir.metrics)), new Set(Object.keys(ir.identifiers))));
|
|
17
39
|
if (leaks.length > 0 || !ir)
|
package/dist/init.js
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
/**
|
|
5
|
+
* Scaffold a report pair that holds water as written — the first compile a new
|
|
6
|
+
* user runs must succeed, so every figure in the template is already grounded.
|
|
7
|
+
*/
|
|
8
|
+
const REPORT = `# Checkout experiment — verification
|
|
9
|
+
|
|
10
|
+
Rolled out in \`{{id:app_version}}\` behind \`{{id:flag}}\`.
|
|
11
|
+
|
|
12
|
+
## Outcome
|
|
13
|
+
|
|
14
|
+
Conversion moved from {{m:conversion_before}} to {{m:conversion_after}} —
|
|
15
|
+
a change of {{m:conversion_change}}.
|
|
16
|
+
|
|
17
|
+
{{claim: state your conclusion here, pinned to the metrics that support it | evidence: conversion_change}}
|
|
18
|
+
|
|
19
|
+
## Next steps
|
|
20
|
+
|
|
21
|
+
Replace these metrics with your own. Every number needs a receipt:
|
|
22
|
+
run \`watertight .\` and fix what leaks.
|
|
23
|
+
`;
|
|
24
|
+
const METRICS = `${JSON.stringify({
|
|
25
|
+
metrics: {
|
|
26
|
+
conversion_before: {
|
|
27
|
+
value: 0.031,
|
|
28
|
+
unit: 'ratio',
|
|
29
|
+
definition: 'purchases / sessions entering checkout',
|
|
30
|
+
source: { type: 'sql', query: 'replace-with-your-query.sql' },
|
|
31
|
+
window: '2026-01-01 ~ 2026-01-14',
|
|
32
|
+
fetched_at: '2026-01-15',
|
|
33
|
+
},
|
|
34
|
+
conversion_after: {
|
|
35
|
+
value: 0.036,
|
|
36
|
+
unit: 'ratio',
|
|
37
|
+
definition: 'purchases / sessions entering checkout',
|
|
38
|
+
source: { type: 'sql', query: 'replace-with-your-query.sql' },
|
|
39
|
+
window: '2026-01-15 ~ 2026-01-28',
|
|
40
|
+
fetched_at: '2026-01-29',
|
|
41
|
+
},
|
|
42
|
+
conversion_change: {
|
|
43
|
+
value: 0.161,
|
|
44
|
+
unit: 'ratio-point',
|
|
45
|
+
definition: 'relative change in conversion',
|
|
46
|
+
derived: { op: 'pct_change', before: 'conversion_before', after: 'conversion_after' },
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
identifiers: { app_version: '1.0.0', flag: 'checkout_v2' },
|
|
50
|
+
}, null, 2)}\n`;
|
|
51
|
+
/** Refuses to touch anything that already exists — init never overwrites. */
|
|
52
|
+
export async function init(dir) {
|
|
53
|
+
const files = [
|
|
54
|
+
[join(dir, 'report.md'), REPORT],
|
|
55
|
+
[join(dir, 'metrics.json'), METRICS],
|
|
56
|
+
];
|
|
57
|
+
for (const [path] of files) {
|
|
58
|
+
if (existsSync(path))
|
|
59
|
+
throw new Error(`refusing to overwrite ${path}`);
|
|
60
|
+
}
|
|
61
|
+
await mkdir(dir, { recursive: true });
|
|
62
|
+
for (const [path, content] of files)
|
|
63
|
+
await writeFile(path, content);
|
|
64
|
+
return { written: files.map(([p]) => p) };
|
|
65
|
+
}
|
package/dist/ir.js
CHANGED
|
@@ -2,9 +2,18 @@
|
|
|
2
2
|
const DEFINITION_REQUIRED = new Set(['ratio', 'ratio-point']);
|
|
3
3
|
function decimals(n) {
|
|
4
4
|
const s = String(n);
|
|
5
|
+
const e = s.indexOf('e');
|
|
6
|
+
if (e !== -1) {
|
|
7
|
+
// 1e-7 has 7 decimals, 1.5e-7 has 8 — exponential notation must not disarm the precision check
|
|
8
|
+
const exp = Number(s.slice(e + 1));
|
|
9
|
+
const mantissa = s.slice(0, e);
|
|
10
|
+
const dot = mantissa.indexOf('.');
|
|
11
|
+
return Math.max(0, (dot === -1 ? 0 : mantissa.length - dot - 1) - exp);
|
|
12
|
+
}
|
|
5
13
|
const dot = s.indexOf('.');
|
|
6
14
|
return dot === -1 ? 0 : s.length - dot - 1;
|
|
7
15
|
}
|
|
16
|
+
const isFiniteNumber = (v) => typeof v === 'number' && Number.isFinite(v);
|
|
8
17
|
/**
|
|
9
18
|
* A stored derived value must be correctly rounded to its own precision: writing 0.15
|
|
10
19
|
* for a computed 0.155 is a mismatch, writing 0.155 for 0.15517 is fine. This is what
|
|
@@ -31,8 +40,10 @@ export function parseIr(raw) {
|
|
|
31
40
|
return { leaks };
|
|
32
41
|
}
|
|
33
42
|
for (const [key, m] of Object.entries(metrics)) {
|
|
34
|
-
|
|
35
|
-
|
|
43
|
+
const valueOk = isFiniteNumber(m.value) ||
|
|
44
|
+
(Array.isArray(m.value) && m.value.length === 2 && m.value.every(isFiniteNumber));
|
|
45
|
+
if (!valueOk) {
|
|
46
|
+
leaks.push({ severity: 'error', rule: 'missing-field', message: `metric "${key}" has no usable numeric value` });
|
|
36
47
|
continue;
|
|
37
48
|
}
|
|
38
49
|
if (typeof m.unit !== 'string' || m.unit.length === 0) {
|
|
@@ -63,7 +74,7 @@ export function parseIr(raw) {
|
|
|
63
74
|
}
|
|
64
75
|
computed += part.value;
|
|
65
76
|
}
|
|
66
|
-
if (!broken &&
|
|
77
|
+
if (!broken && !roundsTo(m.value, computed)) {
|
|
67
78
|
leaks.push({
|
|
68
79
|
severity: 'error',
|
|
69
80
|
rule: 'derived-mismatch',
|
|
@@ -72,7 +83,23 @@ export function parseIr(raw) {
|
|
|
72
83
|
}
|
|
73
84
|
}
|
|
74
85
|
else if (m.derived.op === 'pct_change') {
|
|
75
|
-
const
|
|
86
|
+
const endpoint = (v) => {
|
|
87
|
+
if (typeof v === 'number')
|
|
88
|
+
return v;
|
|
89
|
+
const ref = metrics[v];
|
|
90
|
+
return ref && typeof ref.value === 'number' ? ref.value : undefined;
|
|
91
|
+
};
|
|
92
|
+
const before = endpoint(m.derived.before);
|
|
93
|
+
const after = endpoint(m.derived.after);
|
|
94
|
+
if (before === undefined || after === undefined || before === 0) {
|
|
95
|
+
leaks.push({
|
|
96
|
+
severity: 'error',
|
|
97
|
+
rule: 'bad-derived',
|
|
98
|
+
message: `metric "${key}": pct_change endpoints must be numbers or scalar metric keys (got ${m.derived.before} → ${m.derived.after})`,
|
|
99
|
+
});
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
const computed = (after - before) / before;
|
|
76
103
|
if (!roundsTo(m.value, computed)) {
|
|
77
104
|
leaks.push({
|
|
78
105
|
severity: 'error',
|
|
@@ -82,6 +109,15 @@ export function parseIr(raw) {
|
|
|
82
109
|
});
|
|
83
110
|
}
|
|
84
111
|
}
|
|
112
|
+
else {
|
|
113
|
+
// an op the verifier cannot recompute must never pass as verified
|
|
114
|
+
leaks.push({
|
|
115
|
+
severity: 'error',
|
|
116
|
+
rule: 'bad-derived',
|
|
117
|
+
message: `metric "${key}" has unknown derived op "${m.derived.op}"`,
|
|
118
|
+
detail: 'Supported ops: sum, pct_change. A derivation the compiler cannot recompute cannot hold water.',
|
|
119
|
+
});
|
|
120
|
+
}
|
|
85
121
|
}
|
|
86
122
|
else {
|
|
87
123
|
// a measured value must say where it came from and when
|
package/dist/refresh.js
CHANGED
|
@@ -74,7 +74,8 @@ export async function refresh(irPath, options) {
|
|
|
74
74
|
const raw = JSON.parse(await readFile(irPath, 'utf8'));
|
|
75
75
|
const baseDir = dirname(resolve(irPath));
|
|
76
76
|
const result = { changes: [], skipped: [], errors: [], wrote: false };
|
|
77
|
-
|
|
77
|
+
// date-only, matching the style people write by hand — receipts should look uniform
|
|
78
|
+
const now = new Date().toISOString().slice(0, 10);
|
|
78
79
|
for (const [key, m] of Object.entries(raw.metrics)) {
|
|
79
80
|
if (m.derived || !m.source || Array.isArray(m.value))
|
|
80
81
|
continue;
|
|
@@ -107,31 +108,65 @@ export async function refresh(irPath, options) {
|
|
|
107
108
|
result.errors.push({ key, message: err instanceof Error ? err.message : String(err) });
|
|
108
109
|
}
|
|
109
110
|
}
|
|
110
|
-
// inputs may have moved, so derived values are recomputed rather than left to go stale
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
}, 0);
|
|
120
|
-
}
|
|
121
|
-
else {
|
|
122
|
-
const before = raw.metrics[m.derived.before]?.value;
|
|
123
|
-
const after = raw.metrics[m.derived.after]?.value;
|
|
124
|
-
if (typeof before !== 'number' || typeof after !== 'number' || before === 0)
|
|
111
|
+
// inputs may have moved, so derived values are recomputed rather than left to go stale.
|
|
112
|
+
// Iterated to a fixpoint so a derived that reads another derived settles too; a derivation
|
|
113
|
+
// that cannot be recomputed is a named error — silence is impossible here as everywhere.
|
|
114
|
+
const derivedErrors = new Set();
|
|
115
|
+
const firstBefore = new Map();
|
|
116
|
+
for (let pass = 0, moved = true; moved && pass < 10; pass++) {
|
|
117
|
+
moved = false;
|
|
118
|
+
for (const [key, m] of Object.entries(raw.metrics)) {
|
|
119
|
+
if (!m.derived || Array.isArray(m.value))
|
|
125
120
|
continue;
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
121
|
+
const fail = (message) => {
|
|
122
|
+
if (!derivedErrors.has(key)) {
|
|
123
|
+
derivedErrors.add(key);
|
|
124
|
+
result.errors.push({ key, message });
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
let computed;
|
|
128
|
+
if (m.derived.op === 'sum') {
|
|
129
|
+
computed = m.derived.of.reduce((acc, ref) => {
|
|
130
|
+
const part = raw.metrics[ref];
|
|
131
|
+
return acc + (part && typeof part.value === 'number' ? part.value : NaN);
|
|
132
|
+
}, 0);
|
|
133
|
+
if (!Number.isFinite(computed)) {
|
|
134
|
+
fail('sum references unknown or non-scalar metrics — not recomputed');
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
else if (m.derived.op === 'pct_change') {
|
|
139
|
+
const endpoint = (v) => {
|
|
140
|
+
if (typeof v === 'number')
|
|
141
|
+
return v;
|
|
142
|
+
const ref = raw.metrics[v];
|
|
143
|
+
return ref && typeof ref.value === 'number' ? ref.value : undefined;
|
|
144
|
+
};
|
|
145
|
+
const before = endpoint(m.derived.before);
|
|
146
|
+
const after = endpoint(m.derived.after);
|
|
147
|
+
if (before === undefined || after === undefined || before === 0) {
|
|
148
|
+
fail('pct_change endpoints are unresolvable or zero — not recomputed');
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
// keep the author's stated precision — refresh must not turn 0.155 into 0.1551724
|
|
152
|
+
const decimals = (String(m.value).split('.')[1] ?? '').length;
|
|
153
|
+
computed = Number(((after - before) / before).toFixed(decimals));
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
fail(`unknown derived op "${m.derived.op}" — not recomputed`);
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
if (computed !== m.value) {
|
|
160
|
+
if (!firstBefore.has(key))
|
|
161
|
+
firstBefore.set(key, m.value);
|
|
162
|
+
m.value = computed;
|
|
163
|
+
moved = true;
|
|
164
|
+
}
|
|
133
165
|
}
|
|
134
166
|
}
|
|
167
|
+
for (const [key, before] of firstBefore) {
|
|
168
|
+
result.changes.push({ key, before, after: raw.metrics[key].value });
|
|
169
|
+
}
|
|
135
170
|
if (!options.dryRun && (result.changes.length > 0 || result.errors.length === 0)) {
|
|
136
171
|
await writeFile(irPath, `${JSON.stringify(raw, null, 2)}\n`);
|
|
137
172
|
result.wrote = true;
|
package/dist/render.js
CHANGED
|
@@ -1,24 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Code shows syntax, it does not state facts — a fenced example of {{m:…}} must render
|
|
3
|
+
* verbatim, not substitute. Stash code regions before substitution, restore after.
|
|
4
|
+
*/
|
|
5
|
+
export function protectCode(text) {
|
|
6
|
+
const stash = [];
|
|
7
|
+
const protectedText = text.replace(/```[\s\S]*?```|`[^`\n]*`/g, (m) => {
|
|
8
|
+
stash.push(m);
|
|
9
|
+
return `\u0000${stash.length - 1}\u0000`;
|
|
10
|
+
});
|
|
11
|
+
return { text: protectedText, restore: (s) => s.replace(/\u0000(\d+)\u0000/g, (_, i) => stash[Number(i)]) };
|
|
12
|
+
}
|
|
1
13
|
function escapeHtml(s) {
|
|
2
14
|
return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
3
15
|
}
|
|
4
16
|
export function formatValue(m) {
|
|
5
17
|
if (Array.isArray(m.value)) {
|
|
6
18
|
const [lo, hi] = m.value;
|
|
7
|
-
|
|
19
|
+
// a range keeps the same unit conversion as a scalar — [0.3, 0.5] ratio is 30~50%, not 0.3~0.5%
|
|
20
|
+
if (m.unit === 'ratio' || m.unit === 'ratio-point') {
|
|
21
|
+
const pct = (v) => (v * 100).toFixed(1);
|
|
22
|
+
return `${pct(lo)}~${pct(hi)}${m.unit === 'ratio' ? '%' : '%p'}`;
|
|
23
|
+
}
|
|
24
|
+
return `${lo.toLocaleString()}~${hi.toLocaleString()} ${m.unit}`.trim();
|
|
8
25
|
}
|
|
9
26
|
if (m.unit === 'ratio')
|
|
10
|
-
return `${(m.value * 100).toFixed(m.value < 0.01 ? 2 : 1)}%`;
|
|
27
|
+
return `${(m.value * 100).toFixed(Math.abs(m.value) < 0.01 ? 2 : 1)}%`;
|
|
11
28
|
if (m.unit === 'ratio-point')
|
|
12
29
|
return `${m.value >= 0 ? '+' : ''}${(m.value * 100).toFixed(1)}%p`;
|
|
13
30
|
return `${m.value.toLocaleString()} ${m.unit}`.trim();
|
|
14
31
|
}
|
|
15
|
-
export function receipt(m) {
|
|
32
|
+
export function receipt(m, includeDefinition = true) {
|
|
16
33
|
const source = m.derived
|
|
17
34
|
? m.derived.op === 'sum'
|
|
18
35
|
? `= ${m.derived.of.join(' + ')} (recomputed)`
|
|
19
36
|
: `= ${m.derived.before} → ${m.derived.after} (recomputed)`
|
|
20
37
|
: [m.source?.type, ...Object.entries(m.source ?? {}).filter(([k]) => k !== 'type').map(([, v]) => String(v))].filter(Boolean).join(' · ');
|
|
21
|
-
return [source, m.window, m.fetched_at && `fetched ${m.fetched_at}`, m.definition].filter(Boolean).join(' · ');
|
|
38
|
+
return [source, m.window, m.fetched_at && `fetched ${m.fetched_at}`, includeDefinition && m.definition].filter(Boolean).join(' · ');
|
|
22
39
|
}
|
|
23
40
|
/** Minimal markdown: headings, paragraphs, bold, unordered lists. The narrative layer is deliberately thin. */
|
|
24
41
|
function markdown(src) {
|
|
@@ -37,7 +54,8 @@ function markdown(src) {
|
|
|
37
54
|
.replace(/\*\*([^*]+)\*\*/g, '<b>$1</b>');
|
|
38
55
|
}
|
|
39
56
|
export function render(report, ir) {
|
|
40
|
-
const
|
|
57
|
+
const { text: protectedReport, restore } = protectCode(escapeHtml(report));
|
|
58
|
+
const grounded = restore(protectedReport
|
|
41
59
|
.replace(/\{\{m:([\w-]+)\}\}/g, (_, key) => {
|
|
42
60
|
const m = ir.metrics[key];
|
|
43
61
|
return `<b class="w" title="${escapeHtml(receipt(m))}">${formatValue(m)}<sup>†</sup></b>`;
|
|
@@ -50,7 +68,7 @@ export function render(report, ir) {
|
|
|
50
68
|
.join(' | ');
|
|
51
69
|
return `<span class="c" title="${escapeHtml(receipts)}">${text.trim()}<sup>‡</sup></span>`;
|
|
52
70
|
})
|
|
53
|
-
.replace(/\{\{raw:([^}]*)\}\}/g, (_, text) => escapeHtml(text));
|
|
71
|
+
.replace(/\{\{raw:([^}]*)\}\}/g, (_, text) => escapeHtml(text)));
|
|
54
72
|
return `<!doctype html><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
55
73
|
<style>
|
|
56
74
|
body{font:15px/1.75 -apple-system,system-ui,sans-serif;max-width:720px;margin:40px auto;padding:0 16px;color:#1a1a1a}
|
package/dist/renderMd.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import { formatValue, receipt } from './render.js';
|
|
1
|
+
import { formatValue, protectCode, receipt } from './render.js';
|
|
2
|
+
const SUPERSCRIPT = '⁰¹²³⁴⁵⁶⁷⁸⁹';
|
|
3
|
+
const sup = (n) => `⁽${String(n).split('').map((d) => SUPERSCRIPT[Number(d)]).join('')}⁾`;
|
|
2
4
|
/**
|
|
3
5
|
* Grounded markdown: the portable render target. Every figure is bold with a superscript
|
|
4
6
|
* that points into a receipts appendix, so the output pastes into Notion, a PR body, or
|
|
@@ -7,21 +9,29 @@ import { formatValue, receipt } from './render.js';
|
|
|
7
9
|
*/
|
|
8
10
|
export function renderMarkdown(report, ir) {
|
|
9
11
|
const used = [];
|
|
10
|
-
const
|
|
12
|
+
const { text: protectedReport, restore } = protectCode(report);
|
|
13
|
+
const body = restore(protectedReport
|
|
11
14
|
.replace(/\{\{m:([\w-]+)\}\}/g, (_, key) => {
|
|
12
15
|
if (!used.includes(key))
|
|
13
16
|
used.push(key);
|
|
14
|
-
return `**${formatValue(ir.metrics[key])}**
|
|
17
|
+
return `**${formatValue(ir.metrics[key])}** ${sup(used.indexOf(key) + 1)}`;
|
|
15
18
|
})
|
|
16
19
|
.replace(/\{\{id:([\w-]+)\}\}/g, (_, key) => `\`${ir.identifiers[key]}\``)
|
|
17
20
|
.replace(/\{\{claim:([^|}]*)\|\s*evidence:([^}]*)\}\}/g, (_, text, evidence) => `**${text.trim()}** *(evidence: ${evidence.trim()})*`)
|
|
18
|
-
.replace(/\{\{raw:([^}]*)\}\}/g, (_, text) => text);
|
|
21
|
+
.replace(/\{\{raw:([^}]*)\}\}/g, (_, text) => text));
|
|
22
|
+
// evidence-only metrics get receipts too — a claim's reader must be able to check its keys
|
|
23
|
+
for (const [, , evidence] of protectedReport.matchAll(/\{\{claim:([^|}]*)\|\s*evidence:([^}]*)\}\}/g)) {
|
|
24
|
+
for (const key of evidence.split(',').map((k) => k.trim()).filter(Boolean)) {
|
|
25
|
+
if (ir.metrics[key] && !used.includes(key))
|
|
26
|
+
used.push(key);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
19
29
|
const appendix = used
|
|
20
30
|
.map((key, i) => {
|
|
21
31
|
const m = ir.metrics[key];
|
|
22
32
|
const definition = m.definition ? ` — ${m.definition}` : '';
|
|
23
|
-
return `${i + 1}. **${key}** = ${formatValue(m)}${definition}\n ${receipt(m)}`;
|
|
33
|
+
return `${i + 1}. **${key}** = ${formatValue(m)}${definition}\n ${receipt(m, false)}`;
|
|
24
34
|
})
|
|
25
35
|
.join('\n');
|
|
26
|
-
return `${body}\n\n---\n\n### Receipts (${used.length} metrics)\n\n${appendix}\n`;
|
|
36
|
+
return `${body.trimEnd()}\n\n---\n\n### Receipts (${used.length} metrics)\n\n${appendix}\n`;
|
|
27
37
|
}
|
package/dist/scan.js
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
/** Same-length whitespace, newlines kept — stripping must not move anything */
|
|
2
|
+
const blank = (s) => s.replace(/[^\n]/g, ' ');
|
|
3
|
+
const lineAt = (text, index) => text.slice(0, index).split('\n').length;
|
|
4
|
+
/** Code shows syntax, it does not state facts — refs and numbers inside it are exempt */
|
|
5
|
+
const blankCode = (report) => report.replace(/```[\s\S]*?```/g, blank).replace(/`[^`\n]*`/g, blank);
|
|
1
6
|
/**
|
|
2
7
|
* Find numbers in the narrative that are not bound to the IR. These are the leaks the
|
|
3
8
|
* tool exists to catch: a typed, transcribed, or hallucinated figure reads exactly like
|
|
@@ -5,24 +10,30 @@
|
|
|
5
10
|
*/
|
|
6
11
|
export function scanNakedNumbers(report) {
|
|
7
12
|
const stripped = report
|
|
8
|
-
.replace(/\{\{(m|id):[\w-]+\}\}/g,
|
|
9
|
-
// a claim's prose stays scanned — only its syntax is
|
|
10
|
-
// smuggled into claim text is still a leak
|
|
11
|
-
.replace(/\{\{claim:([^|}]*)\|[^}]*\}\}/g, (
|
|
12
|
-
.
|
|
13
|
-
.
|
|
14
|
-
|
|
15
|
-
.replace(/\
|
|
16
|
-
.replace(
|
|
17
|
-
.replace(
|
|
13
|
+
.replace(/\{\{(m|id):[\w-]+\}\}/g, blank)
|
|
14
|
+
// a claim's prose stays scanned — only its syntax is blanked, so a number
|
|
15
|
+
// smuggled into claim text is still a leak, at its true position
|
|
16
|
+
.replace(/\{\{claim:([^|}]*)\|[^}]*\}\}/g, (whole, text) => {
|
|
17
|
+
const start = whole.indexOf(text);
|
|
18
|
+
return blank(whole.slice(0, start)) + text + blank(whole.slice(start + text.length));
|
|
19
|
+
})
|
|
20
|
+
.replace(/\{\{raw:[^}]*\}\}/g, blank) // explicit, greppable escape hatch
|
|
21
|
+
.replace(/```[\s\S]*?```/g, blank) // fenced code
|
|
22
|
+
.replace(/`[^`\n]*`/g, blank) // inline code
|
|
23
|
+
.replace(/\]\([^)\s]*\)/g, blank) // markdown link targets — URLs locate, they do not measure
|
|
24
|
+
.replace(/https?:\/\/\S+/g, blank) // bare URLs, same reason
|
|
25
|
+
.replace(/\d{4}-\d{2}-\d{2}/g, blank) // ISO dates locate, they do not measure
|
|
26
|
+
.replace(/^#{1,6}(?= )/gm, blank) // heading markers only — heading TEXT is scanned, people summarise numbers there
|
|
27
|
+
.replace(/^\s*\d+\.\s/gm, blank); // ordered-list markers
|
|
18
28
|
const leaks = [];
|
|
19
29
|
for (const m of stripped.matchAll(/\d[\d,.]*\s*(%p?|[가-힣]{1,2})?/g)) {
|
|
20
|
-
const token = m[0].trim();
|
|
30
|
+
const token = m[0].trim().replace(/[.,]+$/, '');
|
|
21
31
|
if (!token)
|
|
22
32
|
continue;
|
|
23
33
|
leaks.push({
|
|
24
34
|
severity: 'error',
|
|
25
35
|
rule: 'naked-number',
|
|
36
|
+
line: lineAt(stripped, m.index),
|
|
26
37
|
message: `"${token}" appears in the narrative without a receipt`,
|
|
27
38
|
detail: 'Bind it to the IR as {{m:…}} or {{id:…}}, or mark deliberate prose as {{raw:…}}.',
|
|
28
39
|
});
|
|
@@ -30,42 +41,69 @@ export function scanNakedNumbers(report) {
|
|
|
30
41
|
return leaks;
|
|
31
42
|
}
|
|
32
43
|
/** Every reference in the narrative must resolve; a dangling one is authoring drift. */
|
|
33
|
-
export function scanRefs(
|
|
44
|
+
export function scanRefs(rawReport, metricKeys, idKeys) {
|
|
34
45
|
const leaks = [];
|
|
35
|
-
|
|
46
|
+
const report = blankCode(rawReport);
|
|
47
|
+
for (const m of report.matchAll(/\{\{claim:([^|}]*)\|\s*evidence:([^}]*)\}\}/g)) {
|
|
48
|
+
const [, text, evidence] = m;
|
|
49
|
+
const line = lineAt(report, m.index);
|
|
36
50
|
const keys = evidence.split(',').map((k) => k.trim()).filter(Boolean);
|
|
37
51
|
if (keys.length === 0) {
|
|
38
52
|
leaks.push({
|
|
39
53
|
severity: 'error',
|
|
40
54
|
rule: 'claim-without-evidence',
|
|
55
|
+
line,
|
|
41
56
|
message: `claim "${text.trim()}" names no evidence`,
|
|
42
57
|
detail: 'A conclusion must point at the metrics it rests on. Whether they support it stays a human judgement — but they must be attached.',
|
|
43
58
|
});
|
|
44
59
|
}
|
|
45
60
|
for (const key of keys) {
|
|
46
61
|
if (!metricKeys.has(key)) {
|
|
47
|
-
leaks.push({ severity: 'error', rule: 'unknown-ref', message: `claim "${text.trim()}" cites unknown metric "${key}"` });
|
|
62
|
+
leaks.push({ severity: 'error', rule: 'unknown-ref', line, message: `claim "${text.trim()}" cites unknown metric "${key}"` });
|
|
48
63
|
}
|
|
49
64
|
}
|
|
50
65
|
}
|
|
51
66
|
// a claim marker missing its evidence clause entirely is malformed, not exempt
|
|
52
|
-
for (const
|
|
67
|
+
for (const m of report.matchAll(/\{\{claim:([^|}]*)\}\}/g)) {
|
|
53
68
|
leaks.push({
|
|
54
69
|
severity: 'error',
|
|
55
70
|
rule: 'claim-without-evidence',
|
|
56
|
-
|
|
71
|
+
line: lineAt(report, m.index),
|
|
72
|
+
message: `claim "${m[1].trim()}" names no evidence`,
|
|
57
73
|
detail: 'Write it as {{claim: … | evidence: metric_key}}.',
|
|
58
74
|
});
|
|
59
75
|
}
|
|
60
|
-
for (const
|
|
61
|
-
if (!metricKeys.has(
|
|
62
|
-
leaks.push({ severity: 'error', rule: 'unknown-ref', message: `{{m:${
|
|
76
|
+
for (const m of report.matchAll(/\{\{m:([\w-]+)\}\}/g)) {
|
|
77
|
+
if (!metricKeys.has(m[1])) {
|
|
78
|
+
leaks.push({ severity: 'error', rule: 'unknown-ref', line: lineAt(report, m.index), message: `{{m:${m[1]}}} is not in the IR` });
|
|
63
79
|
}
|
|
64
80
|
}
|
|
65
|
-
for (const
|
|
66
|
-
if (!idKeys.has(
|
|
67
|
-
leaks.push({ severity: 'error', rule: 'unknown-ref', message: `{{id:${
|
|
81
|
+
for (const m of report.matchAll(/\{\{id:([\w-]+)\}\}/g)) {
|
|
82
|
+
if (!idKeys.has(m[1])) {
|
|
83
|
+
leaks.push({ severity: 'error', rule: 'unknown-ref', line: lineAt(report, m.index), message: `{{id:${m[1]}}} is not in the IR` });
|
|
68
84
|
}
|
|
69
85
|
}
|
|
70
86
|
return leaks;
|
|
71
87
|
}
|
|
88
|
+
/**
|
|
89
|
+
* Anything still shaped like a marker after every recognised form is removed was a typo —
|
|
90
|
+
* and a typo'd marker must be a leak, or it renders verbatim and its number sails through.
|
|
91
|
+
*/
|
|
92
|
+
export function scanMarkers(rawReport) {
|
|
93
|
+
const known = blankCode(rawReport)
|
|
94
|
+
.replace(/\{\{(m|id):[\w-]+\}\}/g, blank)
|
|
95
|
+
.replace(/\{\{raw:[^}]*\}\}/g, blank)
|
|
96
|
+
.replace(/\{\{claim:[^|}]*\|\s*evidence:[^}]*\}\}/g, blank)
|
|
97
|
+
.replace(/\{\{claim:[^|}]*\}\}/g, blank); // no-pipe form is already claim-without-evidence
|
|
98
|
+
const leaks = [];
|
|
99
|
+
for (const m of known.matchAll(/\{\{[^}]*\}\}?/g)) {
|
|
100
|
+
leaks.push({
|
|
101
|
+
severity: 'error',
|
|
102
|
+
rule: 'malformed-marker',
|
|
103
|
+
line: lineAt(known, m.index),
|
|
104
|
+
message: `"${m[0]}" is not a recognised marker`,
|
|
105
|
+
detail: 'Valid forms: {{m:key}}, {{id:key}}, {{raw:…}}, {{claim: text | evidence: keys}}.',
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
return leaks;
|
|
109
|
+
}
|