watertight 0.2.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -80
- package/SKILL.md +2 -0
- package/dist/cli.js +33 -5
- package/dist/compile.js +22 -1
- package/dist/init.js +64 -0
- package/dist/ir.js +17 -1
- package/dist/refresh.js +9 -3
- package/dist/render.js +2 -2
- package/dist/renderMd.js +5 -3
- package/dist/scan.js +31 -20
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,29 +1,43 @@
|
|
|
1
1
|
# watertight
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/watertight)
|
|
4
|
+
[](https://github.com/camosss/watertight/actions/workflows/ci.yml)
|
|
5
|
+

|
|
6
|
+

|
|
7
|
+
[](LICENSE)
|
|
5
8
|
|
|
9
|
+
Reports that hold water — every number carries its **receipt**, and ungrounded claims **fail the build**.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
npx watertight report.md metrics.json
|
|
6
13
|
```
|
|
7
|
-
$ watertight report.md metrics.json
|
|
8
14
|
|
|
9
|
-
|
|
15
|
+
```
|
|
16
|
+
watertight v0.2.0 · 20 grounded metrics · 4 claims · 5 identifiers
|
|
10
17
|
holds water → report.html
|
|
11
18
|
```
|
|
12
19
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
20
|
+
An ungrounded number is a **leak**. A report with zero leaks **holds water**:
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
2 leak(s) — the report does not hold water:
|
|
24
|
+
|
|
25
|
+
✗ [naked-number] report.md:31 — "15%" appears in the narrative without a receipt
|
|
26
|
+
✗ [derived-mismatch] metric "revenue_total" is 1440, but its parts sum to 1428
|
|
27
|
+
```
|
|
19
28
|
|
|
20
|
-
|
|
29
|
+
<br>
|
|
21
30
|
|
|
22
31
|
## Why this exists
|
|
23
32
|
|
|
33
|
+
A number typed into prose has no memory of where it came from. Two weeks later
|
|
34
|
+
nobody can say which query produced it, whether it was rounded, or whether it
|
|
35
|
+
was ever true — and when an LLM helps write the report, "never true" is a live
|
|
36
|
+
possibility.
|
|
37
|
+
|
|
24
38
|
This came out of writing a real ad-revenue verification report with an AI
|
|
25
|
-
assistant. Compiling the draft surfaced three distinct failure classes in
|
|
26
|
-
|
|
39
|
+
assistant. Compiling the draft surfaced three distinct failure classes in one
|
|
40
|
+
document:
|
|
27
41
|
|
|
28
42
|
- a human had written **+15%** for a delta that computed to **+15.5%** —
|
|
29
43
|
a rounding transcription error nobody had caught;
|
|
@@ -31,10 +45,14 @@ one document:
|
|
|
31
45
|
- the assistant confidently "remembered" a figure that appeared nowhere in
|
|
32
46
|
any export.
|
|
33
47
|
|
|
34
|
-
All three are the same disease — a number with no receipt — and all three
|
|
35
|
-
|
|
48
|
+
All three are the same disease — a number with no receipt — and all three are
|
|
49
|
+
mechanically detectable. So: detect them, every build. watertight treats a
|
|
50
|
+
report like source code: the narrative references metrics, the metrics carry
|
|
51
|
+
provenance, and a compiler refuses to build anything it cannot trace.
|
|
36
52
|
|
|
37
|
-
|
|
53
|
+
<br>
|
|
54
|
+
|
|
55
|
+
## Quick start
|
|
38
56
|
|
|
39
57
|
```bash
|
|
40
58
|
npm install -g watertight
|
|
@@ -42,25 +60,12 @@ npm install -g watertight
|
|
|
42
60
|
npx watertight report.md metrics.json
|
|
43
61
|
```
|
|
44
62
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
## How it works
|
|
48
|
-
|
|
49
|
-
You write two files.
|
|
50
|
-
|
|
51
|
-
**`metrics.json` — the IR.** Every value, with its receipt:
|
|
63
|
+
You write two files. **`metrics.json`** holds every value with its receipt —
|
|
64
|
+
a measured metric says where it was fetched, over what window, and when:
|
|
52
65
|
|
|
53
66
|
```json
|
|
54
67
|
{
|
|
55
68
|
"metrics": {
|
|
56
|
-
"conversion_before": {
|
|
57
|
-
"value": 0.031,
|
|
58
|
-
"unit": "ratio",
|
|
59
|
-
"definition": "purchases / sessions entering checkout",
|
|
60
|
-
"source": { "type": "sql", "query": "reports/checkout.sql" },
|
|
61
|
-
"window": "2026-08-15 ~ 2026-08-31",
|
|
62
|
-
"fetched_at": "2026-09-15"
|
|
63
|
-
},
|
|
64
69
|
"conversion_after": {
|
|
65
70
|
"value": 0.036,
|
|
66
71
|
"unit": "ratio",
|
|
@@ -68,34 +73,42 @@ You write two files.
|
|
|
68
73
|
"source": { "type": "sql", "query": "reports/checkout.sql" },
|
|
69
74
|
"window": "2026-09-01 ~ 2026-09-14",
|
|
70
75
|
"fetched_at": "2026-09-15"
|
|
71
|
-
},
|
|
72
|
-
"lift": {
|
|
73
|
-
"value": 0.161,
|
|
74
|
-
"unit": "ratio-point",
|
|
75
|
-
"definition": "relative change in conversion",
|
|
76
|
-
"derived": { "op": "pct_change", "before": "conversion_before", "after": "conversion_after" }
|
|
77
|
-
},
|
|
78
|
-
"revenue_total": {
|
|
79
|
-
"value": 1428,
|
|
80
|
-
"unit": "USD",
|
|
81
|
-
"derived": { "op": "sum", "of": ["revenue_ios", "revenue_android"] }
|
|
82
|
-
},
|
|
83
|
-
"revenue_target": {
|
|
84
|
-
"value": [1000, 5000],
|
|
85
|
-
"unit": "USD",
|
|
86
|
-
"source": { "type": "hypothesis", "doc": "PLAN-42" },
|
|
87
|
-
"window": "planning estimate",
|
|
88
|
-
"fetched_at": "-"
|
|
89
76
|
}
|
|
90
77
|
},
|
|
91
|
-
"identifiers": {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
78
|
+
"identifiers": { "app_version": "3.2.0", "flag": "checkout_v2" }
|
|
79
|
+
}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
A **derived** metric is recomputed on every compile — a total that doesn't add
|
|
83
|
+
up, or a delta that doesn't recompute, is a build failure:
|
|
84
|
+
|
|
85
|
+
```json
|
|
86
|
+
"lift": {
|
|
87
|
+
"value": 0.161,
|
|
88
|
+
"unit": "ratio-point",
|
|
89
|
+
"definition": "relative change in conversion",
|
|
90
|
+
"derived": { "op": "pct_change", "before": "conversion_before", "after": "conversion_after" }
|
|
91
|
+
},
|
|
92
|
+
"revenue_total": {
|
|
93
|
+
"value": 1428,
|
|
94
|
+
"unit": "USD",
|
|
95
|
+
"derived": { "op": "sum", "of": ["revenue_ios", "revenue_android"] }
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
A **range** states a hypothesis honestly — plans are receipts too:
|
|
100
|
+
|
|
101
|
+
```json
|
|
102
|
+
"revenue_target": {
|
|
103
|
+
"value": [1000, 5000],
|
|
104
|
+
"unit": "USD",
|
|
105
|
+
"source": { "type": "hypothesis", "doc": "PLAN-42" },
|
|
106
|
+
"window": "planning estimate",
|
|
107
|
+
"fetched_at": "-"
|
|
95
108
|
}
|
|
96
109
|
```
|
|
97
110
|
|
|
98
|
-
**`report.md
|
|
111
|
+
**`report.md`** is the narrative. No literal figures, only references:
|
|
99
112
|
|
|
100
113
|
```markdown
|
|
101
114
|
Rolled out in `{{id:app_version}}` behind `{{id:flag}}`.
|
|
@@ -113,20 +126,40 @@ Compile:
|
|
|
113
126
|
|
|
114
127
|
```bash
|
|
115
128
|
watertight report.md metrics.json # → report.html (self-contained, hover for receipts)
|
|
116
|
-
watertight . --format md # → grounded markdown
|
|
129
|
+
watertight . --format md # → grounded markdown (below)
|
|
117
130
|
watertight . --check # verify only, write nothing (CI)
|
|
131
|
+
watertight . --check --max-age 30 # also fail receipts older than 30 days
|
|
118
132
|
watertight . --json # machine-readable result
|
|
119
133
|
```
|
|
120
134
|
|
|
121
|
-
|
|
135
|
+
<br>
|
|
122
136
|
|
|
123
|
-
|
|
124
|
-
2 leak(s) — the report does not hold water:
|
|
137
|
+
## What the output looks like
|
|
125
138
|
|
|
126
|
-
|
|
127
|
-
|
|
139
|
+
`--format md` produces grounded markdown that pastes into Notion, a PR body,
|
|
140
|
+
or Slack with its provenance intact — every figure bold with a superscript
|
|
141
|
+
into a receipts appendix:
|
|
142
|
+
|
|
143
|
+
```markdown
|
|
144
|
+
Conversion moved from **3.1%** ⁽¹⁾ to **3.6%** ⁽²⁾, a lift of **+16.1%p** ⁽³⁾.
|
|
145
|
+
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
### Receipts (3 metrics)
|
|
149
|
+
|
|
150
|
+
1. **conversion_before** = 3.1% — purchases / sessions entering checkout
|
|
151
|
+
sql · reports/checkout.sql · 2026-08-15 ~ 2026-08-31 · fetched 2026-09-15
|
|
152
|
+
2. **conversion_after** = 3.6% — purchases / sessions entering checkout
|
|
153
|
+
sql · reports/checkout.sql · 2026-09-01 ~ 2026-09-14 · fetched 2026-09-15
|
|
154
|
+
3. **lift** = +16.1%p — relative change in conversion
|
|
155
|
+
= conversion_before → conversion_after (recomputed)
|
|
128
156
|
```
|
|
129
157
|
|
|
158
|
+
The default HTML render is a single self-contained file where hovering any
|
|
159
|
+
figure shows its receipt.
|
|
160
|
+
|
|
161
|
+
<br>
|
|
162
|
+
|
|
130
163
|
## What it checks
|
|
131
164
|
|
|
132
165
|
| check | catches |
|
|
@@ -139,6 +172,9 @@ Any leak fails the build (exit 1) and names the line:
|
|
|
139
172
|
| `claim-without-evidence` | a `{{claim:}}` with no `evidence:` keys, or keys that don't exist |
|
|
140
173
|
| `bad-derived` | derived ops referencing missing or non-numeric inputs |
|
|
141
174
|
| `empty-ir` | a report "grounded" in nothing |
|
|
175
|
+
| `stale-metric` | with `--max-age <days>`: a receipt whose `fetched_at` is older than the budget — numbers age |
|
|
176
|
+
|
|
177
|
+
<br>
|
|
142
178
|
|
|
143
179
|
## Re-verification
|
|
144
180
|
|
|
@@ -150,25 +186,6 @@ can reach, rewrites `value` and `fetched_at`, and recomputes derived values:
|
|
|
150
186
|
- `command` sources — run **only** with the explicit `--allow-commands` flag,
|
|
151
187
|
because refreshing an IR you didn't author must never execute its shell
|
|
152
188
|
commands
|
|
153
|
-
- any other source type can be covered by a **custom fetcher** — a JS module
|
|
154
|
-
you point at explicitly:
|
|
155
|
-
|
|
156
|
-
```bash
|
|
157
|
-
watertight refresh . --fetchers ./my-fetchers.mjs
|
|
158
|
-
```
|
|
159
|
-
|
|
160
|
-
```js
|
|
161
|
-
// my-fetchers.mjs — export one function per source type
|
|
162
|
-
export async function mixpanel(source) {
|
|
163
|
-
// credentials from the environment, receipt fields from the IR
|
|
164
|
-
return valueFetchedFrom(source.project, source.bookmark)
|
|
165
|
-
}
|
|
166
|
-
```
|
|
167
|
-
|
|
168
|
-
[`examples/fetchers/mixpanel.mjs`](./examples/fetchers/mixpanel.mjs) is a
|
|
169
|
-
working example. Loading a module runs its code, so only pass files you
|
|
170
|
-
wrote or trust — the IR itself can never name a fetcher. Built-in csv/json/
|
|
171
|
-
command adapters always win over a custom one of the same name.
|
|
172
189
|
- everything else (dashboards, vendor reports, hypotheses) is **named as
|
|
173
190
|
skipped** — never silently assumed fresh
|
|
174
191
|
|
|
@@ -176,6 +193,34 @@ Derived values follow their inputs: sums are recomputed exactly, percentage
|
|
|
176
193
|
changes at the precision the author stated. `--dry-run` previews changes
|
|
177
194
|
without writing.
|
|
178
195
|
|
|
196
|
+
<br>
|
|
197
|
+
|
|
198
|
+
## Custom fetchers
|
|
199
|
+
|
|
200
|
+
Any other source type can be refreshed through a JS module you point at
|
|
201
|
+
explicitly:
|
|
202
|
+
|
|
203
|
+
```bash
|
|
204
|
+
watertight refresh . --fetchers ./my-fetchers.mjs
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
```js
|
|
208
|
+
// my-fetchers.mjs — export one function per source type
|
|
209
|
+
export async function mixpanel(source) {
|
|
210
|
+
// credentials from the environment, receipt fields from the IR
|
|
211
|
+
return valueFetchedFrom(source.project, source.bookmark)
|
|
212
|
+
}
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
[`examples/fetchers/mixpanel.mjs`](./examples/fetchers/mixpanel.mjs) is a
|
|
216
|
+
working reference. The trust boundary is explicit: a fetcher module loads only
|
|
217
|
+
when the person running refresh names it on the command line — the IR itself
|
|
218
|
+
can never designate one, so an untrusted `metrics.json` still cannot execute
|
|
219
|
+
anything. Built-in `csv`/`json`/`command` adapters always win over a custom
|
|
220
|
+
one of the same name.
|
|
221
|
+
|
|
222
|
+
<br>
|
|
223
|
+
|
|
179
224
|
## For AI-authored reports
|
|
180
225
|
|
|
181
226
|
[`SKILL.md`](./SKILL.md) is an authoring contract for coding agents: gather
|
|
@@ -183,6 +228,8 @@ receipts first, build the IR from real sources only, reference — never type
|
|
|
183
228
|
figures, and fix leaks by re-fetching, not by weakening the text. The compile
|
|
184
229
|
step turns "please don't hallucinate numbers" from a request into a gate.
|
|
185
230
|
|
|
231
|
+
<br>
|
|
232
|
+
|
|
186
233
|
## What it deliberately does not do
|
|
187
234
|
|
|
188
235
|
- **Judge whether evidence supports a claim.** `{{claim:}}` proves evidence
|
|
@@ -194,6 +241,8 @@ step turns "please don't hallucinate numbers" from a request into a gate.
|
|
|
194
241
|
- **Use an LLM.** Every check is deterministic. The point is to be the fixed
|
|
195
242
|
ground an LLM-assisted workflow can push against.
|
|
196
243
|
|
|
244
|
+
<br>
|
|
245
|
+
|
|
197
246
|
## Related work
|
|
198
247
|
|
|
199
248
|
- [Proof-Carrying Numbers (arXiv:2509.06902)](https://arxiv.org/abs/2509.06902)
|
|
@@ -204,6 +253,8 @@ step turns "please don't hallucinate numbers" from a request into a gate.
|
|
|
204
253
|
proves the "reports as source code" mechanism — but doesn't enforce that
|
|
205
254
|
prose figures stay grounded.
|
|
206
255
|
|
|
256
|
+
<br>
|
|
257
|
+
|
|
207
258
|
## License
|
|
208
259
|
|
|
209
|
-
MIT
|
|
260
|
+
`watertight` is released under an MIT license. See [License](LICENSE) for more information.
|
package/SKILL.md
CHANGED
|
@@ -77,8 +77,10 @@ Never type a figure into prose. Reference it:
|
|
|
77
77
|
watertight report.md metrics.json # writes report.html
|
|
78
78
|
watertight . --format md # grounded markdown (Notion / PR / Slack)
|
|
79
79
|
watertight . --check # CI mode: verify, write nothing
|
|
80
|
+
watertight . --check --max-age 30 # also fail receipts older than 30 days
|
|
80
81
|
```
|
|
81
82
|
|
|
83
|
+
Every leak names its line (`report.md:31`), so fix them where they live.
|
|
82
84
|
Fix leaks by *going and getting the receipt* — running the query, opening
|
|
83
85
|
the export — never by deleting the number, weakening the claim, or wrapping
|
|
84
86
|
a measurement in `{{raw:}}` to silence the checker. A `derived-mismatch` is
|
package/dist/cli.js
CHANGED
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
import { access, readFile, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { join, resolve } from 'node:path';
|
|
4
4
|
import { compile } from './compile.js';
|
|
5
|
+
import { basename } from 'node:path';
|
|
5
6
|
import { refresh } from './refresh.js';
|
|
7
|
+
import { init } from './init.js';
|
|
6
8
|
import { pathToFileURL } from 'node:url';
|
|
7
9
|
const USAGE = `watertight — reports that hold water. Every number carries its receipt;
|
|
8
10
|
an ungrounded claim is a leak, and a report with leaks does not build.
|
|
@@ -11,8 +13,11 @@ Usage
|
|
|
11
13
|
watertight <dir> compile <dir>/report.md + <dir>/metrics.json → <dir>/report.html
|
|
12
14
|
watertight <report> <ir> explicit file paths
|
|
13
15
|
watertight refresh <dir> re-fetch metric values from their sources, update metrics.json
|
|
16
|
+
watertight init [dir] scaffold a report.md + metrics.json pair that already holds water
|
|
14
17
|
|
|
15
18
|
Options
|
|
19
|
+
--max-age <days> compile only: fail any metric whose fetched_at is older —
|
|
20
|
+
numbers age, and a stale receipt is quietly becoming a leak
|
|
16
21
|
--format <html|md> output format (default: html). md is grounded markdown with a
|
|
17
22
|
receipts appendix — pastes into Notion, PR bodies or Slack intact
|
|
18
23
|
--out <file> where to write the output (default: report.html / report.grounded.md)
|
|
@@ -39,6 +44,7 @@ function parseArgs(argv) {
|
|
|
39
44
|
let allowCommands = false;
|
|
40
45
|
let fetchersPath;
|
|
41
46
|
let format = 'html';
|
|
47
|
+
let maxAgeDays;
|
|
42
48
|
let help = false;
|
|
43
49
|
let version = false;
|
|
44
50
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -55,6 +61,13 @@ function parseArgs(argv) {
|
|
|
55
61
|
fetchersPath = args[++i];
|
|
56
62
|
else if (arg === '--format')
|
|
57
63
|
format = args[++i] === 'md' ? 'md' : 'html';
|
|
64
|
+
else if (arg === '--max-age') {
|
|
65
|
+
maxAgeDays = Number(args[++i]);
|
|
66
|
+
if (!Number.isFinite(maxAgeDays) || maxAgeDays < 0) {
|
|
67
|
+
console.error(`error: --max-age needs a number of days, got "${args[i]}"`);
|
|
68
|
+
process.exit(2);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
58
71
|
else if (arg === '--json')
|
|
59
72
|
json = true;
|
|
60
73
|
else if (arg === '-h' || arg === '--help')
|
|
@@ -64,10 +77,10 @@ function parseArgs(argv) {
|
|
|
64
77
|
else if (!arg.startsWith('-'))
|
|
65
78
|
positional.push(arg);
|
|
66
79
|
}
|
|
67
|
-
const command = positional[0] === 'refresh' ? 'refresh' : 'compile';
|
|
68
|
-
if (command
|
|
80
|
+
const command = positional[0] === 'refresh' ? 'refresh' : positional[0] === 'init' ? 'init' : 'compile';
|
|
81
|
+
if (command !== 'compile')
|
|
69
82
|
positional.shift();
|
|
70
|
-
return { command, positional, out, check, json, help, version, dryRun, allowCommands, format, fetchersPath };
|
|
83
|
+
return { command, positional, out, check, json, help, version, dryRun, allowCommands, format, fetchersPath, maxAgeDays };
|
|
71
84
|
}
|
|
72
85
|
async function exists(path) {
|
|
73
86
|
try {
|
|
@@ -85,6 +98,20 @@ async function main() {
|
|
|
85
98
|
return console.log(pkg.version);
|
|
86
99
|
if (opts.help)
|
|
87
100
|
return console.log(USAGE);
|
|
101
|
+
if (opts.command === 'init') {
|
|
102
|
+
const dir = resolve(opts.positional[0] ?? '.');
|
|
103
|
+
try {
|
|
104
|
+
const { written } = await init(dir);
|
|
105
|
+
for (const f of written)
|
|
106
|
+
console.log(` + ${f}`);
|
|
107
|
+
console.log('\nCompile it: watertight ' + (opts.positional[0] ?? '.'));
|
|
108
|
+
process.exit(0);
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
console.error(`error: ${err instanceof Error ? err.message : String(err)}`);
|
|
112
|
+
process.exit(2);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
88
115
|
// a mistyped path must fail loudly, never read as an empty report
|
|
89
116
|
let reportPath;
|
|
90
117
|
let irPath;
|
|
@@ -132,7 +159,7 @@ async function main() {
|
|
|
132
159
|
: `\n${r.changes.length} change(s)${opts.dryRun ? ' (dry run — nothing written)' : r.wrote ? ` — updated ${irPath}` : ''}`);
|
|
133
160
|
process.exit(r.errors.length > 0 ? 1 : 0);
|
|
134
161
|
}
|
|
135
|
-
const result = await compile(reportPath, irPath, opts.format);
|
|
162
|
+
const result = await compile(reportPath, irPath, { format: opts.format, maxAgeDays: opts.maxAgeDays });
|
|
136
163
|
const defaultName = opts.format === 'md' ? 'report.grounded.md' : 'report.html';
|
|
137
164
|
const outPath = resolve(opts.out ?? join(reportPath, '..', defaultName));
|
|
138
165
|
if (opts.json) {
|
|
@@ -143,7 +170,8 @@ async function main() {
|
|
|
143
170
|
if (result.leaks.length > 0) {
|
|
144
171
|
console.log(`\n${result.leaks.length} leak(s) — the report does not hold water:\n`);
|
|
145
172
|
for (const leak of result.leaks) {
|
|
146
|
-
|
|
173
|
+
const where = leak.line !== undefined ? `${basename(reportPath)}:${leak.line} — ` : '';
|
|
174
|
+
console.log(` ✗ [${leak.rule}] ${where}${leak.message}`);
|
|
147
175
|
if (leak.detail)
|
|
148
176
|
console.log(` ${leak.detail}`);
|
|
149
177
|
}
|
package/dist/compile.js
CHANGED
|
@@ -3,9 +3,30 @@ import { parseIr } from './ir.js';
|
|
|
3
3
|
import { render } from './render.js';
|
|
4
4
|
import { renderMarkdown } from './renderMd.js';
|
|
5
5
|
import { scanNakedNumbers, scanRefs } from './scan.js';
|
|
6
|
-
export async function compile(reportPath, irPath,
|
|
6
|
+
export async function compile(reportPath, irPath, options = 'html') {
|
|
7
|
+
const opts = typeof options === 'string' ? { format: options } : options;
|
|
8
|
+
const format = opts.format ?? 'html';
|
|
7
9
|
const report = await readFile(reportPath, 'utf8');
|
|
8
10
|
const { ir, leaks } = parseIr(JSON.parse(await readFile(irPath, 'utf8')));
|
|
11
|
+
if (ir && opts.maxAgeDays !== undefined) {
|
|
12
|
+
const now = opts.now ?? new Date();
|
|
13
|
+
for (const [key, m] of Object.entries(ir.metrics)) {
|
|
14
|
+
if (!m.fetched_at)
|
|
15
|
+
continue;
|
|
16
|
+
const fetched = new Date(m.fetched_at);
|
|
17
|
+
if (Number.isNaN(fetched.getTime()))
|
|
18
|
+
continue; // "-" and friends: no date to age
|
|
19
|
+
const days = Math.floor((now.getTime() - fetched.getTime()) / 86_400_000);
|
|
20
|
+
if (days > opts.maxAgeDays) {
|
|
21
|
+
leaks.push({
|
|
22
|
+
severity: 'error',
|
|
23
|
+
rule: 'stale-metric',
|
|
24
|
+
message: `metric "${key}" was fetched ${m.fetched_at} — ${days} days ago, older than max-age ${opts.maxAgeDays}`,
|
|
25
|
+
detail: 'Run watertight refresh, or re-fetch by hand and update fetched_at.',
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
9
30
|
const grounded = {
|
|
10
31
|
metrics: [...report.matchAll(/\{\{m:/g)].length,
|
|
11
32
|
identifiers: [...report.matchAll(/\{\{id:/g)].length,
|
package/dist/init.js
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { writeFile } from 'node:fs/promises';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
/**
|
|
5
|
+
* Scaffold a report pair that holds water as written — the first compile a new
|
|
6
|
+
* user runs must succeed, so every figure in the template is already grounded.
|
|
7
|
+
*/
|
|
8
|
+
const REPORT = `# Checkout experiment — verification
|
|
9
|
+
|
|
10
|
+
Rolled out in \`{{id:app_version}}\` behind \`{{id:flag}}\`.
|
|
11
|
+
|
|
12
|
+
## Outcome
|
|
13
|
+
|
|
14
|
+
Conversion moved from {{m:conversion_before}} to {{m:conversion_after}} —
|
|
15
|
+
a change of {{m:conversion_change}}.
|
|
16
|
+
|
|
17
|
+
{{claim: state your conclusion here, pinned to the metrics that support it | evidence: conversion_change}}
|
|
18
|
+
|
|
19
|
+
## Next steps
|
|
20
|
+
|
|
21
|
+
Replace these metrics with your own. Every number needs a receipt:
|
|
22
|
+
run \`watertight .\` and fix what leaks.
|
|
23
|
+
`;
|
|
24
|
+
const METRICS = `${JSON.stringify({
|
|
25
|
+
metrics: {
|
|
26
|
+
conversion_before: {
|
|
27
|
+
value: 0.031,
|
|
28
|
+
unit: 'ratio',
|
|
29
|
+
definition: 'purchases / sessions entering checkout',
|
|
30
|
+
source: { type: 'sql', query: 'replace-with-your-query.sql' },
|
|
31
|
+
window: '2026-01-01 ~ 2026-01-14',
|
|
32
|
+
fetched_at: '2026-01-15',
|
|
33
|
+
},
|
|
34
|
+
conversion_after: {
|
|
35
|
+
value: 0.036,
|
|
36
|
+
unit: 'ratio',
|
|
37
|
+
definition: 'purchases / sessions entering checkout',
|
|
38
|
+
source: { type: 'sql', query: 'replace-with-your-query.sql' },
|
|
39
|
+
window: '2026-01-15 ~ 2026-01-28',
|
|
40
|
+
fetched_at: '2026-01-29',
|
|
41
|
+
},
|
|
42
|
+
conversion_change: {
|
|
43
|
+
value: 0.161,
|
|
44
|
+
unit: 'ratio-point',
|
|
45
|
+
definition: 'relative change in conversion',
|
|
46
|
+
derived: { op: 'pct_change', before: 'conversion_before', after: 'conversion_after' },
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
identifiers: { app_version: '1.0.0', flag: 'checkout_v2' },
|
|
50
|
+
}, null, 2)}\n`;
|
|
51
|
+
/** Refuses to touch anything that already exists — init never overwrites. */
|
|
52
|
+
export async function init(dir) {
|
|
53
|
+
const files = [
|
|
54
|
+
[join(dir, 'report.md'), REPORT],
|
|
55
|
+
[join(dir, 'metrics.json'), METRICS],
|
|
56
|
+
];
|
|
57
|
+
for (const [path] of files) {
|
|
58
|
+
if (existsSync(path))
|
|
59
|
+
throw new Error(`refusing to overwrite ${path}`);
|
|
60
|
+
}
|
|
61
|
+
for (const [path, content] of files)
|
|
62
|
+
await writeFile(path, content);
|
|
63
|
+
return { written: files.map(([p]) => p) };
|
|
64
|
+
}
|
package/dist/ir.js
CHANGED
|
@@ -72,7 +72,23 @@ export function parseIr(raw) {
|
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
74
|
else if (m.derived.op === 'pct_change') {
|
|
75
|
-
const
|
|
75
|
+
const endpoint = (v) => {
|
|
76
|
+
if (typeof v === 'number')
|
|
77
|
+
return v;
|
|
78
|
+
const ref = metrics[v];
|
|
79
|
+
return ref && typeof ref.value === 'number' ? ref.value : undefined;
|
|
80
|
+
};
|
|
81
|
+
const before = endpoint(m.derived.before);
|
|
82
|
+
const after = endpoint(m.derived.after);
|
|
83
|
+
if (before === undefined || after === undefined || before === 0) {
|
|
84
|
+
leaks.push({
|
|
85
|
+
severity: 'error',
|
|
86
|
+
rule: 'bad-derived',
|
|
87
|
+
message: `metric "${key}": pct_change endpoints must be numbers or scalar metric keys (got ${m.derived.before} → ${m.derived.after})`,
|
|
88
|
+
});
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
const computed = (after - before) / before;
|
|
76
92
|
if (!roundsTo(m.value, computed)) {
|
|
77
93
|
leaks.push({
|
|
78
94
|
severity: 'error',
|
package/dist/refresh.js
CHANGED
|
@@ -119,9 +119,15 @@ export async function refresh(irPath, options) {
|
|
|
119
119
|
}, 0);
|
|
120
120
|
}
|
|
121
121
|
else {
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
|
|
122
|
+
const endpoint = (v) => {
|
|
123
|
+
if (typeof v === 'number')
|
|
124
|
+
return v;
|
|
125
|
+
const ref = raw.metrics[v];
|
|
126
|
+
return ref && typeof ref.value === 'number' ? ref.value : undefined;
|
|
127
|
+
};
|
|
128
|
+
const before = endpoint(m.derived.before);
|
|
129
|
+
const after = endpoint(m.derived.after);
|
|
130
|
+
if (before === undefined || after === undefined || before === 0)
|
|
125
131
|
continue;
|
|
126
132
|
// keep the author's stated precision — refresh must not turn 0.155 into 0.1551724
|
|
127
133
|
const decimals = (String(m.value).split('.')[1] ?? '').length;
|
package/dist/render.js
CHANGED
|
@@ -12,13 +12,13 @@ export function formatValue(m) {
|
|
|
12
12
|
return `${m.value >= 0 ? '+' : ''}${(m.value * 100).toFixed(1)}%p`;
|
|
13
13
|
return `${m.value.toLocaleString()} ${m.unit}`.trim();
|
|
14
14
|
}
|
|
15
|
-
export function receipt(m) {
|
|
15
|
+
export function receipt(m, includeDefinition = true) {
|
|
16
16
|
const source = m.derived
|
|
17
17
|
? m.derived.op === 'sum'
|
|
18
18
|
? `= ${m.derived.of.join(' + ')} (recomputed)`
|
|
19
19
|
: `= ${m.derived.before} → ${m.derived.after} (recomputed)`
|
|
20
20
|
: [m.source?.type, ...Object.entries(m.source ?? {}).filter(([k]) => k !== 'type').map(([, v]) => String(v))].filter(Boolean).join(' · ');
|
|
21
|
-
return [source, m.window, m.fetched_at && `fetched ${m.fetched_at}`, m.definition].filter(Boolean).join(' · ');
|
|
21
|
+
return [source, m.window, m.fetched_at && `fetched ${m.fetched_at}`, includeDefinition && m.definition].filter(Boolean).join(' · ');
|
|
22
22
|
}
|
|
23
23
|
/** Minimal markdown: headings, paragraphs, bold, unordered lists. The narrative layer is deliberately thin. */
|
|
24
24
|
function markdown(src) {
|
package/dist/renderMd.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { formatValue, receipt } from './render.js';
|
|
2
|
+
const SUPERSCRIPT = '⁰¹²³⁴⁵⁶⁷⁸⁹';
|
|
3
|
+
const sup = (n) => `⁽${String(n).split('').map((d) => SUPERSCRIPT[Number(d)]).join('')}⁾`;
|
|
2
4
|
/**
|
|
3
5
|
* Grounded markdown: the portable render target. Every figure is bold with a superscript
|
|
4
6
|
* that points into a receipts appendix, so the output pastes into Notion, a PR body, or
|
|
@@ -11,7 +13,7 @@ export function renderMarkdown(report, ir) {
|
|
|
11
13
|
.replace(/\{\{m:([\w-]+)\}\}/g, (_, key) => {
|
|
12
14
|
if (!used.includes(key))
|
|
13
15
|
used.push(key);
|
|
14
|
-
return `**${formatValue(ir.metrics[key])}**
|
|
16
|
+
return `**${formatValue(ir.metrics[key])}** ${sup(used.indexOf(key) + 1)}`;
|
|
15
17
|
})
|
|
16
18
|
.replace(/\{\{id:([\w-]+)\}\}/g, (_, key) => `\`${ir.identifiers[key]}\``)
|
|
17
19
|
.replace(/\{\{claim:([^|}]*)\|\s*evidence:([^}]*)\}\}/g, (_, text, evidence) => `**${text.trim()}** *(evidence: ${evidence.trim()})*`)
|
|
@@ -20,8 +22,8 @@ export function renderMarkdown(report, ir) {
|
|
|
20
22
|
.map((key, i) => {
|
|
21
23
|
const m = ir.metrics[key];
|
|
22
24
|
const definition = m.definition ? ` — ${m.definition}` : '';
|
|
23
|
-
return `${i + 1}. **${key}** = ${formatValue(m)}${definition}\n ${receipt(m)}`;
|
|
25
|
+
return `${i + 1}. **${key}** = ${formatValue(m)}${definition}\n ${receipt(m, false)}`;
|
|
24
26
|
})
|
|
25
27
|
.join('\n');
|
|
26
|
-
return `${body}\n\n---\n\n### Receipts (${used.length} metrics)\n\n${appendix}\n`;
|
|
28
|
+
return `${body.trimEnd()}\n\n---\n\n### Receipts (${used.length} metrics)\n\n${appendix}\n`;
|
|
27
29
|
}
|
package/dist/scan.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
/** Same-length whitespace, newlines kept — stripping must not move anything */
|
|
2
|
+
const blank = (s) => s.replace(/[^\n]/g, ' ');
|
|
3
|
+
const lineAt = (text, index) => text.slice(0, index).split('\n').length;
|
|
1
4
|
/**
|
|
2
5
|
* Find numbers in the narrative that are not bound to the IR. These are the leaks the
|
|
3
6
|
* tool exists to catch: a typed, transcribed, or hallucinated figure reads exactly like
|
|
@@ -5,16 +8,19 @@
|
|
|
5
8
|
*/
|
|
6
9
|
export function scanNakedNumbers(report) {
|
|
7
10
|
const stripped = report
|
|
8
|
-
.replace(/\{\{(m|id):[\w-]+\}\}/g,
|
|
9
|
-
// a claim's prose stays scanned — only its syntax is
|
|
10
|
-
// smuggled into claim text is still a leak
|
|
11
|
-
.replace(/\{\{claim:([^|}]*)\|[^}]*\}\}/g, (
|
|
12
|
-
.
|
|
13
|
-
.
|
|
14
|
-
|
|
15
|
-
.replace(/\
|
|
16
|
-
.replace(
|
|
17
|
-
.replace(
|
|
11
|
+
.replace(/\{\{(m|id):[\w-]+\}\}/g, blank)
|
|
12
|
+
// a claim's prose stays scanned — only its syntax is blanked, so a number
|
|
13
|
+
// smuggled into claim text is still a leak, at its true position
|
|
14
|
+
.replace(/\{\{claim:([^|}]*)\|[^}]*\}\}/g, (whole, text) => {
|
|
15
|
+
const start = whole.indexOf(text);
|
|
16
|
+
return blank(whole.slice(0, start)) + text + blank(whole.slice(start + text.length));
|
|
17
|
+
})
|
|
18
|
+
.replace(/\{\{raw:[^}]*\}\}/g, blank) // explicit, greppable escape hatch
|
|
19
|
+
.replace(/```[\s\S]*?```/g, blank) // fenced code
|
|
20
|
+
.replace(/`[^`\n]*`/g, blank) // inline code
|
|
21
|
+
.replace(/\d{4}-\d{2}-\d{2}/g, blank) // ISO dates locate, they do not measure
|
|
22
|
+
.replace(/^#+ .*$/gm, blank) // headings
|
|
23
|
+
.replace(/^\s*\d+\.\s/gm, blank); // ordered-list markers
|
|
18
24
|
const leaks = [];
|
|
19
25
|
for (const m of stripped.matchAll(/\d[\d,.]*\s*(%p?|[가-힣]{1,2})?/g)) {
|
|
20
26
|
const token = m[0].trim();
|
|
@@ -23,6 +29,7 @@ export function scanNakedNumbers(report) {
|
|
|
23
29
|
leaks.push({
|
|
24
30
|
severity: 'error',
|
|
25
31
|
rule: 'naked-number',
|
|
32
|
+
line: lineAt(stripped, m.index),
|
|
26
33
|
message: `"${token}" appears in the narrative without a receipt`,
|
|
27
34
|
detail: 'Bind it to the IR as {{m:…}} or {{id:…}}, or mark deliberate prose as {{raw:…}}.',
|
|
28
35
|
});
|
|
@@ -32,39 +39,43 @@ export function scanNakedNumbers(report) {
|
|
|
32
39
|
/** Every reference in the narrative must resolve; a dangling one is authoring drift. */
|
|
33
40
|
export function scanRefs(report, metricKeys, idKeys) {
|
|
34
41
|
const leaks = [];
|
|
35
|
-
for (const
|
|
42
|
+
for (const m of report.matchAll(/\{\{claim:([^|}]*)\|\s*evidence:([^}]*)\}\}/g)) {
|
|
43
|
+
const [, text, evidence] = m;
|
|
44
|
+
const line = lineAt(report, m.index);
|
|
36
45
|
const keys = evidence.split(',').map((k) => k.trim()).filter(Boolean);
|
|
37
46
|
if (keys.length === 0) {
|
|
38
47
|
leaks.push({
|
|
39
48
|
severity: 'error',
|
|
40
49
|
rule: 'claim-without-evidence',
|
|
50
|
+
line,
|
|
41
51
|
message: `claim "${text.trim()}" names no evidence`,
|
|
42
52
|
detail: 'A conclusion must point at the metrics it rests on. Whether they support it stays a human judgement — but they must be attached.',
|
|
43
53
|
});
|
|
44
54
|
}
|
|
45
55
|
for (const key of keys) {
|
|
46
56
|
if (!metricKeys.has(key)) {
|
|
47
|
-
leaks.push({ severity: 'error', rule: 'unknown-ref', message: `claim "${text.trim()}" cites unknown metric "${key}"` });
|
|
57
|
+
leaks.push({ severity: 'error', rule: 'unknown-ref', line, message: `claim "${text.trim()}" cites unknown metric "${key}"` });
|
|
48
58
|
}
|
|
49
59
|
}
|
|
50
60
|
}
|
|
51
61
|
// a claim marker missing its evidence clause entirely is malformed, not exempt
|
|
52
|
-
for (const
|
|
62
|
+
for (const m of report.matchAll(/\{\{claim:([^|}]*)\}\}/g)) {
|
|
53
63
|
leaks.push({
|
|
54
64
|
severity: 'error',
|
|
55
65
|
rule: 'claim-without-evidence',
|
|
56
|
-
|
|
66
|
+
line: lineAt(report, m.index),
|
|
67
|
+
message: `claim "${m[1].trim()}" names no evidence`,
|
|
57
68
|
detail: 'Write it as {{claim: … | evidence: metric_key}}.',
|
|
58
69
|
});
|
|
59
70
|
}
|
|
60
|
-
for (const
|
|
61
|
-
if (!metricKeys.has(
|
|
62
|
-
leaks.push({ severity: 'error', rule: 'unknown-ref', message: `{{m:${
|
|
71
|
+
for (const m of report.matchAll(/\{\{m:([\w-]+)\}\}/g)) {
|
|
72
|
+
if (!metricKeys.has(m[1])) {
|
|
73
|
+
leaks.push({ severity: 'error', rule: 'unknown-ref', line: lineAt(report, m.index), message: `{{m:${m[1]}}} is not in the IR` });
|
|
63
74
|
}
|
|
64
75
|
}
|
|
65
|
-
for (const
|
|
66
|
-
if (!idKeys.has(
|
|
67
|
-
leaks.push({ severity: 'error', rule: 'unknown-ref', message: `{{id:${
|
|
76
|
+
for (const m of report.matchAll(/\{\{id:([\w-]+)\}\}/g)) {
|
|
77
|
+
if (!idKeys.has(m[1])) {
|
|
78
|
+
leaks.push({ severity: 'error', rule: 'unknown-ref', line: lineAt(report, m.index), message: `{{id:${m[1]}}} is not in the IR` });
|
|
68
79
|
}
|
|
69
80
|
}
|
|
70
81
|
return leaks;
|