watertight 0.1.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +134 -62
- package/SKILL.md +5 -0
- package/dist/cli.js +57 -7
- package/dist/compile.js +22 -1
- package/dist/init.js +64 -0
- package/dist/ir.js +17 -1
- package/dist/refresh.js +28 -7
- package/dist/render.js +2 -2
- package/dist/renderMd.js +5 -3
- package/dist/scan.js +31 -20
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -1,29 +1,43 @@
|
|
|
1
1
|
# watertight
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/watertight)
|
|
4
|
+
[](https://github.com/camosss/watertight/actions/workflows/ci.yml)
|
|
5
|
+

|
|
6
|
+

|
|
7
|
+
[](LICENSE)
|
|
5
8
|
|
|
9
|
+
Reports that hold water — every number carries its **receipt**, and ungrounded claims **fail the build**.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
npx watertight report.md metrics.json
|
|
6
13
|
```
|
|
7
|
-
$ watertight report.md metrics.json
|
|
8
14
|
|
|
9
|
-
|
|
15
|
+
```
|
|
16
|
+
watertight v0.2.0 · 20 grounded metrics · 4 claims · 5 identifiers
|
|
10
17
|
holds water → report.html
|
|
11
18
|
```
|
|
12
19
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
20
|
+
An ungrounded number is a **leak**. A report with zero leaks **holds water**:
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
2 leak(s) — the report does not hold water:
|
|
24
|
+
|
|
25
|
+
✗ [naked-number] report.md:31 — "15%" appears in the narrative without a receipt
|
|
26
|
+
✗ [derived-mismatch] metric "revenue_total" is 1440, but its parts sum to 1428
|
|
27
|
+
```
|
|
19
28
|
|
|
20
|
-
|
|
29
|
+
<br>
|
|
21
30
|
|
|
22
31
|
## Why this exists
|
|
23
32
|
|
|
33
|
+
A number typed into prose has no memory of where it came from. Two weeks later
|
|
34
|
+
nobody can say which query produced it, whether it was rounded, or whether it
|
|
35
|
+
was ever true — and when an LLM helps write the report, "never true" is a live
|
|
36
|
+
possibility.
|
|
37
|
+
|
|
24
38
|
This came out of writing a real ad-revenue verification report with an AI
|
|
25
|
-
assistant. Compiling the draft surfaced three distinct failure classes in
|
|
26
|
-
|
|
39
|
+
assistant. Compiling the draft surfaced three distinct failure classes in one
|
|
40
|
+
document:
|
|
27
41
|
|
|
28
42
|
- a human had written **+15%** for a delta that computed to **+15.5%** —
|
|
29
43
|
a rounding transcription error nobody had caught;
|
|
@@ -31,10 +45,14 @@ one document:
|
|
|
31
45
|
- the assistant confidently "remembered" a figure that appeared nowhere in
|
|
32
46
|
any export.
|
|
33
47
|
|
|
34
|
-
All three are the same disease — a number with no receipt — and all three
|
|
35
|
-
|
|
48
|
+
All three are the same disease — a number with no receipt — and all three are
|
|
49
|
+
mechanically detectable. So: detect them, every build. watertight treats a
|
|
50
|
+
report like source code: the narrative references metrics, the metrics carry
|
|
51
|
+
provenance, and a compiler refuses to build anything it cannot trace.
|
|
36
52
|
|
|
37
|
-
|
|
53
|
+
<br>
|
|
54
|
+
|
|
55
|
+
## Quick start
|
|
38
56
|
|
|
39
57
|
```bash
|
|
40
58
|
npm install -g watertight
|
|
@@ -42,25 +60,12 @@ npm install -g watertight
|
|
|
42
60
|
npx watertight report.md metrics.json
|
|
43
61
|
```
|
|
44
62
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
## How it works
|
|
48
|
-
|
|
49
|
-
You write two files.
|
|
50
|
-
|
|
51
|
-
**`metrics.json` — the IR.** Every value, with its receipt:
|
|
63
|
+
You write two files. **`metrics.json`** holds every value with its receipt —
|
|
64
|
+
a measured metric says where it was fetched, over what window, and when:
|
|
52
65
|
|
|
53
66
|
```json
|
|
54
67
|
{
|
|
55
68
|
"metrics": {
|
|
56
|
-
"conversion_before": {
|
|
57
|
-
"value": 0.031,
|
|
58
|
-
"unit": "ratio",
|
|
59
|
-
"definition": "purchases / sessions entering checkout",
|
|
60
|
-
"source": { "type": "sql", "query": "reports/checkout.sql" },
|
|
61
|
-
"window": "2026-08-15 ~ 2026-08-31",
|
|
62
|
-
"fetched_at": "2026-09-15"
|
|
63
|
-
},
|
|
64
69
|
"conversion_after": {
|
|
65
70
|
"value": 0.036,
|
|
66
71
|
"unit": "ratio",
|
|
@@ -68,34 +73,42 @@ You write two files.
|
|
|
68
73
|
"source": { "type": "sql", "query": "reports/checkout.sql" },
|
|
69
74
|
"window": "2026-09-01 ~ 2026-09-14",
|
|
70
75
|
"fetched_at": "2026-09-15"
|
|
71
|
-
},
|
|
72
|
-
"lift": {
|
|
73
|
-
"value": 0.161,
|
|
74
|
-
"unit": "ratio-point",
|
|
75
|
-
"definition": "relative change in conversion",
|
|
76
|
-
"derived": { "op": "pct_change", "before": "conversion_before", "after": "conversion_after" }
|
|
77
|
-
},
|
|
78
|
-
"revenue_total": {
|
|
79
|
-
"value": 1428,
|
|
80
|
-
"unit": "USD",
|
|
81
|
-
"derived": { "op": "sum", "of": ["revenue_ios", "revenue_android"] }
|
|
82
|
-
},
|
|
83
|
-
"revenue_target": {
|
|
84
|
-
"value": [1000, 5000],
|
|
85
|
-
"unit": "USD",
|
|
86
|
-
"source": { "type": "hypothesis", "doc": "PLAN-42" },
|
|
87
|
-
"window": "planning estimate",
|
|
88
|
-
"fetched_at": "-"
|
|
89
76
|
}
|
|
90
77
|
},
|
|
91
|
-
"identifiers": {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
78
|
+
"identifiers": { "app_version": "3.2.0", "flag": "checkout_v2" }
|
|
79
|
+
}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
A **derived** metric is recomputed on every compile — a total that doesn't add
|
|
83
|
+
up, or a delta that doesn't recompute, is a build failure:
|
|
84
|
+
|
|
85
|
+
```json
|
|
86
|
+
"lift": {
|
|
87
|
+
"value": 0.161,
|
|
88
|
+
"unit": "ratio-point",
|
|
89
|
+
"definition": "relative change in conversion",
|
|
90
|
+
"derived": { "op": "pct_change", "before": "conversion_before", "after": "conversion_after" }
|
|
91
|
+
},
|
|
92
|
+
"revenue_total": {
|
|
93
|
+
"value": 1428,
|
|
94
|
+
"unit": "USD",
|
|
95
|
+
"derived": { "op": "sum", "of": ["revenue_ios", "revenue_android"] }
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
A **range** states a hypothesis honestly — plans are receipts too:
|
|
100
|
+
|
|
101
|
+
```json
|
|
102
|
+
"revenue_target": {
|
|
103
|
+
"value": [1000, 5000],
|
|
104
|
+
"unit": "USD",
|
|
105
|
+
"source": { "type": "hypothesis", "doc": "PLAN-42" },
|
|
106
|
+
"window": "planning estimate",
|
|
107
|
+
"fetched_at": "-"
|
|
95
108
|
}
|
|
96
109
|
```
|
|
97
110
|
|
|
98
|
-
**`report.md
|
|
111
|
+
**`report.md`** is the narrative. No literal figures, only references:
|
|
99
112
|
|
|
100
113
|
```markdown
|
|
101
114
|
Rolled out in `{{id:app_version}}` behind `{{id:flag}}`.
|
|
@@ -113,20 +126,40 @@ Compile:
|
|
|
113
126
|
|
|
114
127
|
```bash
|
|
115
128
|
watertight report.md metrics.json # → report.html (self-contained, hover for receipts)
|
|
116
|
-
watertight . --format md # → grounded markdown
|
|
129
|
+
watertight . --format md # → grounded markdown (below)
|
|
117
130
|
watertight . --check # verify only, write nothing (CI)
|
|
131
|
+
watertight . --check --max-age 30 # also fail receipts older than 30 days
|
|
118
132
|
watertight . --json # machine-readable result
|
|
119
133
|
```
|
|
120
134
|
|
|
121
|
-
|
|
135
|
+
<br>
|
|
122
136
|
|
|
123
|
-
|
|
124
|
-
2 leak(s) — the report does not hold water:
|
|
137
|
+
## What the output looks like
|
|
125
138
|
|
|
126
|
-
|
|
127
|
-
|
|
139
|
+
`--format md` produces grounded markdown that pastes into Notion, a PR body,
|
|
140
|
+
or Slack with its provenance intact — every figure bold with a superscript
|
|
141
|
+
into a receipts appendix:
|
|
142
|
+
|
|
143
|
+
```markdown
|
|
144
|
+
Conversion moved from **3.1%** ⁽¹⁾ to **3.6%** ⁽²⁾, a lift of **+16.1%p** ⁽³⁾.
|
|
145
|
+
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
### Receipts (3 metrics)
|
|
149
|
+
|
|
150
|
+
1. **conversion_before** = 3.1% — purchases / sessions entering checkout
|
|
151
|
+
sql · reports/checkout.sql · 2026-08-15 ~ 2026-08-31 · fetched 2026-09-15
|
|
152
|
+
2. **conversion_after** = 3.6% — purchases / sessions entering checkout
|
|
153
|
+
sql · reports/checkout.sql · 2026-09-01 ~ 2026-09-14 · fetched 2026-09-15
|
|
154
|
+
3. **lift** = +16.1%p — relative change in conversion
|
|
155
|
+
= conversion_before → conversion_after (recomputed)
|
|
128
156
|
```
|
|
129
157
|
|
|
158
|
+
The default HTML render is a single self-contained file where hovering any
|
|
159
|
+
figure shows its receipt.
|
|
160
|
+
|
|
161
|
+
<br>
|
|
162
|
+
|
|
130
163
|
## What it checks
|
|
131
164
|
|
|
132
165
|
| check | catches |
|
|
@@ -139,6 +172,9 @@ Any leak fails the build (exit 1) and names the line:
|
|
|
139
172
|
| `claim-without-evidence` | a `{{claim:}}` with no `evidence:` keys, or keys that don't exist |
|
|
140
173
|
| `bad-derived` | derived ops referencing missing or non-numeric inputs |
|
|
141
174
|
| `empty-ir` | a report "grounded" in nothing |
|
|
175
|
+
| `stale-metric` | with `--max-age <days>`: a receipt whose `fetched_at` is older than the budget — numbers age |
|
|
176
|
+
|
|
177
|
+
<br>
|
|
142
178
|
|
|
143
179
|
## Re-verification
|
|
144
180
|
|
|
@@ -153,7 +189,37 @@ can reach, rewrites `value` and `fetched_at`, and recomputes derived values:
|
|
|
153
189
|
- everything else (dashboards, vendor reports, hypotheses) is **named as
|
|
154
190
|
skipped** — never silently assumed fresh
|
|
155
191
|
|
|
156
|
-
|
|
192
|
+
Derived values follow their inputs: sums are recomputed exactly, percentage
|
|
193
|
+
changes at the precision the author stated. `--dry-run` previews changes
|
|
194
|
+
without writing.
|
|
195
|
+
|
|
196
|
+
<br>
|
|
197
|
+
|
|
198
|
+
## Custom fetchers
|
|
199
|
+
|
|
200
|
+
Any other source type can be refreshed through a JS module you point at
|
|
201
|
+
explicitly:
|
|
202
|
+
|
|
203
|
+
```bash
|
|
204
|
+
watertight refresh . --fetchers ./my-fetchers.mjs
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
```js
|
|
208
|
+
// my-fetchers.mjs — export one function per source type
|
|
209
|
+
export async function mixpanel(source) {
|
|
210
|
+
// credentials from the environment, receipt fields from the IR
|
|
211
|
+
return valueFetchedFrom(source.project, source.bookmark)
|
|
212
|
+
}
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
[`examples/fetchers/mixpanel.mjs`](./examples/fetchers/mixpanel.mjs) is a
|
|
216
|
+
working reference. The trust boundary is explicit: a fetcher module loads only
|
|
217
|
+
when the person running refresh names it on the command line — the IR itself
|
|
218
|
+
can never designate one, so an untrusted `metrics.json` still cannot execute
|
|
219
|
+
anything. Built-in `csv`/`json`/`command` adapters always win over a custom
|
|
220
|
+
one of the same name.
|
|
221
|
+
|
|
222
|
+
<br>
|
|
157
223
|
|
|
158
224
|
## For AI-authored reports
|
|
159
225
|
|
|
@@ -162,6 +228,8 @@ receipts first, build the IR from real sources only, reference — never type
|
|
|
162
228
|
figures, and fix leaks by re-fetching, not by weakening the text. The compile
|
|
163
229
|
step turns "please don't hallucinate numbers" from a request into a gate.
|
|
164
230
|
|
|
231
|
+
<br>
|
|
232
|
+
|
|
165
233
|
## What it deliberately does not do
|
|
166
234
|
|
|
167
235
|
- **Judge whether evidence supports a claim.** `{{claim:}}` proves evidence
|
|
@@ -173,6 +241,8 @@ step turns "please don't hallucinate numbers" from a request into a gate.
|
|
|
173
241
|
- **Use an LLM.** Every check is deterministic. The point is to be the fixed
|
|
174
242
|
ground an LLM-assisted workflow can push against.
|
|
175
243
|
|
|
244
|
+
<br>
|
|
245
|
+
|
|
176
246
|
## Related work
|
|
177
247
|
|
|
178
248
|
- [Proof-Carrying Numbers (arXiv:2509.06902)](https://arxiv.org/abs/2509.06902)
|
|
@@ -183,6 +253,8 @@ step turns "please don't hallucinate numbers" from a request into a gate.
|
|
|
183
253
|
proves the "reports as source code" mechanism — but doesn't enforce that
|
|
184
254
|
prose figures stay grounded.
|
|
185
255
|
|
|
256
|
+
<br>
|
|
257
|
+
|
|
186
258
|
## License
|
|
187
259
|
|
|
188
|
-
MIT
|
|
260
|
+
`watertight` is released under an MIT license. See [License](LICENSE) for more information.
|
package/SKILL.md
CHANGED
|
@@ -77,8 +77,10 @@ Never type a figure into prose. Reference it:
|
|
|
77
77
|
watertight report.md metrics.json # writes report.html
|
|
78
78
|
watertight . --format md # grounded markdown (Notion / PR / Slack)
|
|
79
79
|
watertight . --check # CI mode: verify, write nothing
|
|
80
|
+
watertight . --check --max-age 30 # also fail receipts older than 30 days
|
|
80
81
|
```
|
|
81
82
|
|
|
83
|
+
Every leak names its line (`report.md:31`), so fix them where they live.
|
|
82
84
|
Fix leaks by *going and getting the receipt* — running the query, opening
|
|
83
85
|
the export — never by deleting the number, weakening the claim, or wrapping
|
|
84
86
|
a measurement in `{{raw:}}` to silence the checker. A `derived-mismatch` is
|
|
@@ -89,6 +91,9 @@ inputs: recompute at the source and correct whichever side is wrong.
|
|
|
89
91
|
sources, updates `fetched_at`, recomputes derived values, and names every
|
|
90
92
|
metric it could *not* refresh. `command` sources run only under
|
|
91
93
|
`--allow-commands`; never pass that flag on an IR you did not author.
|
|
94
|
+
Other source types (vendor APIs, analytics tools) can be refreshed through
|
|
95
|
+
`--fetchers <module.mjs>` — one exported function per source type, with
|
|
96
|
+
credentials from the environment, never from the IR.
|
|
92
97
|
|
|
93
98
|
## What stays yours
|
|
94
99
|
|
package/dist/cli.js
CHANGED
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
import { access, readFile, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { join, resolve } from 'node:path';
|
|
4
4
|
import { compile } from './compile.js';
|
|
5
|
+
import { basename } from 'node:path';
|
|
5
6
|
import { refresh } from './refresh.js';
|
|
7
|
+
import { init } from './init.js';
|
|
8
|
+
import { pathToFileURL } from 'node:url';
|
|
6
9
|
const USAGE = `watertight — reports that hold water. Every number carries its receipt;
|
|
7
10
|
an ungrounded claim is a leak, and a report with leaks does not build.
|
|
8
11
|
|
|
@@ -10,13 +13,19 @@ Usage
|
|
|
10
13
|
watertight <dir> compile <dir>/report.md + <dir>/metrics.json → <dir>/report.html
|
|
11
14
|
watertight <report> <ir> explicit file paths
|
|
12
15
|
watertight refresh <dir> re-fetch metric values from their sources, update metrics.json
|
|
16
|
+
watertight init [dir] scaffold a report.md + metrics.json pair that already holds water
|
|
13
17
|
|
|
14
18
|
Options
|
|
19
|
+
--max-age <days> compile only: fail any metric whose fetched_at is older —
|
|
20
|
+
numbers age, and a stale receipt is quietly becoming a leak
|
|
15
21
|
--format <html|md> output format (default: html). md is grounded markdown with a
|
|
16
22
|
receipts appendix — pastes into Notion, PR bodies or Slack intact
|
|
17
23
|
--out <file> where to write the output (default: report.html / report.grounded.md)
|
|
18
24
|
--check verify only, write nothing
|
|
19
25
|
--dry-run refresh only: show what would change, write nothing
|
|
26
|
+
--fetchers <file> refresh only: a JS module of custom source adapters, e.g.
|
|
27
|
+
export function mixpanel(source) { ... return value }
|
|
28
|
+
Loading a module runs its code — only pass files you wrote or trust.
|
|
20
29
|
--allow-commands refresh only: let "command" sources run shell (off by default —
|
|
21
30
|
an IR from someone else's repo must not execute code on your machine)
|
|
22
31
|
--json machine-readable result on stdout
|
|
@@ -33,7 +42,9 @@ function parseArgs(argv) {
|
|
|
33
42
|
let json = false;
|
|
34
43
|
let dryRun = false;
|
|
35
44
|
let allowCommands = false;
|
|
45
|
+
let fetchersPath;
|
|
36
46
|
let format = 'html';
|
|
47
|
+
let maxAgeDays;
|
|
37
48
|
let help = false;
|
|
38
49
|
let version = false;
|
|
39
50
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -46,8 +57,17 @@ function parseArgs(argv) {
|
|
|
46
57
|
dryRun = true;
|
|
47
58
|
else if (arg === '--allow-commands')
|
|
48
59
|
allowCommands = true;
|
|
60
|
+
else if (arg === '--fetchers')
|
|
61
|
+
fetchersPath = args[++i];
|
|
49
62
|
else if (arg === '--format')
|
|
50
63
|
format = args[++i] === 'md' ? 'md' : 'html';
|
|
64
|
+
else if (arg === '--max-age') {
|
|
65
|
+
maxAgeDays = Number(args[++i]);
|
|
66
|
+
if (!Number.isFinite(maxAgeDays) || maxAgeDays < 0) {
|
|
67
|
+
console.error(`error: --max-age needs a number of days, got "${args[i]}"`);
|
|
68
|
+
process.exit(2);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
51
71
|
else if (arg === '--json')
|
|
52
72
|
json = true;
|
|
53
73
|
else if (arg === '-h' || arg === '--help')
|
|
@@ -57,10 +77,10 @@ function parseArgs(argv) {
|
|
|
57
77
|
else if (!arg.startsWith('-'))
|
|
58
78
|
positional.push(arg);
|
|
59
79
|
}
|
|
60
|
-
const command = positional[0] === 'refresh' ? 'refresh' : 'compile';
|
|
61
|
-
if (command
|
|
80
|
+
const command = positional[0] === 'refresh' ? 'refresh' : positional[0] === 'init' ? 'init' : 'compile';
|
|
81
|
+
if (command !== 'compile')
|
|
62
82
|
positional.shift();
|
|
63
|
-
return { command, positional, out, check, json, help, version, dryRun, allowCommands, format };
|
|
83
|
+
return { command, positional, out, check, json, help, version, dryRun, allowCommands, format, fetchersPath, maxAgeDays };
|
|
64
84
|
}
|
|
65
85
|
async function exists(path) {
|
|
66
86
|
try {
|
|
@@ -78,6 +98,20 @@ async function main() {
|
|
|
78
98
|
return console.log(pkg.version);
|
|
79
99
|
if (opts.help)
|
|
80
100
|
return console.log(USAGE);
|
|
101
|
+
if (opts.command === 'init') {
|
|
102
|
+
const dir = resolve(opts.positional[0] ?? '.');
|
|
103
|
+
try {
|
|
104
|
+
const { written } = await init(dir);
|
|
105
|
+
for (const f of written)
|
|
106
|
+
console.log(` + ${f}`);
|
|
107
|
+
console.log('\nCompile it: watertight ' + (opts.positional[0] ?? '.'));
|
|
108
|
+
process.exit(0);
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
console.error(`error: ${err instanceof Error ? err.message : String(err)}`);
|
|
112
|
+
process.exit(2);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
81
115
|
// a mistyped path must fail loudly, never read as an empty report
|
|
82
116
|
let reportPath;
|
|
83
117
|
let irPath;
|
|
@@ -90,7 +124,8 @@ async function main() {
|
|
|
90
124
|
reportPath = join(dir, 'report.md');
|
|
91
125
|
irPath = join(dir, 'metrics.json');
|
|
92
126
|
}
|
|
93
|
-
|
|
127
|
+
// refresh works on the IR alone — an IR-only directory is a legitimate workspace
|
|
128
|
+
for (const path of opts.command === 'refresh' ? [irPath] : [reportPath, irPath]) {
|
|
94
129
|
if (!(await exists(path))) {
|
|
95
130
|
console.error(`Not found: ${path}`);
|
|
96
131
|
console.error('Expected report.md and metrics.json — see --help.');
|
|
@@ -98,7 +133,21 @@ async function main() {
|
|
|
98
133
|
}
|
|
99
134
|
}
|
|
100
135
|
if (opts.command === 'refresh') {
|
|
101
|
-
|
|
136
|
+
let fetchers;
|
|
137
|
+
if (opts.fetchersPath) {
|
|
138
|
+
const mod = await import(pathToFileURL(resolve(opts.fetchersPath)).href);
|
|
139
|
+
const exports = (typeof mod.default === 'object' && mod.default !== null ? mod.default : mod);
|
|
140
|
+
fetchers = Object.fromEntries(Object.entries(exports).filter(([, v]) => typeof v === 'function'));
|
|
141
|
+
if (Object.keys(fetchers).length === 0) {
|
|
142
|
+
console.error(`error: ${opts.fetchersPath} exports no functions — nothing to fetch with`);
|
|
143
|
+
process.exit(2);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
const r = await refresh(irPath, { allowCommands: opts.allowCommands, dryRun: opts.dryRun, fetchers });
|
|
147
|
+
if (opts.json) {
|
|
148
|
+
console.log(JSON.stringify(r, null, 2));
|
|
149
|
+
process.exit(r.errors.length > 0 ? 1 : 0);
|
|
150
|
+
}
|
|
102
151
|
for (const c of r.changes)
|
|
103
152
|
console.log(` ${c.key}: ${c.before.toLocaleString()} → ${c.after.toLocaleString()}`);
|
|
104
153
|
for (const s of r.skipped)
|
|
@@ -110,7 +159,7 @@ async function main() {
|
|
|
110
159
|
: `\n${r.changes.length} change(s)${opts.dryRun ? ' (dry run — nothing written)' : r.wrote ? ` — updated ${irPath}` : ''}`);
|
|
111
160
|
process.exit(r.errors.length > 0 ? 1 : 0);
|
|
112
161
|
}
|
|
113
|
-
const result = await compile(reportPath, irPath, opts.format);
|
|
162
|
+
const result = await compile(reportPath, irPath, { format: opts.format, maxAgeDays: opts.maxAgeDays });
|
|
114
163
|
const defaultName = opts.format === 'md' ? 'report.grounded.md' : 'report.html';
|
|
115
164
|
const outPath = resolve(opts.out ?? join(reportPath, '..', defaultName));
|
|
116
165
|
if (opts.json) {
|
|
@@ -121,7 +170,8 @@ async function main() {
|
|
|
121
170
|
if (result.leaks.length > 0) {
|
|
122
171
|
console.log(`\n${result.leaks.length} leak(s) — the report does not hold water:\n`);
|
|
123
172
|
for (const leak of result.leaks) {
|
|
124
|
-
|
|
173
|
+
const where = leak.line !== undefined ? `${basename(reportPath)}:${leak.line} — ` : '';
|
|
174
|
+
console.log(` ✗ [${leak.rule}] ${where}${leak.message}`);
|
|
125
175
|
if (leak.detail)
|
|
126
176
|
console.log(` ${leak.detail}`);
|
|
127
177
|
}
|
package/dist/compile.js
CHANGED
|
@@ -3,9 +3,30 @@ import { parseIr } from './ir.js';
|
|
|
3
3
|
import { render } from './render.js';
|
|
4
4
|
import { renderMarkdown } from './renderMd.js';
|
|
5
5
|
import { scanNakedNumbers, scanRefs } from './scan.js';
|
|
6
|
-
export async function compile(reportPath, irPath,
|
|
6
|
+
export async function compile(reportPath, irPath, options = 'html') {
|
|
7
|
+
const opts = typeof options === 'string' ? { format: options } : options;
|
|
8
|
+
const format = opts.format ?? 'html';
|
|
7
9
|
const report = await readFile(reportPath, 'utf8');
|
|
8
10
|
const { ir, leaks } = parseIr(JSON.parse(await readFile(irPath, 'utf8')));
|
|
11
|
+
if (ir && opts.maxAgeDays !== undefined) {
|
|
12
|
+
const now = opts.now ?? new Date();
|
|
13
|
+
for (const [key, m] of Object.entries(ir.metrics)) {
|
|
14
|
+
if (!m.fetched_at)
|
|
15
|
+
continue;
|
|
16
|
+
const fetched = new Date(m.fetched_at);
|
|
17
|
+
if (Number.isNaN(fetched.getTime()))
|
|
18
|
+
continue; // "-" and friends: no date to age
|
|
19
|
+
const days = Math.floor((now.getTime() - fetched.getTime()) / 86_400_000);
|
|
20
|
+
if (days > opts.maxAgeDays) {
|
|
21
|
+
leaks.push({
|
|
22
|
+
severity: 'error',
|
|
23
|
+
rule: 'stale-metric',
|
|
24
|
+
message: `metric "${key}" was fetched ${m.fetched_at} — ${days} days ago, older than max-age ${opts.maxAgeDays}`,
|
|
25
|
+
detail: 'Run watertight refresh, or re-fetch by hand and update fetched_at.',
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
9
30
|
const grounded = {
|
|
10
31
|
metrics: [...report.matchAll(/\{\{m:/g)].length,
|
|
11
32
|
identifiers: [...report.matchAll(/\{\{id:/g)].length,
|
package/dist/init.js
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { writeFile } from 'node:fs/promises';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
/**
|
|
5
|
+
* Scaffold a report pair that holds water as written — the first compile a new
|
|
6
|
+
* user runs must succeed, so every figure in the template is already grounded.
|
|
7
|
+
*/
|
|
8
|
+
const REPORT = `# Checkout experiment — verification
|
|
9
|
+
|
|
10
|
+
Rolled out in \`{{id:app_version}}\` behind \`{{id:flag}}\`.
|
|
11
|
+
|
|
12
|
+
## Outcome
|
|
13
|
+
|
|
14
|
+
Conversion moved from {{m:conversion_before}} to {{m:conversion_after}} —
|
|
15
|
+
a change of {{m:conversion_change}}.
|
|
16
|
+
|
|
17
|
+
{{claim: state your conclusion here, pinned to the metrics that support it | evidence: conversion_change}}
|
|
18
|
+
|
|
19
|
+
## Next steps
|
|
20
|
+
|
|
21
|
+
Replace these metrics with your own. Every number needs a receipt:
|
|
22
|
+
run \`watertight .\` and fix what leaks.
|
|
23
|
+
`;
|
|
24
|
+
const METRICS = `${JSON.stringify({
|
|
25
|
+
metrics: {
|
|
26
|
+
conversion_before: {
|
|
27
|
+
value: 0.031,
|
|
28
|
+
unit: 'ratio',
|
|
29
|
+
definition: 'purchases / sessions entering checkout',
|
|
30
|
+
source: { type: 'sql', query: 'replace-with-your-query.sql' },
|
|
31
|
+
window: '2026-01-01 ~ 2026-01-14',
|
|
32
|
+
fetched_at: '2026-01-15',
|
|
33
|
+
},
|
|
34
|
+
conversion_after: {
|
|
35
|
+
value: 0.036,
|
|
36
|
+
unit: 'ratio',
|
|
37
|
+
definition: 'purchases / sessions entering checkout',
|
|
38
|
+
source: { type: 'sql', query: 'replace-with-your-query.sql' },
|
|
39
|
+
window: '2026-01-15 ~ 2026-01-28',
|
|
40
|
+
fetched_at: '2026-01-29',
|
|
41
|
+
},
|
|
42
|
+
conversion_change: {
|
|
43
|
+
value: 0.161,
|
|
44
|
+
unit: 'ratio-point',
|
|
45
|
+
definition: 'relative change in conversion',
|
|
46
|
+
derived: { op: 'pct_change', before: 'conversion_before', after: 'conversion_after' },
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
identifiers: { app_version: '1.0.0', flag: 'checkout_v2' },
|
|
50
|
+
}, null, 2)}\n`;
|
|
51
|
+
/** Refuses to touch anything that already exists — init never overwrites. */
|
|
52
|
+
export async function init(dir) {
|
|
53
|
+
const files = [
|
|
54
|
+
[join(dir, 'report.md'), REPORT],
|
|
55
|
+
[join(dir, 'metrics.json'), METRICS],
|
|
56
|
+
];
|
|
57
|
+
for (const [path] of files) {
|
|
58
|
+
if (existsSync(path))
|
|
59
|
+
throw new Error(`refusing to overwrite ${path}`);
|
|
60
|
+
}
|
|
61
|
+
for (const [path, content] of files)
|
|
62
|
+
await writeFile(path, content);
|
|
63
|
+
return { written: files.map(([p]) => p) };
|
|
64
|
+
}
|
package/dist/ir.js
CHANGED
|
@@ -72,7 +72,23 @@ export function parseIr(raw) {
|
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
74
|
else if (m.derived.op === 'pct_change') {
|
|
75
|
-
const
|
|
75
|
+
const endpoint = (v) => {
|
|
76
|
+
if (typeof v === 'number')
|
|
77
|
+
return v;
|
|
78
|
+
const ref = metrics[v];
|
|
79
|
+
return ref && typeof ref.value === 'number' ? ref.value : undefined;
|
|
80
|
+
};
|
|
81
|
+
const before = endpoint(m.derived.before);
|
|
82
|
+
const after = endpoint(m.derived.after);
|
|
83
|
+
if (before === undefined || after === undefined || before === 0) {
|
|
84
|
+
leaks.push({
|
|
85
|
+
severity: 'error',
|
|
86
|
+
rule: 'bad-derived',
|
|
87
|
+
message: `metric "${key}": pct_change endpoints must be numbers or scalar metric keys (got ${m.derived.before} → ${m.derived.after})`,
|
|
88
|
+
});
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
const computed = (after - before) / before;
|
|
76
92
|
if (!roundsTo(m.value, computed)) {
|
|
77
93
|
leaks.push({
|
|
78
94
|
severity: 'error',
|
package/dist/refresh.js
CHANGED
|
@@ -91,8 +91,11 @@ export async function refresh(irPath, options) {
|
|
|
91
91
|
}
|
|
92
92
|
value = fetchCommand(m.source, baseDir);
|
|
93
93
|
}
|
|
94
|
+
else if (options.fetchers?.[m.source.type]) {
|
|
95
|
+
value = toNumber(await options.fetchers[m.source.type](m.source, { key, metric: m, baseDir }), `fetcher "${m.source.type}" for "${key}"`);
|
|
96
|
+
}
|
|
94
97
|
else {
|
|
95
|
-
result.skipped.push({ key, reason: `no
|
|
98
|
+
result.skipped.push({ key, reason: `no adapter for source type "${m.source.type}"` });
|
|
96
99
|
continue;
|
|
97
100
|
}
|
|
98
101
|
if (value !== m.value)
|
|
@@ -104,14 +107,32 @@ export async function refresh(irPath, options) {
|
|
|
104
107
|
result.errors.push({ key, message: err instanceof Error ? err.message : String(err) });
|
|
105
108
|
}
|
|
106
109
|
}
|
|
107
|
-
//
|
|
110
|
+
// inputs may have moved, so derived values are recomputed rather than left to go stale
|
|
108
111
|
for (const [key, m] of Object.entries(raw.metrics)) {
|
|
109
|
-
if (m.derived
|
|
112
|
+
if (!m.derived || Array.isArray(m.value))
|
|
110
113
|
continue;
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
114
|
+
let computed;
|
|
115
|
+
if (m.derived.op === 'sum') {
|
|
116
|
+
computed = m.derived.of.reduce((acc, ref) => {
|
|
117
|
+
const part = raw.metrics[ref];
|
|
118
|
+
return acc + (part && typeof part.value === 'number' ? part.value : NaN);
|
|
119
|
+
}, 0);
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
const endpoint = (v) => {
|
|
123
|
+
if (typeof v === 'number')
|
|
124
|
+
return v;
|
|
125
|
+
const ref = raw.metrics[v];
|
|
126
|
+
return ref && typeof ref.value === 'number' ? ref.value : undefined;
|
|
127
|
+
};
|
|
128
|
+
const before = endpoint(m.derived.before);
|
|
129
|
+
const after = endpoint(m.derived.after);
|
|
130
|
+
if (before === undefined || after === undefined || before === 0)
|
|
131
|
+
continue;
|
|
132
|
+
// keep the author's stated precision — refresh must not turn 0.155 into 0.1551724
|
|
133
|
+
const decimals = (String(m.value).split('.')[1] ?? '').length;
|
|
134
|
+
computed = Number(((after - before) / before).toFixed(decimals));
|
|
135
|
+
}
|
|
115
136
|
if (Number.isFinite(computed) && computed !== m.value) {
|
|
116
137
|
result.changes.push({ key, before: m.value, after: computed });
|
|
117
138
|
m.value = computed;
|
package/dist/render.js
CHANGED
|
@@ -12,13 +12,13 @@ export function formatValue(m) {
|
|
|
12
12
|
return `${m.value >= 0 ? '+' : ''}${(m.value * 100).toFixed(1)}%p`;
|
|
13
13
|
return `${m.value.toLocaleString()} ${m.unit}`.trim();
|
|
14
14
|
}
|
|
15
|
-
export function receipt(m) {
|
|
15
|
+
export function receipt(m, includeDefinition = true) {
|
|
16
16
|
const source = m.derived
|
|
17
17
|
? m.derived.op === 'sum'
|
|
18
18
|
? `= ${m.derived.of.join(' + ')} (recomputed)`
|
|
19
19
|
: `= ${m.derived.before} → ${m.derived.after} (recomputed)`
|
|
20
20
|
: [m.source?.type, ...Object.entries(m.source ?? {}).filter(([k]) => k !== 'type').map(([, v]) => String(v))].filter(Boolean).join(' · ');
|
|
21
|
-
return [source, m.window, m.fetched_at && `fetched ${m.fetched_at}`, m.definition].filter(Boolean).join(' · ');
|
|
21
|
+
return [source, m.window, m.fetched_at && `fetched ${m.fetched_at}`, includeDefinition && m.definition].filter(Boolean).join(' · ');
|
|
22
22
|
}
|
|
23
23
|
/** Minimal markdown: headings, paragraphs, bold, unordered lists. The narrative layer is deliberately thin. */
|
|
24
24
|
function markdown(src) {
|
package/dist/renderMd.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { formatValue, receipt } from './render.js';
|
|
2
|
+
const SUPERSCRIPT = '⁰¹²³⁴⁵⁶⁷⁸⁹';
|
|
3
|
+
const sup = (n) => `⁽${String(n).split('').map((d) => SUPERSCRIPT[Number(d)]).join('')}⁾`;
|
|
2
4
|
/**
|
|
3
5
|
* Grounded markdown: the portable render target. Every figure is bold with a superscript
|
|
4
6
|
* that points into a receipts appendix, so the output pastes into Notion, a PR body, or
|
|
@@ -11,7 +13,7 @@ export function renderMarkdown(report, ir) {
|
|
|
11
13
|
.replace(/\{\{m:([\w-]+)\}\}/g, (_, key) => {
|
|
12
14
|
if (!used.includes(key))
|
|
13
15
|
used.push(key);
|
|
14
|
-
return `**${formatValue(ir.metrics[key])}**
|
|
16
|
+
return `**${formatValue(ir.metrics[key])}** ${sup(used.indexOf(key) + 1)}`;
|
|
15
17
|
})
|
|
16
18
|
.replace(/\{\{id:([\w-]+)\}\}/g, (_, key) => `\`${ir.identifiers[key]}\``)
|
|
17
19
|
.replace(/\{\{claim:([^|}]*)\|\s*evidence:([^}]*)\}\}/g, (_, text, evidence) => `**${text.trim()}** *(evidence: ${evidence.trim()})*`)
|
|
@@ -20,8 +22,8 @@ export function renderMarkdown(report, ir) {
|
|
|
20
22
|
.map((key, i) => {
|
|
21
23
|
const m = ir.metrics[key];
|
|
22
24
|
const definition = m.definition ? ` — ${m.definition}` : '';
|
|
23
|
-
return `${i + 1}. **${key}** = ${formatValue(m)}${definition}\n ${receipt(m)}`;
|
|
25
|
+
return `${i + 1}. **${key}** = ${formatValue(m)}${definition}\n ${receipt(m, false)}`;
|
|
24
26
|
})
|
|
25
27
|
.join('\n');
|
|
26
|
-
return `${body}\n\n---\n\n### Receipts (${used.length} metrics)\n\n${appendix}\n`;
|
|
28
|
+
return `${body.trimEnd()}\n\n---\n\n### Receipts (${used.length} metrics)\n\n${appendix}\n`;
|
|
27
29
|
}
|
package/dist/scan.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
/** Same-length whitespace, newlines kept — stripping must not move anything */
|
|
2
|
+
const blank = (s) => s.replace(/[^\n]/g, ' ');
|
|
3
|
+
const lineAt = (text, index) => text.slice(0, index).split('\n').length;
|
|
1
4
|
/**
|
|
2
5
|
* Find numbers in the narrative that are not bound to the IR. These are the leaks the
|
|
3
6
|
* tool exists to catch: a typed, transcribed, or hallucinated figure reads exactly like
|
|
@@ -5,16 +8,19 @@
|
|
|
5
8
|
*/
|
|
6
9
|
export function scanNakedNumbers(report) {
|
|
7
10
|
const stripped = report
|
|
8
|
-
.replace(/\{\{(m|id):[\w-]+\}\}/g,
|
|
9
|
-
// a claim's prose stays scanned — only its syntax is
|
|
10
|
-
// smuggled into claim text is still a leak
|
|
11
|
-
.replace(/\{\{claim:([^|}]*)\|[^}]*\}\}/g, (
|
|
12
|
-
.
|
|
13
|
-
.
|
|
14
|
-
|
|
15
|
-
.replace(/\
|
|
16
|
-
.replace(
|
|
17
|
-
.replace(
|
|
11
|
+
.replace(/\{\{(m|id):[\w-]+\}\}/g, blank)
|
|
12
|
+
// a claim's prose stays scanned — only its syntax is blanked, so a number
|
|
13
|
+
// smuggled into claim text is still a leak, at its true position
|
|
14
|
+
.replace(/\{\{claim:([^|}]*)\|[^}]*\}\}/g, (whole, text) => {
|
|
15
|
+
const start = whole.indexOf(text);
|
|
16
|
+
return blank(whole.slice(0, start)) + text + blank(whole.slice(start + text.length));
|
|
17
|
+
})
|
|
18
|
+
.replace(/\{\{raw:[^}]*\}\}/g, blank) // explicit, greppable escape hatch
|
|
19
|
+
.replace(/```[\s\S]*?```/g, blank) // fenced code
|
|
20
|
+
.replace(/`[^`\n]*`/g, blank) // inline code
|
|
21
|
+
.replace(/\d{4}-\d{2}-\d{2}/g, blank) // ISO dates locate, they do not measure
|
|
22
|
+
.replace(/^#+ .*$/gm, blank) // headings
|
|
23
|
+
.replace(/^\s*\d+\.\s/gm, blank); // ordered-list markers
|
|
18
24
|
const leaks = [];
|
|
19
25
|
for (const m of stripped.matchAll(/\d[\d,.]*\s*(%p?|[가-힣]{1,2})?/g)) {
|
|
20
26
|
const token = m[0].trim();
|
|
@@ -23,6 +29,7 @@ export function scanNakedNumbers(report) {
|
|
|
23
29
|
leaks.push({
|
|
24
30
|
severity: 'error',
|
|
25
31
|
rule: 'naked-number',
|
|
32
|
+
line: lineAt(stripped, m.index),
|
|
26
33
|
message: `"${token}" appears in the narrative without a receipt`,
|
|
27
34
|
detail: 'Bind it to the IR as {{m:…}} or {{id:…}}, or mark deliberate prose as {{raw:…}}.',
|
|
28
35
|
});
|
|
@@ -32,39 +39,43 @@ export function scanNakedNumbers(report) {
|
|
|
32
39
|
/** Every reference in the narrative must resolve; a dangling one is authoring drift. */
|
|
33
40
|
export function scanRefs(report, metricKeys, idKeys) {
|
|
34
41
|
const leaks = [];
|
|
35
|
-
for (const
|
|
42
|
+
for (const m of report.matchAll(/\{\{claim:([^|}]*)\|\s*evidence:([^}]*)\}\}/g)) {
|
|
43
|
+
const [, text, evidence] = m;
|
|
44
|
+
const line = lineAt(report, m.index);
|
|
36
45
|
const keys = evidence.split(',').map((k) => k.trim()).filter(Boolean);
|
|
37
46
|
if (keys.length === 0) {
|
|
38
47
|
leaks.push({
|
|
39
48
|
severity: 'error',
|
|
40
49
|
rule: 'claim-without-evidence',
|
|
50
|
+
line,
|
|
41
51
|
message: `claim "${text.trim()}" names no evidence`,
|
|
42
52
|
detail: 'A conclusion must point at the metrics it rests on. Whether they support it stays a human judgement — but they must be attached.',
|
|
43
53
|
});
|
|
44
54
|
}
|
|
45
55
|
for (const key of keys) {
|
|
46
56
|
if (!metricKeys.has(key)) {
|
|
47
|
-
leaks.push({ severity: 'error', rule: 'unknown-ref', message: `claim "${text.trim()}" cites unknown metric "${key}"` });
|
|
57
|
+
leaks.push({ severity: 'error', rule: 'unknown-ref', line, message: `claim "${text.trim()}" cites unknown metric "${key}"` });
|
|
48
58
|
}
|
|
49
59
|
}
|
|
50
60
|
}
|
|
51
61
|
// a claim marker missing its evidence clause entirely is malformed, not exempt
|
|
52
|
-
for (const
|
|
62
|
+
for (const m of report.matchAll(/\{\{claim:([^|}]*)\}\}/g)) {
|
|
53
63
|
leaks.push({
|
|
54
64
|
severity: 'error',
|
|
55
65
|
rule: 'claim-without-evidence',
|
|
56
|
-
|
|
66
|
+
line: lineAt(report, m.index),
|
|
67
|
+
message: `claim "${m[1].trim()}" names no evidence`,
|
|
57
68
|
detail: 'Write it as {{claim: … | evidence: metric_key}}.',
|
|
58
69
|
});
|
|
59
70
|
}
|
|
60
|
-
for (const
|
|
61
|
-
if (!metricKeys.has(
|
|
62
|
-
leaks.push({ severity: 'error', rule: 'unknown-ref', message: `{{m:${
|
|
71
|
+
for (const m of report.matchAll(/\{\{m:([\w-]+)\}\}/g)) {
|
|
72
|
+
if (!metricKeys.has(m[1])) {
|
|
73
|
+
leaks.push({ severity: 'error', rule: 'unknown-ref', line: lineAt(report, m.index), message: `{{m:${m[1]}}} is not in the IR` });
|
|
63
74
|
}
|
|
64
75
|
}
|
|
65
|
-
for (const
|
|
66
|
-
if (!idKeys.has(
|
|
67
|
-
leaks.push({ severity: 'error', rule: 'unknown-ref', message: `{{id:${
|
|
76
|
+
for (const m of report.matchAll(/\{\{id:([\w-]+)\}\}/g)) {
|
|
77
|
+
if (!idKeys.has(m[1])) {
|
|
78
|
+
leaks.push({ severity: 'error', rule: 'unknown-ref', line: lineAt(report, m.index), message: `{{id:${m[1]}}} is not in the IR` });
|
|
68
79
|
}
|
|
69
80
|
}
|
|
70
81
|
return leaks;
|
package/package.json
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "watertight",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Reports that hold water
|
|
3
|
+
"version": "0.5.0",
|
|
4
|
+
"description": "Reports that hold water — every number carries its receipt, and ungrounded claims fail the build.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
|
-
"watertight": "
|
|
7
|
+
"watertight": "dist/cli.js"
|
|
8
8
|
},
|
|
9
9
|
"files": [
|
|
10
10
|
"dist",
|