stilltrue 0.0.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -3
- package/dist/cli.js +106 -15
- package/dist/compare.d.ts +2 -0
- package/dist/compare.js +57 -0
- package/dist/config.d.ts +38 -28
- package/dist/config.js +6 -5
- package/dist/helpers.d.ts +28 -0
- package/dist/helpers.js +69 -0
- package/dist/history.d.ts +11 -0
- package/dist/history.js +37 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +4 -1
- package/dist/loadConfig.d.ts +6 -0
- package/dist/loadConfig.js +32 -0
- package/dist/report.d.ts +2 -0
- package/dist/report.js +120 -0
- package/dist/runner.d.ts +9 -0
- package/dist/runner.js +45 -0
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -32,11 +32,47 @@ Drift has three outcomes, never two: `pass`, `rot` (facts changed — fail the
|
|
|
32
32
|
build), and `error` (source unreachable — warn, don't fail). Conflating the
|
|
33
33
|
last two causes alarm fatigue.
|
|
34
34
|
|
|
35
|
+
## Quick start
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
// stilltrue.config.ts (or .mjs / .js)
|
|
39
|
+
import { defineStilltrue, corpus, json, surname } from 'stilltrue';
|
|
40
|
+
|
|
41
|
+
export default defineStilltrue({
|
|
42
|
+
drift: [
|
|
43
|
+
{
|
|
44
|
+
name: 'school-board-roster',
|
|
45
|
+
source: corpus(['https://district.example.org/board']),
|
|
46
|
+
expect: json('./data/school-board.json', (d) => d.members.map((m) => surname(m.name))),
|
|
47
|
+
compare: 'contains-all',
|
|
48
|
+
},
|
|
49
|
+
],
|
|
50
|
+
});
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
```yaml
|
|
54
|
+
# .github/workflows/drift.yml
|
|
55
|
+
on:
|
|
56
|
+
schedule: [{ cron: '17 14 * * 3' }] # off-beat minute: GitHub drops :00 crons
|
|
57
|
+
jobs:
|
|
58
|
+
drift:
|
|
59
|
+
runs-on: ubuntu-latest
|
|
60
|
+
steps:
|
|
61
|
+
- uses: actions/checkout@v4
|
|
62
|
+
- uses: actions/setup-node@v4
|
|
63
|
+
with: { node-version: '22' }
|
|
64
|
+
- run: npm ci
|
|
65
|
+
- run: npx stilltrue drift
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`source` is your code (fetch+parse, or LLM extraction — the comparison stays
|
|
69
|
+
deterministic either way). `expect` is your curated file. Rot fails the run
|
|
70
|
+
with one message per vanished fact; unreachable sources only warn.
|
|
71
|
+
|
|
35
72
|
## Status
|
|
36
73
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
the drift runner, then swapping AskGwinnett's bespoke GitHub Actions to it.
|
|
74
|
+
v0.1 — `drift` is real and running in production (AskGwinnett). `golden` and
|
|
75
|
+
`verify` are designed but not yet shipped; see the brief.
|
|
40
76
|
|
|
41
77
|
## License
|
|
42
78
|
|
package/dist/cli.js
CHANGED
|
@@ -1,19 +1,110 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
|
-
* npx stilltrue drift
|
|
4
|
-
*
|
|
5
|
-
*
|
|
3
|
+
* npx stilltrue drift [--config <path>] [--only <name,name>] [--json <path>]
|
|
4
|
+
*
|
|
5
|
+
* Exit codes: 0 = all pass (errors warn but never fail — a judge/source
|
|
6
|
+
* outage must not block CI), 1 = rot detected or usage error.
|
|
6
7
|
*/
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
8
|
+
import { writeFile } from 'node:fs/promises';
|
|
9
|
+
import { resolve } from 'node:path';
|
|
10
|
+
import { parseArgs } from 'node:util';
|
|
11
|
+
import { findConfigPath, loadConfig } from "./loadConfig.js";
|
|
12
|
+
import { runDrift } from "./runner.js";
|
|
13
|
+
import { DEFAULT_HISTORY_PATH, loadHistory, recordRun } from "./history.js";
|
|
14
|
+
import { renderReport } from "./report.js";
|
|
15
|
+
const GH = !!process.env.GITHUB_ACTIONS;
|
|
16
|
+
function printResult(r) {
|
|
17
|
+
const icon = { pass: '✓', rot: '✗', error: '⚠' }[r.outcome];
|
|
18
|
+
console.log(`${icon} ${r.name} — ${r.outcome} (${r.durationMs}ms)`);
|
|
19
|
+
for (const w of r.warnings) {
|
|
20
|
+
console.log(` warning: ${w}`);
|
|
21
|
+
if (GH)
|
|
22
|
+
console.log(`::warning::${r.name}: ${w}`);
|
|
23
|
+
}
|
|
24
|
+
for (const m of r.messages) {
|
|
25
|
+
console.log(` ${m}`);
|
|
26
|
+
if (GH)
|
|
27
|
+
console.log(`::${r.outcome === 'rot' ? 'error' : 'warning'}::${r.name}: ${m}`);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
async function drift(argv) {
|
|
31
|
+
const { values } = parseArgs({
|
|
32
|
+
args: argv,
|
|
33
|
+
options: {
|
|
34
|
+
config: { type: 'string' },
|
|
35
|
+
only: { type: 'string' },
|
|
36
|
+
json: { type: 'string' },
|
|
37
|
+
record: { type: 'boolean' },
|
|
38
|
+
history: { type: 'string' },
|
|
39
|
+
},
|
|
40
|
+
});
|
|
41
|
+
const configPath = findConfigPath(process.cwd(), values.config);
|
|
42
|
+
const { config, configDir } = await loadConfig(configPath);
|
|
43
|
+
if (!config.drift?.length) {
|
|
44
|
+
console.error('no drift checks configured');
|
|
45
|
+
return 1;
|
|
46
|
+
}
|
|
47
|
+
const results = await runDrift(config, {
|
|
48
|
+
configDir,
|
|
49
|
+
only: values.only ? values.only.split(',').map((s) => s.trim()) : undefined,
|
|
50
|
+
});
|
|
51
|
+
for (const r of results)
|
|
52
|
+
printResult(r);
|
|
53
|
+
const rotted = results.filter((r) => r.outcome === 'rot').length;
|
|
54
|
+
const errored = results.filter((r) => r.outcome === 'error').length;
|
|
55
|
+
const passed = results.filter((r) => r.outcome === 'pass').length;
|
|
56
|
+
console.log(`\n${passed} pass, ${rotted} rot, ${errored} error (errors warn, never fail)`);
|
|
57
|
+
if (values.json) {
|
|
58
|
+
await writeFile(values.json, JSON.stringify(results, null, 2));
|
|
59
|
+
console.log(`results written to ${values.json}`);
|
|
60
|
+
}
|
|
61
|
+
if (values.record) {
|
|
62
|
+
const historyPath = resolve(values.history ?? DEFAULT_HISTORY_PATH);
|
|
63
|
+
await recordRun(results, historyPath);
|
|
64
|
+
console.log(`run recorded to ${historyPath}`);
|
|
65
|
+
}
|
|
66
|
+
return rotted > 0 ? 1 : 0;
|
|
67
|
+
}
|
|
68
|
+
async function report(argv) {
|
|
69
|
+
const { values } = parseArgs({
|
|
70
|
+
args: argv,
|
|
71
|
+
options: {
|
|
72
|
+
history: { type: 'string' },
|
|
73
|
+
out: { type: 'string' },
|
|
74
|
+
title: { type: 'string' },
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
const historyPath = resolve(values.history ?? DEFAULT_HISTORY_PATH);
|
|
78
|
+
const history = await loadHistory(historyPath);
|
|
79
|
+
if (!history.length) {
|
|
80
|
+
console.error(`no recorded runs at ${historyPath} — run \`stilltrue drift --record\` first (use --history to point elsewhere)`);
|
|
81
|
+
return 1;
|
|
82
|
+
}
|
|
83
|
+
const out = resolve(values.out ?? 'stilltrue-report.html');
|
|
84
|
+
await writeFile(out, renderReport(history, values.title));
|
|
85
|
+
console.log(`report written to ${out} (${history.length} runs)`);
|
|
86
|
+
return 0;
|
|
87
|
+
}
|
|
88
|
+
const [command, ...rest] = process.argv.slice(2);
|
|
89
|
+
try {
|
|
90
|
+
switch (command) {
|
|
91
|
+
case 'drift':
|
|
92
|
+
process.exit(await drift(rest));
|
|
93
|
+
break;
|
|
94
|
+
case 'report':
|
|
95
|
+
process.exit(await report(rest));
|
|
96
|
+
break;
|
|
97
|
+
case 'golden':
|
|
98
|
+
console.error('stilltrue golden: not implemented yet — drift shipped first (docs/BRIEF.md)');
|
|
99
|
+
process.exit(1);
|
|
100
|
+
break;
|
|
101
|
+
default:
|
|
102
|
+
console.log('usage: stilltrue drift [--config <path>] [--only <names>] [--json <path>] [--record] [--history <path>]\n' +
|
|
103
|
+
' stilltrue report [--history <path>] [--out <path>] [--title <text>]');
|
|
104
|
+
process.exit(command ? 1 : 0);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
console.error(`stilltrue: ${err instanceof Error ? err.message : String(err)}`);
|
|
109
|
+
process.exit(1);
|
|
18
110
|
}
|
|
19
|
-
export {};
|
package/dist/compare.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 'contains-all': expected is a list of marker strings, actual is a text
|
|
3
|
+
* corpus. Case-insensitive. The drift signal used by the original
|
|
4
|
+
* school-board check: a curated name/fact vanishing from the live pages.
|
|
5
|
+
*/
|
|
6
|
+
function containsAll(expected, actual) {
|
|
7
|
+
if (!Array.isArray(expected)) {
|
|
8
|
+
throw new TypeError(`contains-all: expected must be string[], got ${typeof expected}`);
|
|
9
|
+
}
|
|
10
|
+
if (typeof actual !== 'string') {
|
|
11
|
+
throw new TypeError(`contains-all: actual must be a string corpus, got ${typeof actual}`);
|
|
12
|
+
}
|
|
13
|
+
const corpus = actual.toLowerCase();
|
|
14
|
+
return expected
|
|
15
|
+
.filter((marker) => !corpus.includes(String(marker).toLowerCase()))
|
|
16
|
+
.map((marker) => `marker no longer present in source: "${marker}"`);
|
|
17
|
+
}
|
|
18
|
+
function isPlainObject(v) {
|
|
19
|
+
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
20
|
+
}
|
|
21
|
+
/** Structural deep-equal with pathed mismatch messages. */
|
|
22
|
+
function deepEqual(expected, actual, path = '$') {
|
|
23
|
+
if (Array.isArray(expected) && Array.isArray(actual)) {
|
|
24
|
+
const messages = [];
|
|
25
|
+
if (expected.length !== actual.length) {
|
|
26
|
+
messages.push(`${path}: length ${expected.length} → ${actual.length}`);
|
|
27
|
+
}
|
|
28
|
+
const len = Math.min(expected.length, actual.length);
|
|
29
|
+
for (let i = 0; i < len; i++) {
|
|
30
|
+
messages.push(...deepEqual(expected[i], actual[i], `${path}[${i}]`));
|
|
31
|
+
}
|
|
32
|
+
return messages;
|
|
33
|
+
}
|
|
34
|
+
if (isPlainObject(expected) && isPlainObject(actual)) {
|
|
35
|
+
const messages = [];
|
|
36
|
+
for (const key of new Set([...Object.keys(expected), ...Object.keys(actual)])) {
|
|
37
|
+
if (!(key in actual))
|
|
38
|
+
messages.push(`${path}.${key}: missing from source`);
|
|
39
|
+
else if (!(key in expected))
|
|
40
|
+
messages.push(`${path}.${key}: unexpected new value ${JSON.stringify(actual[key])}`);
|
|
41
|
+
else
|
|
42
|
+
messages.push(...deepEqual(expected[key], actual[key], `${path}.${key}`));
|
|
43
|
+
}
|
|
44
|
+
return messages;
|
|
45
|
+
}
|
|
46
|
+
if (expected !== actual) {
|
|
47
|
+
return [`${path}: expected ${JSON.stringify(expected)}, source has ${JSON.stringify(actual)}`];
|
|
48
|
+
}
|
|
49
|
+
return [];
|
|
50
|
+
}
|
|
51
|
+
export function resolveCompare(compare) {
|
|
52
|
+
if (compare === undefined || compare === 'deep-equal')
|
|
53
|
+
return (e, a) => deepEqual(e, a);
|
|
54
|
+
if (compare === 'contains-all')
|
|
55
|
+
return (e, a) => containsAll(e, a);
|
|
56
|
+
return compare;
|
|
57
|
+
}
|
package/dist/config.d.ts
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The config seam — the
|
|
3
|
-
* the
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
2
|
+
* The config seam — the package owns the *how*, the consumer's config owns
|
|
3
|
+
* the *what*. Designed against the real production check it replaces
|
|
4
|
+
* (bfoster-services scripts/check-school-board-drift.js): fetch pages into a
|
|
5
|
+
* text corpus, assert curated markers still appear, tolerate partial fetch
|
|
6
|
+
* failures with warnings, and never conflate "facts changed" with "source
|
|
7
|
+
* unreachable".
|
|
7
8
|
*/
|
|
8
|
-
/**
|
|
9
|
+
/** Three outcomes, never two (docs/BRIEF.md, First session §4). */
|
|
9
10
|
export type DriftOutcome =
|
|
10
|
-
/** Live source still matches the
|
|
11
|
+
/** Live source still matches the curated facts. */
|
|
11
12
|
'pass'
|
|
12
13
|
/** Facts no longer match — fail the build. A human fixes. */
|
|
13
14
|
| 'rot'
|
|
@@ -16,36 +17,45 @@ export type DriftOutcome =
|
|
|
16
17
|
export interface DriftResult {
|
|
17
18
|
name: string;
|
|
18
19
|
outcome: DriftOutcome;
|
|
19
|
-
/**
|
|
20
|
+
/** One issue per entry: the mismatches for `rot`, the cause for `error`. */
|
|
20
21
|
messages: string[];
|
|
21
|
-
/**
|
|
22
|
-
|
|
23
|
-
expected: unknown;
|
|
24
|
-
actual: unknown;
|
|
25
|
-
};
|
|
22
|
+
/** Non-fatal notes (e.g. one of several pages unreachable). */
|
|
23
|
+
warnings: string[];
|
|
26
24
|
checkedAt: string;
|
|
25
|
+
durationMs: number;
|
|
26
|
+
}
|
|
27
|
+
/** Passed to consumer `source`/`expect` functions. */
|
|
28
|
+
export interface CheckContext {
|
|
29
|
+
/** Record a non-fatal problem (partial fetch failure, odd markup, …). */
|
|
30
|
+
warn: (message: string) => void;
|
|
31
|
+
/** Absolute directory of the loaded config file — resolve data paths from here. */
|
|
32
|
+
configDir: string;
|
|
27
33
|
}
|
|
28
34
|
/**
|
|
29
|
-
*
|
|
35
|
+
* Deterministic comparison: returns mismatch messages, [] = pass.
|
|
36
|
+
* Built-ins: 'contains-all' (expected: string[] markers, actual: corpus
|
|
37
|
+
* string, case-insensitive) and 'deep-equal'.
|
|
38
|
+
*/
|
|
39
|
+
export type CompareFn<E, A> = (expected: E, actual: A) => string[];
|
|
40
|
+
export type Compare<E, A> = 'contains-all' | 'deep-equal' | CompareFn<E, A>;
|
|
41
|
+
/**
|
|
42
|
+
* One curated fact-set vs its live authoritative source.
|
|
30
43
|
*
|
|
31
|
-
* `source()` is
|
|
32
|
-
*
|
|
33
|
-
*
|
|
44
|
+
* `source()` is consumer code — deterministic fetch+parse or LLM extraction,
|
|
45
|
+
* the package neither knows nor cares; a throw means outcome `error`, never
|
|
46
|
+
* `rot`. The comparison is always deterministic.
|
|
34
47
|
*/
|
|
35
|
-
export interface DriftCheck<
|
|
48
|
+
export interface DriftCheck<E = unknown, A = unknown> {
|
|
36
49
|
name: string;
|
|
37
|
-
/** Fetch the live source
|
|
38
|
-
source: () => Promise<
|
|
39
|
-
/**
|
|
40
|
-
expect:
|
|
41
|
-
/**
|
|
42
|
-
|
|
43
|
-
* custom deterministic function returning mismatch messages ([] = pass).
|
|
44
|
-
*/
|
|
45
|
-
compare?: string | ((expected: T, actual: T) => string[]);
|
|
50
|
+
/** Fetch the live source. Throw => `error`. */
|
|
51
|
+
source: (ctx: CheckContext) => Promise<A>;
|
|
52
|
+
/** The curated expectation: a value, or a function deriving it (e.g. from a data file). */
|
|
53
|
+
expect: E | ((ctx: CheckContext) => E | Promise<E>);
|
|
54
|
+
/** Defaults to 'deep-equal'. */
|
|
55
|
+
compare?: Compare<E, A>;
|
|
46
56
|
}
|
|
47
57
|
export interface StilltrueConfig {
|
|
48
|
-
drift?: DriftCheck<any>[];
|
|
58
|
+
drift?: DriftCheck<any, any>[];
|
|
49
59
|
/** golden and verify are later phases — see docs/BRIEF.md build plan. */
|
|
50
60
|
golden?: unknown;
|
|
51
61
|
verify?: unknown;
|
package/dist/config.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The config seam — the
|
|
3
|
-
* the
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
2
|
+
* The config seam — the package owns the *how*, the consumer's config owns
|
|
3
|
+
* the *what*. Designed against the real production check it replaces
|
|
4
|
+
* (bfoster-services scripts/check-school-board-drift.js): fetch pages into a
|
|
5
|
+
* text corpus, assert curated markers still appear, tolerate partial fetch
|
|
6
|
+
* failures with warnings, and never conflate "facts changed" with "source
|
|
7
|
+
* unreachable".
|
|
7
8
|
*/
|
|
8
9
|
/** Identity function for typed configs: `export default defineStilltrue({...})`. */
|
|
9
10
|
export function defineStilltrue(config) {
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { CheckContext } from './config.ts';
|
|
2
|
+
export interface CorpusOptions {
|
|
3
|
+
/** Per-page fetch timeout, ms. Default 20000. */
|
|
4
|
+
timeoutMs?: number;
|
|
5
|
+
/** Sent as User-Agent so site operators can identify the check. */
|
|
6
|
+
userAgent?: string;
|
|
7
|
+
}
|
|
8
|
+
/** Strip a page to comparable text (same normalization as the original check). */
|
|
9
|
+
export declare function htmlToText(html: string): string;
|
|
10
|
+
/**
|
|
11
|
+
* Source helper: fetch pages and concatenate them into one text corpus.
|
|
12
|
+
* Individual unreachable pages warn; ALL pages unreachable throws, which the
|
|
13
|
+
* runner reports as `error` (source down ≠ facts rotted).
|
|
14
|
+
*/
|
|
15
|
+
export declare function corpus(urls: string[], options?: CorpusOptions): (ctx: CheckContext) => Promise<string>;
|
|
16
|
+
/**
|
|
17
|
+
* Expect helper: read a curated JSON data file (path relative to the config
|
|
18
|
+
* file) and optionally derive the comparable expectation from it — e.g. the
|
|
19
|
+
* marker list for 'contains-all'.
|
|
20
|
+
*/
|
|
21
|
+
export declare function json<T = any, E = T>(path: string, derive?: (data: T) => E): (ctx: CheckContext) => Promise<E>;
|
|
22
|
+
/**
|
|
23
|
+
* Surname as the drift signal — titles and first names vary in page copy
|
|
24
|
+
* ("Dr. Jane Smith" vs "Superintendent Smith"); the surname is stable.
|
|
25
|
+
* Leading honorifics and trailing generational suffixes are stripped
|
|
26
|
+
* ("Jasper Watkins III" → "Watkins").
|
|
27
|
+
*/
|
|
28
|
+
export declare function surname(fullName: string): string;
|
package/dist/helpers.js
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
/** Strip a page to comparable text (same normalization as the original check). */
|
|
4
|
+
export function htmlToText(html) {
|
|
5
|
+
return html
|
|
6
|
+
.replace(/<script[\s\S]*?<\/script>/gi, ' ')
|
|
7
|
+
.replace(/<style[\s\S]*?<\/style>/gi, ' ')
|
|
8
|
+
.replace(/<[^>]+>/g, ' ')
|
|
9
|
+
.replace(/&/g, '&')
|
|
10
|
+
.replace(/ /g, ' ')
|
|
11
|
+
.replace(/\s+/g, ' ');
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Source helper: fetch pages and concatenate them into one text corpus.
|
|
15
|
+
* Individual unreachable pages warn; ALL pages unreachable throws, which the
|
|
16
|
+
* runner reports as `error` (source down ≠ facts rotted).
|
|
17
|
+
*/
|
|
18
|
+
export function corpus(urls, options = {}) {
|
|
19
|
+
const { timeoutMs = 20_000, userAgent = 'stilltrue drift check (github.com/sg-bfoster/stilltrue)' } = options;
|
|
20
|
+
return async (ctx) => {
|
|
21
|
+
let text = '';
|
|
22
|
+
const unreachable = [];
|
|
23
|
+
for (const url of urls) {
|
|
24
|
+
try {
|
|
25
|
+
const res = await fetch(url, {
|
|
26
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
27
|
+
headers: { 'User-Agent': userAgent },
|
|
28
|
+
});
|
|
29
|
+
if (!res.ok)
|
|
30
|
+
throw new Error(`HTTP ${res.status}`);
|
|
31
|
+
text += ` ${htmlToText(await res.text())}`;
|
|
32
|
+
}
|
|
33
|
+
catch (err) {
|
|
34
|
+
unreachable.push(`${url} (${err instanceof Error ? err.message : String(err)})`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
if (!text.trim()) {
|
|
38
|
+
throw new Error(`all pages unreachable: ${unreachable.join('; ')}`);
|
|
39
|
+
}
|
|
40
|
+
for (const miss of unreachable)
|
|
41
|
+
ctx.warn(`page unreachable: ${miss}`);
|
|
42
|
+
return text;
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Expect helper: read a curated JSON data file (path relative to the config
|
|
47
|
+
* file) and optionally derive the comparable expectation from it — e.g. the
|
|
48
|
+
* marker list for 'contains-all'.
|
|
49
|
+
*/
|
|
50
|
+
export function json(path, derive) {
|
|
51
|
+
return async (ctx) => {
|
|
52
|
+
const data = JSON.parse(await readFile(resolve(ctx.configDir, path), 'utf8'));
|
|
53
|
+
return derive ? derive(data) : data;
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Surname as the drift signal — titles and first names vary in page copy
|
|
58
|
+
* ("Dr. Jane Smith" vs "Superintendent Smith"); the surname is stable.
|
|
59
|
+
* Leading honorifics and trailing generational suffixes are stripped
|
|
60
|
+
* ("Jasper Watkins III" → "Watkins").
|
|
61
|
+
*/
|
|
62
|
+
export function surname(fullName) {
|
|
63
|
+
const cleaned = String(fullName)
|
|
64
|
+
.replace(/^(dr|mr|mrs|ms)\.?\s+/i, '')
|
|
65
|
+
.replace(/,?\s+(jr|sr|ii|iii|iv|v)\.?$/i, '')
|
|
66
|
+
.trim();
|
|
67
|
+
const parts = cleaned.split(/\s+/);
|
|
68
|
+
return parts[parts.length - 1] ?? '';
|
|
69
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { DriftResult } from './config.ts';
|
|
2
|
+
/** One recorded `stilltrue drift` run. */
|
|
3
|
+
export interface RunRecord {
|
|
4
|
+
at: string;
|
|
5
|
+
results: DriftResult[];
|
|
6
|
+
}
|
|
7
|
+
export declare const DEFAULT_HISTORY_PATH = ".stilltrue/history.jsonl";
|
|
8
|
+
/** Append a run to the JSONL history file (created on first record). */
|
|
9
|
+
export declare function recordRun(results: DriftResult[], historyPath: string): Promise<RunRecord>;
|
|
10
|
+
/** Load history oldest-first. Missing file → []. Corrupt lines are skipped. */
|
|
11
|
+
export declare function loadHistory(historyPath: string): Promise<RunRecord[]>;
|
package/dist/history.js
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { mkdir, readFile, appendFile } from 'node:fs/promises';
|
|
2
|
+
import { dirname } from 'node:path';
|
|
3
|
+
export const DEFAULT_HISTORY_PATH = '.stilltrue/history.jsonl';
|
|
4
|
+
/** Append a run to the JSONL history file (created on first record). */
|
|
5
|
+
export async function recordRun(results, historyPath) {
|
|
6
|
+
const record = { at: new Date().toISOString(), results };
|
|
7
|
+
await mkdir(dirname(historyPath), { recursive: true });
|
|
8
|
+
await appendFile(historyPath, JSON.stringify(record) + '\n');
|
|
9
|
+
return record;
|
|
10
|
+
}
|
|
11
|
+
/** Load history oldest-first. Missing file → []. Corrupt lines are skipped. */
|
|
12
|
+
export async function loadHistory(historyPath) {
|
|
13
|
+
let raw;
|
|
14
|
+
try {
|
|
15
|
+
raw = await readFile(historyPath, 'utf8');
|
|
16
|
+
}
|
|
17
|
+
catch (err) {
|
|
18
|
+
if (err.code === 'ENOENT')
|
|
19
|
+
return [];
|
|
20
|
+
throw err;
|
|
21
|
+
}
|
|
22
|
+
const records = [];
|
|
23
|
+
for (const line of raw.split('\n')) {
|
|
24
|
+
if (!line.trim())
|
|
25
|
+
continue;
|
|
26
|
+
try {
|
|
27
|
+
const parsed = JSON.parse(line);
|
|
28
|
+
if (parsed && typeof parsed.at === 'string' && Array.isArray(parsed.results)) {
|
|
29
|
+
records.push(parsed);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
// A truncated line (interrupted CI write) shouldn't sink the report.
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return records;
|
|
37
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1 +1,4 @@
|
|
|
1
|
-
export { defineStilltrue, type StilltrueConfig, type DriftCheck, type DriftResult, type DriftOutcome, } from './config.
|
|
1
|
+
export { defineStilltrue, type StilltrueConfig, type DriftCheck, type DriftResult, type DriftOutcome, type CheckContext, type Compare, type CompareFn, } from './config.ts';
|
|
2
|
+
export { corpus, json, surname, htmlToText, type CorpusOptions } from './helpers.ts';
|
|
3
|
+
export { runDrift, runDriftCheck, type RunDriftOptions } from './runner.ts';
|
|
4
|
+
export { resolveCompare } from './compare.ts';
|
package/dist/index.js
CHANGED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { dirname, isAbsolute, join, resolve } from 'node:path';
|
|
3
|
+
import { createJiti } from 'jiti';
|
|
4
|
+
const CANDIDATES = [
|
|
5
|
+
'stilltrue.config.ts',
|
|
6
|
+
'stilltrue.config.mts',
|
|
7
|
+
'stilltrue.config.js',
|
|
8
|
+
'stilltrue.config.mjs',
|
|
9
|
+
'stilltrue.config.cjs',
|
|
10
|
+
];
|
|
11
|
+
export function findConfigPath(cwd, explicit) {
|
|
12
|
+
if (explicit) {
|
|
13
|
+
const p = isAbsolute(explicit) ? explicit : resolve(cwd, explicit);
|
|
14
|
+
if (!existsSync(p))
|
|
15
|
+
throw new Error(`config not found: ${p}`);
|
|
16
|
+
return p;
|
|
17
|
+
}
|
|
18
|
+
for (const name of CANDIDATES) {
|
|
19
|
+
const p = join(cwd, name);
|
|
20
|
+
if (existsSync(p))
|
|
21
|
+
return p;
|
|
22
|
+
}
|
|
23
|
+
throw new Error(`no stilltrue.config.{ts,js,mjs,cjs} found in ${cwd}`);
|
|
24
|
+
}
|
|
25
|
+
export async function loadConfig(configPath) {
|
|
26
|
+
const jiti = createJiti(import.meta.url, { interopDefault: true });
|
|
27
|
+
const config = (await jiti.import(configPath, { default: true }));
|
|
28
|
+
if (!config || typeof config !== 'object') {
|
|
29
|
+
throw new Error(`config at ${configPath} has no default export object`);
|
|
30
|
+
}
|
|
31
|
+
return { config, configDir: dirname(configPath) };
|
|
32
|
+
}
|
package/dist/report.d.ts
ADDED
package/dist/report.js
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
const STATUS = {
|
|
2
|
+
pass: { color: '#0ca30c', glyph: '✓', ink: '#ffffff', label: 'pass' },
|
|
3
|
+
rot: { color: '#d03b3b', glyph: '✗', ink: '#ffffff', label: 'rot' },
|
|
4
|
+
error: { color: '#fab219', glyph: '⚠', ink: '#1a1a19', label: 'error' },
|
|
5
|
+
};
|
|
6
|
+
const MAX_COLUMNS = 60;
|
|
7
|
+
const MAX_LOG = 50;
|
|
8
|
+
function esc(s) {
|
|
9
|
+
return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
10
|
+
}
|
|
11
|
+
function day(iso) {
|
|
12
|
+
return iso.slice(5, 10); // MM-DD
|
|
13
|
+
}
|
|
14
|
+
export function renderReport(history, title = 'stilltrue drift report') {
|
|
15
|
+
const runs = history.slice(-MAX_COLUMNS);
|
|
16
|
+
const latest = runs[runs.length - 1];
|
|
17
|
+
// Stable row order: latest run's order first, then any check seen earlier.
|
|
18
|
+
const names = [];
|
|
19
|
+
for (const record of [...runs].reverse()) {
|
|
20
|
+
for (const r of record.results)
|
|
21
|
+
if (!names.includes(r.name))
|
|
22
|
+
names.push(r.name);
|
|
23
|
+
}
|
|
24
|
+
const counts = { pass: 0, rot: 0, error: 0 };
|
|
25
|
+
for (const r of latest?.results ?? [])
|
|
26
|
+
counts[r.outcome]++;
|
|
27
|
+
const tiles = ['pass', 'rot', 'error']
|
|
28
|
+
.map((o) => `<div class="tile">
|
|
29
|
+
<div class="tile-num" style="color:${counts[o] && o !== 'pass' ? STATUS[o].color : 'var(--ink)'}">${counts[o]}</div>
|
|
30
|
+
<div class="tile-label">${STATUS[o].glyph} ${STATUS[o].label}</div>
|
|
31
|
+
</div>`)
|
|
32
|
+
.join('');
|
|
33
|
+
const headerCells = runs
|
|
34
|
+
.map((rec, i) => {
|
|
35
|
+
const sparse = runs.length <= 10 || i % Math.ceil(runs.length / 10) === 0 || i === runs.length - 1;
|
|
36
|
+
return `<th title="${esc(rec.at)}">${sparse ? day(rec.at) : ''}</th>`;
|
|
37
|
+
})
|
|
38
|
+
.join('');
|
|
39
|
+
const rows = names
|
|
40
|
+
.map((name) => {
|
|
41
|
+
const cells = runs
|
|
42
|
+
.map((rec) => {
|
|
43
|
+
const r = rec.results.find((x) => x.name === name);
|
|
44
|
+
if (!r)
|
|
45
|
+
return '<td><span class="cell none" title="not run"></span></td>';
|
|
46
|
+
const s = STATUS[r.outcome];
|
|
47
|
+
const detail = [r.outcome, ...r.messages, ...r.warnings.map((w) => `warning: ${w}`)].join('\n');
|
|
48
|
+
return `<td><span class="cell" style="background:${s.color};color:${s.ink}" title="${esc(rec.at)}\n${esc(detail)}">${s.glyph}</span></td>`;
|
|
49
|
+
})
|
|
50
|
+
.join('');
|
|
51
|
+
return `<tr><th class="check-name">${esc(name)}</th>${cells}</tr>`;
|
|
52
|
+
})
|
|
53
|
+
.join('');
|
|
54
|
+
const events = [...runs]
|
|
55
|
+
.reverse()
|
|
56
|
+
.flatMap((rec) => rec.results
|
|
57
|
+
.filter((r) => r.outcome !== 'pass' || r.warnings.length)
|
|
58
|
+
.map((r) => ({ at: rec.at, ...r })))
|
|
59
|
+
.slice(0, MAX_LOG);
|
|
60
|
+
const log = events.length
|
|
61
|
+
? events
|
|
62
|
+
.map((e) => {
|
|
63
|
+
const s = STATUS[e.outcome];
|
|
64
|
+
const items = [...e.messages, ...e.warnings.map((w) => `warning: ${w}`)]
|
|
65
|
+
.map((m) => `<li>${esc(m)}</li>`)
|
|
66
|
+
.join('');
|
|
67
|
+
return `<details ${e.outcome === 'rot' ? 'open' : ''}>
|
|
68
|
+
<summary><span class="badge" style="background:${s.color};color:${s.ink}">${s.glyph} ${s.label}</span>
|
|
69
|
+
<strong>${esc(e.name)}</strong> <span class="muted">${esc(e.at)}</span></summary>
|
|
70
|
+
<ul>${items}</ul>
|
|
71
|
+
</details>`;
|
|
72
|
+
})
|
|
73
|
+
.join('')
|
|
74
|
+
: '<p class="muted">Nothing has rotted, errored, or warned in recorded history.</p>';
|
|
75
|
+
return `<!doctype html>
|
|
76
|
+
<meta charset="utf-8">
|
|
77
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
78
|
+
<title>${esc(title)}</title>
|
|
79
|
+
<style>
|
|
80
|
+
:root { --bg:#ffffff; --ink:#1a1a19; --ink-2:#5f5e5a; --line:#e4e2dd; --surface:#f6f5f2; }
|
|
81
|
+
@media (prefers-color-scheme: dark) {
|
|
82
|
+
:root { --bg:#1a1a19; --ink:#f2f1ee; --ink-2:#a5a39d; --line:#3a3936; --surface:#242422; }
|
|
83
|
+
}
|
|
84
|
+
* { box-sizing: border-box; }
|
|
85
|
+
body { margin:0; padding:2rem 1.5rem; background:var(--bg); color:var(--ink);
|
|
86
|
+
font:15px/1.5 ui-sans-serif, system-ui, sans-serif; max-width:64rem; margin-inline:auto; }
|
|
87
|
+
h1 { font-size:1.3rem; margin:0 0 .25rem; }
|
|
88
|
+
h2 { font-size:1rem; margin:2rem 0 .75rem; }
|
|
89
|
+
.muted { color:var(--ink-2); font-size:.85rem; }
|
|
90
|
+
.tiles { display:flex; gap:.75rem; margin:1.25rem 0; flex-wrap:wrap; }
|
|
91
|
+
.tile { background:var(--surface); border:1px solid var(--line); border-radius:8px;
|
|
92
|
+
padding:.75rem 1.25rem; min-width:6.5rem; }
|
|
93
|
+
.tile-num { font-size:1.6rem; font-weight:600; font-variant-numeric:tabular-nums; }
|
|
94
|
+
.tile-label { color:var(--ink-2); font-size:.85rem; }
|
|
95
|
+
.grid-wrap { overflow-x:auto; border:1px solid var(--line); border-radius:8px; padding:.75rem; }
|
|
96
|
+
table { border-collapse:separate; border-spacing:2px; }
|
|
97
|
+
th { font-weight:500; color:var(--ink-2); font-size:.7rem; text-align:left; white-space:nowrap; }
|
|
98
|
+
thead th { vertical-align:bottom; padding-bottom:.25rem; }
|
|
99
|
+
.check-name { font-size:.85rem; color:var(--ink); padding-right:.75rem; position:sticky; left:0; background:var(--bg); }
|
|
100
|
+
.cell { display:inline-flex; width:20px; height:20px; border-radius:4px;
|
|
101
|
+
align-items:center; justify-content:center; font-size:12px; }
|
|
102
|
+
.cell.none { background:var(--surface); border:1px dashed var(--line); }
|
|
103
|
+
.badge { display:inline-block; border-radius:4px; padding:0 .45rem; font-size:.8rem; margin-right:.5rem; }
|
|
104
|
+
details { border-left:2px solid var(--line); padding:.35rem .75rem; margin:.4rem 0; }
|
|
105
|
+
summary { cursor:pointer; }
|
|
106
|
+
details ul { margin:.4rem 0 .2rem; padding-left:1.2rem; }
|
|
107
|
+
details li { font-size:.9rem; }
|
|
108
|
+
</style>
|
|
109
|
+
<h1>${esc(title)}</h1>
|
|
110
|
+
<p class="muted">${runs.length} recorded run${runs.length === 1 ? '' : 's'}${latest ? `, latest ${esc(latest.at)}` : ''} — pass / rot (facts changed) / error (source unreachable)</p>
|
|
111
|
+
<div class="tiles">${tiles}</div>
|
|
112
|
+
<h2>Timeline</h2>
|
|
113
|
+
<div class="grid-wrap"><table>
|
|
114
|
+
<thead><tr><th></th>${headerCells}</tr></thead>
|
|
115
|
+
<tbody>${rows}</tbody>
|
|
116
|
+
</table></div>
|
|
117
|
+
<h2>Rot, errors & warnings</h2>
|
|
118
|
+
${log}
|
|
119
|
+
`;
|
|
120
|
+
}
|
package/dist/runner.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { DriftCheck, DriftResult, StilltrueConfig } from './config.ts';
|
|
2
|
+
export interface RunDriftOptions {
|
|
3
|
+
/** Run only checks whose name is in this list. */
|
|
4
|
+
only?: string[];
|
|
5
|
+
/** Directory the config file was loaded from (data paths resolve here). */
|
|
6
|
+
configDir: string;
|
|
7
|
+
}
|
|
8
|
+
export declare function runDriftCheck(check: DriftCheck<any, any>, configDir: string): Promise<DriftResult>;
|
|
9
|
+
export declare function runDrift(config: StilltrueConfig, options: RunDriftOptions): Promise<DriftResult[]>;
|
package/dist/runner.js
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { resolveCompare } from "./compare.js";
|
|
2
|
+
export async function runDriftCheck(check, configDir) {
|
|
3
|
+
const warnings = [];
|
|
4
|
+
const ctx = { warn: (m) => warnings.push(m), configDir };
|
|
5
|
+
const started = Date.now();
|
|
6
|
+
const done = (outcome, messages) => ({
|
|
7
|
+
name: check.name,
|
|
8
|
+
outcome,
|
|
9
|
+
messages,
|
|
10
|
+
warnings,
|
|
11
|
+
checkedAt: new Date(started).toISOString(),
|
|
12
|
+
durationMs: Date.now() - started,
|
|
13
|
+
});
|
|
14
|
+
let expected;
|
|
15
|
+
try {
|
|
16
|
+
expected = typeof check.expect === 'function' ? await check.expect(ctx) : check.expect;
|
|
17
|
+
}
|
|
18
|
+
catch (err) {
|
|
19
|
+
// A broken expectation is consumer misconfiguration, not source rot or
|
|
20
|
+
// source outage — surface it as `error` so CI warns loudly.
|
|
21
|
+
return done('error', [`expect failed: ${err instanceof Error ? err.message : String(err)}`]);
|
|
22
|
+
}
|
|
23
|
+
let actual;
|
|
24
|
+
try {
|
|
25
|
+
actual = await check.source(ctx);
|
|
26
|
+
}
|
|
27
|
+
catch (err) {
|
|
28
|
+
return done('error', [`source failed: ${err instanceof Error ? err.message : String(err)}`]);
|
|
29
|
+
}
|
|
30
|
+
try {
|
|
31
|
+
const mismatches = resolveCompare(check.compare)(expected, actual);
|
|
32
|
+
return mismatches.length ? done('rot', mismatches) : done('pass', []);
|
|
33
|
+
}
|
|
34
|
+
catch (err) {
|
|
35
|
+
return done('error', [`compare failed: ${err instanceof Error ? err.message : String(err)}`]);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export async function runDrift(config, options) {
|
|
39
|
+
const checks = (config.drift ?? []).filter((c) => !options.only || options.only.includes(c.name));
|
|
40
|
+
const results = [];
|
|
41
|
+
for (const check of checks) {
|
|
42
|
+
results.push(await runDriftCheck(check, options.configDir));
|
|
43
|
+
}
|
|
44
|
+
return results;
|
|
45
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "stilltrue",
|
|
3
|
-
"version": "0.0
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Your tests check your code. stilltrue checks your facts. CI checks for AI-in-the-loop apps: drift (curated facts vs live sources), golden (regression evals), verify (judge answers against sources).",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"drift",
|
|
@@ -25,9 +25,10 @@
|
|
|
25
25
|
"scripts": {
|
|
26
26
|
"build": "tsc",
|
|
27
27
|
"dev": "tsc --watch",
|
|
28
|
-
"test": "node --test
|
|
28
|
+
"test": "node --test test/*.test.ts"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
+
"jiti": "^2.7.0",
|
|
31
32
|
"zod": "^3.23.0"
|
|
32
33
|
},
|
|
33
34
|
"devDependencies": {
|