stilltrue 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +43 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +19 -0
- package/dist/config.d.ts +54 -0
- package/dist/config.js +11 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/package.json +40 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Brian Foster
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# stilltrue
|
|
2
|
+
|
|
3
|
+
> Your tests check your code. **stilltrue** checks your facts.
|
|
4
|
+
|
|
5
|
+
AI-in-the-loop apps quietly start lying three ways: code changes break old
|
|
6
|
+
answers, the world changes and curated facts rot, and the model embellishes
|
|
7
|
+
beyond its sources. stilltrue is one watchdog per failure mode, run from CI —
|
|
8
|
+
no server, no dashboard, no SaaS.
|
|
9
|
+
|
|
10
|
+
| Command | Watchdog | Status |
|
|
11
|
+
|---|---|---|
|
|
12
|
+
| `npx stilltrue drift` | Curated facts vs their live authoritative sources — fails the build on rot | **v0.1 focus** |
|
|
13
|
+
| `npx stilltrue golden` | Deterministic regression evals for answers | later (may wrap promptfoo) |
|
|
14
|
+
| `verify` (library) | Judge answers against their sources in the request path | later (P9-shaped staged pipeline) |
|
|
15
|
+
|
|
16
|
+
**Why drift first:** "drift detection" in the market means statistical ML
|
|
17
|
+
feature drift; website-change monitors alert humans, not CI. Nobody ships
|
|
18
|
+
*"assert my curated facts still match their live source, as a CI check."*
|
|
19
|
+
This was built bespoke three times across three domains before becoming a
|
|
20
|
+
package — see [docs/BRIEF.md](docs/BRIEF.md) for the full design brief,
|
|
21
|
+
positioning, and build plan.
|
|
22
|
+
|
|
23
|
+
## Core principle
|
|
24
|
+
|
|
25
|
+
**AI may extract, but must never decide.** The default drift path has zero AI:
|
|
26
|
+
fetch → parse → compare → diff. A consumer's `source()` may use an LLM to pull
|
|
27
|
+
structure out of a messy page, but the comparison and the verdict are always
|
|
28
|
+
deterministic. Same inputs, same verdict — a nondeterministic alarm trains
|
|
29
|
+
people to ignore it.
|
|
30
|
+
|
|
31
|
+
Drift has three outcomes, never two: `pass`, `rot` (facts changed — fail the
|
|
32
|
+
build), and `error` (source unreachable — warn, don't fail). Conflating the
|
|
33
|
+
last two causes alarm fatigue.
|
|
34
|
+
|
|
35
|
+
## Status
|
|
36
|
+
|
|
37
|
+
Pre-v0.1. Current state: config seam design (`src/config.ts`) validated
|
|
38
|
+
against two real production checks (`examples/stilltrue.config.ts`). Next:
|
|
39
|
+
the drift runner, then swapping AskGwinnett's bespoke GitHub Actions to it.
|
|
40
|
+
|
|
41
|
+
## License
|
|
42
|
+
|
|
43
|
+
MIT
|
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* npx stilltrue drift — run drift checks (v0.1 scope)
|
|
4
|
+
* npx stilltrue golden — regression evals (later)
|
|
5
|
+
* npx stilltrue report — HTML drift report (later)
|
|
6
|
+
*/
|
|
7
|
+
const [command] = process.argv.slice(2);
|
|
8
|
+
switch (command) {
|
|
9
|
+
case 'drift':
|
|
10
|
+
console.error('stilltrue drift: not implemented yet — see docs/BRIEF.md, First session §3');
|
|
11
|
+
process.exit(1);
|
|
12
|
+
case undefined:
|
|
13
|
+
console.log('usage: stilltrue <drift|golden|report>');
|
|
14
|
+
process.exit(1);
|
|
15
|
+
default:
|
|
16
|
+
console.error(`stilltrue: unknown command "${command}"`);
|
|
17
|
+
process.exit(1);
|
|
18
|
+
}
|
|
19
|
+
export {};
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The config seam — the entire product is "the package owns the *how*,
|
|
3
|
+
* the consumer's config owns the *what*." (docs/BRIEF.md, First session §2)
|
|
4
|
+
*
|
|
5
|
+
* Designed against two real checks before the runner exists:
|
|
6
|
+
* see examples/stilltrue.config.ts (GCPS board roster + BOC refresh).
|
|
7
|
+
*/
|
|
8
|
+
/** What a drift check found. Three outcomes, never two (BRIEF §First session 4). */
|
|
9
|
+
export type DriftOutcome =
|
|
10
|
+
/** Live source still matches the expected file. */
|
|
11
|
+
'pass'
|
|
12
|
+
/** Facts no longer match — fail the build. A human fixes. */
|
|
13
|
+
| 'rot'
|
|
14
|
+
/** Source unreachable / timeout / WAF 403 — warn loudly, do NOT fail. */
|
|
15
|
+
| 'error';
|
|
16
|
+
export interface DriftResult {
|
|
17
|
+
name: string;
|
|
18
|
+
outcome: DriftOutcome;
|
|
19
|
+
/** Human-readable detail: the diff for `rot`, the cause for `error`. */
|
|
20
|
+
messages: string[];
|
|
21
|
+
/** Structured diff when outcome is `rot`, for the HTML report. */
|
|
22
|
+
diff?: {
|
|
23
|
+
expected: unknown;
|
|
24
|
+
actual: unknown;
|
|
25
|
+
};
|
|
26
|
+
checkedAt: string;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* One curated fact vs its live authoritative source.
|
|
30
|
+
*
|
|
31
|
+
* `source()` is the consumer's code — it may fetch+parse deterministically
|
|
32
|
+
* or use an LLM to extract structure from a messy page; the package neither
|
|
33
|
+
* knows nor cares. The comparison stays deterministic either way.
|
|
34
|
+
*/
|
|
35
|
+
export interface DriftCheck<T = unknown> {
|
|
36
|
+
name: string;
|
|
37
|
+
/** Fetch the live source and return structured data. Throw => `error`, never `rot`. */
|
|
38
|
+
source: () => Promise<T>;
|
|
39
|
+
/** Path to the curated expected file (JSON), or the expected value itself. */
|
|
40
|
+
expect: string | T;
|
|
41
|
+
/**
|
|
42
|
+
* How to compare. A named built-in ('deep-equal', 'surnames', ...) or a
|
|
43
|
+
* custom deterministic function returning mismatch messages ([] = pass).
|
|
44
|
+
*/
|
|
45
|
+
compare?: string | ((expected: T, actual: T) => string[]);
|
|
46
|
+
}
|
|
47
|
+
export interface StilltrueConfig {
|
|
48
|
+
drift?: DriftCheck<any>[];
|
|
49
|
+
/** golden and verify are later phases — see docs/BRIEF.md build plan. */
|
|
50
|
+
golden?: unknown;
|
|
51
|
+
verify?: unknown;
|
|
52
|
+
}
|
|
53
|
+
/** Identity function for typed configs: `export default defineStilltrue({...})`. */
|
|
54
|
+
export declare function defineStilltrue(config: StilltrueConfig): StilltrueConfig;
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The config seam — the entire product is "the package owns the *how*,
|
|
3
|
+
* the consumer's config owns the *what*." (docs/BRIEF.md, First session §2)
|
|
4
|
+
*
|
|
5
|
+
* Designed against two real checks before the runner exists:
|
|
6
|
+
* see examples/stilltrue.config.ts (GCPS board roster + BOC refresh).
|
|
7
|
+
*/
|
|
8
|
+
/** Identity function for typed configs: `export default defineStilltrue({...})`. */
|
|
9
|
+
export function defineStilltrue(config) {
|
|
10
|
+
return config;
|
|
11
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { defineStilltrue, type StilltrueConfig, type DriftCheck, type DriftResult, type DriftOutcome, } from './config.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { defineStilltrue, } from './config.js';
|
package/package.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "stilltrue",
|
|
3
|
+
"version": "0.0.1",
|
|
4
|
+
"description": "Your tests check your code. stilltrue checks your facts. CI checks for AI-in-the-loop apps: drift (curated facts vs live sources), golden (regression evals), verify (judge answers against sources).",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"drift",
|
|
7
|
+
"facts",
|
|
8
|
+
"eval",
|
|
9
|
+
"llm",
|
|
10
|
+
"ci",
|
|
11
|
+
"regression",
|
|
12
|
+
"groundedness"
|
|
13
|
+
],
|
|
14
|
+
"license": "MIT",
|
|
15
|
+
"author": "Brian Foster",
|
|
16
|
+
"type": "module",
|
|
17
|
+
"main": "./dist/index.js",
|
|
18
|
+
"types": "./dist/index.d.ts",
|
|
19
|
+
"bin": {
|
|
20
|
+
"stilltrue": "./dist/cli.js"
|
|
21
|
+
},
|
|
22
|
+
"files": [
|
|
23
|
+
"dist"
|
|
24
|
+
],
|
|
25
|
+
"scripts": {
|
|
26
|
+
"build": "tsc",
|
|
27
|
+
"dev": "tsc --watch",
|
|
28
|
+
"test": "node --test --experimental-strip-types test/*.test.ts"
|
|
29
|
+
},
|
|
30
|
+
"dependencies": {
|
|
31
|
+
"zod": "^3.23.0"
|
|
32
|
+
},
|
|
33
|
+
"devDependencies": {
|
|
34
|
+
"@types/node": "^22.0.0",
|
|
35
|
+
"typescript": "^5.6.0"
|
|
36
|
+
},
|
|
37
|
+
"engines": {
|
|
38
|
+
"node": ">=20"
|
|
39
|
+
}
|
|
40
|
+
}
|