pi-mega-compact 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/vector-cortex.js +20 -0
- package/dist/dedup/digest.js +30 -0
- package/dist/dedup/l1-lsh-v2.js +60 -0
- package/dist/dedup/l1-lsh.js +52 -0
- package/dist/dedup/l1-minhash-v2.js +193 -0
- package/dist/dedup/l1-minhash.js +91 -0
- package/dist/dedup/l1-verify.js +54 -0
- package/dist/dedup/mmr.js +45 -0
- package/dist/dedup/normalize.js +39 -0
- package/dist/dedup/raptor/buildHistory.js +164 -0
- package/dist/dedup/raptor/guardrails.js +83 -0
- package/dist/dedup/raptor/incremental.js +304 -0
- package/dist/dedup/raptor/index.js +190 -0
- package/dist/dedup/raptor/kmeans.js +152 -0
- package/dist/dedup/raptor/multilevel.js +184 -0
- package/dist/dedup/raptor/retrieval.js +93 -0
- package/dist/dedup/raptor/summarizer.js +86 -0
- package/dist/dedup/raptor/tree.js +202 -0
- package/dist/dedup/topk.js +60 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +2 -0
- package/dist/extensions/mega-events/context-handler/dbMirrorAppend.js +63 -0
- package/dist/extensions/mega-events/context-handler/gateCheck.js +59 -0
- package/dist/extensions/mega-events/context-handler/liveTrim.js +178 -0
- package/dist/extensions/mega-events/context-handler/pipelineRun.js +37 -0
- package/dist/extensions/mega-events/context-handler.js +39 -305
- package/dist/src/config/vector-cortex.js +20 -0
- package/dist/src/config.js +1 -1
- package/dist/src/dedup/l1-lsh-v2.js +60 -0
- package/dist/src/dedup/l1-minhash-v2.js +193 -0
- package/dist/src/dedup/normalize.js +1 -1
- package/dist/src/vector-cortex/conformance/emit.js +46 -0
- package/dist/src/vector-cortex/conformance/manifest.js +235 -0
- package/dist/src/vector-cortex/conformance/runner.js +111 -0
- package/dist/src/vector-cortex/conformance/triadB-reader.js +95 -0
- package/dist/src/vector-cortex/encoder/asset.js +142 -0
- package/dist/src/vector-cortex/encoder/emit.js +42 -0
- package/dist/src/vector-cortex/encoder/runtime.js +228 -0
- package/dist/src/vector-cortex/encoder/types.js +85 -0
- package/dist/src/vector-cortex/migrations/minhash-v2.js +209 -0
- package/dist/vector-cortex/conformance/emit.js +46 -0
- package/dist/vector-cortex/conformance/manifest.js +235 -0
- package/dist/vector-cortex/conformance/runner.js +111 -0
- package/dist/vector-cortex/conformance/triadB-reader.js +95 -0
- package/dist/vector-cortex/encoder/asset.js +142 -0
- package/dist/vector-cortex/encoder/emit.js +42 -0
- package/dist/vector-cortex/encoder/runtime.js +228 -0
- package/dist/vector-cortex/encoder/types.js +85 -0
- package/dist/vector-cortex/migrations/minhash-v2.js +209 -0
- package/extensions/dashboard-client/dist/assets/{AreaChart-CAnWomhw.js → AreaChart-BQXFF7UI.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{AreaChart-CAnWomhw.js.map → AreaChart-BQXFF7UI.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{BarChart-lZ2-s71k.js → BarChart-BVtoz9Eg.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{BarChart-lZ2-s71k.js.map → BarChart-BVtoz9Eg.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{CacheTab-BL2Thh66.js → CacheTab-trI2TIGy.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{CacheTab-BL2Thh66.js.map → CacheTab-trI2TIGy.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{EventsTab-DEh2Z9fp.js → EventsTab-CNUbk9__.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{EventsTab-DEh2Z9fp.js.map → EventsTab-CNUbk9__.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{HealthTab-DTwKclxk.js → HealthTab-K-bd_CM-.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{HealthTab-DTwKclxk.js.map → HealthTab-K-bd_CM-.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DDfBEdbX.js → MaintenanceTab-CoEioKzx.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DDfBEdbX.js.map → MaintenanceTab-CoEioKzx.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MemoryMapTab-Dgfmg-0Q.js → MemoryMapTab-CJo24ulv.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MemoryMapTab-Dgfmg-0Q.js.map → MemoryMapTab-CJo24ulv.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{MetricsTab-DOcMm0Tx.js → MetricsTab-DCNlQWWP.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MetricsTab-DOcMm0Tx.js.map → MetricsTab-DCNlQWWP.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{OverviewTab-DHT5aoHg.js → OverviewTab-CQQlyIQ6.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{OverviewTab-DHT5aoHg.js.map → OverviewTab-CQQlyIQ6.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{ReposTab-CGAmtR_c.js → ReposTab-Cqm82_yn.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{ReposTab-CGAmtR_c.js.map → ReposTab-Cqm82_yn.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SessionsTab-BIgx6iKT.js → SessionsTab-A8xTAdET.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SessionsTab-BIgx6iKT.js.map → SessionsTab-A8xTAdET.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SetupTab-ADghKuEI.js → SetupTab-CqaSgaZl.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SetupTab-ADghKuEI.js.map → SetupTab-CqaSgaZl.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-C33UiiOM.js → TimeSavedCard-C_f3q8GR.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-C33UiiOM.js.map → TimeSavedCard-C_f3q8GR.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TurnsTab-DdbL9wPN.js → TurnsTab-BEsafACo.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TurnsTab-DdbL9wPN.js.map → TurnsTab-BEsafACo.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/VectorCortexTab-DD1CMK80.js +2 -0
- package/extensions/dashboard-client/dist/assets/VectorCortexTab-DD1CMK80.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{WikiTab-Ch00OFKs.js → WikiTab-Bf01M2_Z.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{WikiTab-Ch00OFKs.js.map → WikiTab-Bf01M2_Z.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{button-ByeEdo2g.js → button-BEJPTk_E.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{button-ByeEdo2g.js.map → button-BEJPTk_E.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{card-DesoLXvc.js → card-DK1V382s.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{card-DesoLXvc.js.map → card-DK1V382s.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-CCAB7Iz4.js → generateCategoricalChart-CRtaPphL.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-CCAB7Iz4.js.map → generateCategoricalChart-CRtaPphL.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{index-B6cgyAPr.js → index-CB6BslE4.js} +3 -3
- package/extensions/dashboard-client/dist/assets/{index-B6cgyAPr.js.map → index-CB6BslE4.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{switch-DIZC-oe-.js → switch-C4FV6B9k.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{switch-DIZC-oe-.js.map → switch-C4FV6B9k.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{toggle-CIoWaH2e.js → toggle-GBmeENMd.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{toggle-CIoWaH2e.js.map → toggle-GBmeENMd.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{useSSE-Bv4943l1.js → useSSE-BOKZGSn1.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{useSSE-Bv4943l1.js.map → useSSE-BOKZGSn1.js.map} +1 -1
- package/extensions/dashboard-client/dist/index.html +1 -1
- package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +1 -1
- package/extensions/dashboard-client/tsconfig.json +1 -1
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +12 -0
- package/extensions/mega-events/context-handler/dbMirrorAppend.ts +93 -0
- package/extensions/mega-events/context-handler/gateCheck.ts +101 -0
- package/extensions/mega-events/context-handler/liveTrim.ts +241 -0
- package/extensions/mega-events/context-handler/pipelineRun.ts +79 -0
- package/extensions/mega-events/context-handler.ts +45 -347
- package/package.json +1 -1
- package/src/config/vector-cortex.ts +22 -0
- package/src/config.ts +2 -0
- package/src/dedup/l1-lsh-v2.ts +78 -0
- package/src/dedup/l1-minhash-v2.ts +221 -0
- package/src/dedup/normalize.ts +1 -1
- package/src/store/sqlite/game-scores.ts +2 -1
- package/src/vector-cortex/conformance/emit.ts +56 -0
- package/src/vector-cortex/conformance/manifest.ts +298 -0
- package/src/vector-cortex/conformance/runner.ts +165 -0
- package/src/vector-cortex/conformance/triadB-reader.ts +110 -0
- package/src/vector-cortex/encoder/asset.ts +155 -0
- package/src/vector-cortex/encoder/emit.ts +51 -0
- package/src/vector-cortex/encoder/runtime.ts +283 -0
- package/src/vector-cortex/encoder/types.ts +165 -0
- package/src/vector-cortex/migrations/minhash-v2.ts +263 -0
- package/extensions/dashboard-client/dist/assets/VectorCortexTab-Cl1pbEdR.js +0 -2
- package/extensions/dashboard-client/dist/assets/VectorCortexTab-Cl1pbEdR.js.map +0 -1
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* l1-minhash-v2.ts — MinHash V2 signatures (VC1C, M4 minhash-v2).
|
|
3
|
+
*
|
|
4
|
+
* The v1 path (l1-minhash.ts) uses 32-bit universal hashing with `p=2^31-1`
|
|
5
|
+
* and Number math, which is exact because a*x < 2^62 can overflow double
|
|
6
|
+
* precision (>= 2^53) and is reduced too eagerly. MinHashV2 freezes a v2
|
|
7
|
+
* scheme that is CROSS-LANGUAGE deterministic and byte-exact:
|
|
8
|
+
*
|
|
9
|
+
* - normalization: the stable `normalize()` from normalize.ts (NFC, NFKC
|
|
10
|
+
* case-fold, CRLF->LF, whitespace collapse, 32K cap);
|
|
11
|
+
* - shingles: windows of FIVE CODE POINTS over the normalized text
|
|
12
|
+
* (SHINGLE_CP=5), each shingle hashed to a u64 with 64-bit FNV-1a over its
|
|
13
|
+
* UTF-8 bytes;
|
|
14
|
+
* - 256 PUBLISHED unsigned (a_i, b_i) seed pairs, derived deterministically
|
|
15
|
+
* via splitmix64 and listed in
|
|
16
|
+
* conformance/vector-cortex/v2/minhash/seeds-v2.json;
|
|
17
|
+
* - p = 2^61 - 1 (2305843009213693951), EXACT BigInt multiply/modulo — a_i*x
|
|
18
|
+
* can reach ~2^124 so Number math would silently corrupt high-bit products;
|
|
19
|
+
* - signature = 256 slots, slot i = min over shingles of (a_i*x + b_i) mod p;
|
|
20
|
+
* the empty signature is all-`p` sentinel slots;
|
|
21
|
+
* - encoding: 256 x u64 LITTLE-ENDIAN bytes (2048 bytes total);
|
|
22
|
+
* - bands: 64 bands of FOUR u64 slots each (256 total), see l1-lsh-v2.ts.
|
|
23
|
+
*
|
|
24
|
+
* NEVER compares a v2 signature against a v1 signature — cross-version compare
|
|
25
|
+
* is rejected with `MINHASH_VERSION_MISMATCH` (see minhash-v2 migration).
|
|
26
|
+
*
|
|
27
|
+
* Pure compute, no deps, no storage, no network (PREVENT-PI-004 / PREVENT-011).
|
|
28
|
+
*/
|
|
29
|
+
import { normalize } from "./normalize.js";
|
|
30
|
+
/** MinHashV2 scheme version tag (frozen). */
|
|
31
|
+
export const MINHASH_VERSION = 2;
|
|
32
|
+
/** The published count of slots / seed pairs (frozen). */
|
|
33
|
+
export const NUM_HASHES_V2 = 256;
|
|
34
|
+
/** Five-code-point shingle length (frozen). */
|
|
35
|
+
export const SHINGLE_CP = 5;
|
|
36
|
+
/** p = 2^61 - 1 (Mersenne prime), as exact BigInt. */
|
|
37
|
+
export const P_V2 = (1n << 61n) - 1n;
|
|
38
|
+
/** Little-endian bytes per u64 signature slot. */
|
|
39
|
+
export const U64_BYTES = 8;
|
|
40
|
+
/** Signature byte length: 256 slots x 8 LE bytes. */
|
|
41
|
+
export const SIGNATURE_BYTES_V2 = NUM_HASHES_V2 * U64_BYTES;
|
|
42
|
+
/** Fixed splitmix64 initial state so the seed table is reproducible everywhere. */
|
|
43
|
+
export const SEED_STATE_INIT = 0x5eedc0def001ba11n & 0xffffffffffffffffn;
|
|
44
|
+
/** splitmix64 mixing constants (frozen). */
|
|
45
|
+
const SM_GOLDEN = 0x9e3779b97f4a7c15n;
|
|
46
|
+
const SM_M1 = 0xbf58476d1ce4e5b9n;
|
|
47
|
+
const SM_M2 = 0x94d049bb133111ebn;
|
|
48
|
+
const MASK64 = 0xffffffffffffffffn;
|
|
49
|
+
function mix(z0) {
|
|
50
|
+
let z = z0;
|
|
51
|
+
z = (z ^ (z >> 30n)) * SM_M1;
|
|
52
|
+
z &= MASK64;
|
|
53
|
+
z = (z ^ (z >> 27n)) * SM_M2;
|
|
54
|
+
z &= MASK64;
|
|
55
|
+
return z ^ (z >> 31n);
|
|
56
|
+
}
|
|
57
|
+
/** Create a splitmix64 PRNG from a fixed unsigned 64-bit state. */
|
|
58
|
+
export function splitmix64(state = SEED_STATE_INIT) {
|
|
59
|
+
let s = state & MASK64;
|
|
60
|
+
return () => {
|
|
61
|
+
s = (s + SM_GOLDEN) & MASK64;
|
|
62
|
+
return mix(s) & MASK64;
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Derive the frozen 256 published (a_i, b_i) seed pairs. Each pair maps two
|
|
67
|
+
* splitmix64 draws into [1, p-1]. Deterministic and cross-language reproducible
|
|
68
|
+
* from SEED_STATE_INIT — mirrors `seeds-v2.json`.
|
|
69
|
+
*
|
|
70
|
+
* The table is frozen: computed ONCE at module load and reused for every call.
|
|
71
|
+
* Recomputing the 256 splitmix64 pairs (and re-walking all shingles) per
|
|
72
|
+
* signature/backfill row is wasteful for a large v1 index backfill, and the
|
|
73
|
+
* table is immutable by construction, so a single shared instance is safe and
|
|
74
|
+
* byte-identical. The returned sub-arrays share the frozen pair objects; they
|
|
75
|
+
* are themselves frozen so no caller can mutate the shared cache.
|
|
76
|
+
*/
|
|
77
|
+
export function minhashV2Seeds(count = NUM_HASHES_V2) {
|
|
78
|
+
return FROZEN_SEEDS.slice(0, count);
|
|
79
|
+
}
|
|
80
|
+
/** Singleton seed table, computed exactly once and frozen (see minhashV2Seeds). */
|
|
81
|
+
const FROZEN_SEEDS = (() => {
|
|
82
|
+
const prng = splitmix64();
|
|
83
|
+
const pairs = [];
|
|
84
|
+
for (let i = 0; i < NUM_HASHES_V2; i++) {
|
|
85
|
+
const a = (prng() % (P_V2 - 1n)) + 1n;
|
|
86
|
+
const b = (prng() % (P_V2 - 1n)) + 1n;
|
|
87
|
+
pairs.push(Object.freeze({ a, b }));
|
|
88
|
+
}
|
|
89
|
+
return Object.freeze(pairs);
|
|
90
|
+
})();
|
|
91
|
+
/** 64-bit FNV-1a over a UTF-8 string (for shingle hashing). */
|
|
92
|
+
function fnv1a64String(s) {
|
|
93
|
+
let h = 0xcbf29ce484222325n;
|
|
94
|
+
const bytes = Buffer.from(s, "utf8");
|
|
95
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
96
|
+
h ^= BigInt(bytes[i] ?? 0);
|
|
97
|
+
h = (h * 0x100000001b3n) & MASK64;
|
|
98
|
+
}
|
|
99
|
+
return h;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Five-code-point shingle set (deduped, capped) of normalized text as u64
|
|
103
|
+
* hashes. Iterates over code points (not UTF-16 code units) so astral-plane
|
|
104
|
+
* characters do not split a shingle.
|
|
105
|
+
*/
|
|
106
|
+
export function shinglesV2(text, shingleCp = SHINGLE_CP, maxShingles = 50_000) {
|
|
107
|
+
const norm = normalize(text);
|
|
108
|
+
if (norm.length === 0)
|
|
109
|
+
return [];
|
|
110
|
+
const cps = Array.from(norm); // code points
|
|
111
|
+
const set = new Set();
|
|
112
|
+
if (cps.length < shingleCp) {
|
|
113
|
+
set.add(fnv1a64String(norm));
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
for (let i = 0; i + shingleCp <= cps.length; i++) {
|
|
117
|
+
const sh = cps.slice(i, i + shingleCp).join("");
|
|
118
|
+
set.add(fnv1a64String(sh));
|
|
119
|
+
if (set.size >= maxShingles)
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return [...set];
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Compute the 256-slot MinHashV2 signature of a text as an array of exact u64
|
|
127
|
+
* BigInt values (each in [0, p], p = sentinel when empty). Uses EXACT BigInt
|
|
128
|
+
* multiply/modulo so high-bit products (a*x ~ up to 2^124) are never corrupted.
|
|
129
|
+
*/
|
|
130
|
+
export function minhashV2Signature(text) {
|
|
131
|
+
const grams = shinglesV2(text);
|
|
132
|
+
const seeds = minhashV2Seeds();
|
|
133
|
+
const sig = new Array(NUM_HASHES_V2).fill(P_V2);
|
|
134
|
+
if (grams.length === 0)
|
|
135
|
+
return sig;
|
|
136
|
+
for (let i = 0; i < NUM_HASHES_V2; i++) {
|
|
137
|
+
const { a, b } = seeds[i];
|
|
138
|
+
let min = P_V2;
|
|
139
|
+
for (const x of grams) {
|
|
140
|
+
const h = (a * x + b) % P_V2;
|
|
141
|
+
if (h < min)
|
|
142
|
+
min = h;
|
|
143
|
+
}
|
|
144
|
+
sig[i] = min;
|
|
145
|
+
}
|
|
146
|
+
return sig;
|
|
147
|
+
}
|
|
148
|
+
/** Encode a 256-slot signature to 2048 bytes as 256 x u64 little-endian. */
|
|
149
|
+
export function encodeSignatureV2(sig) {
|
|
150
|
+
if (sig.length !== NUM_HASHES_V2) {
|
|
151
|
+
throw new Error(`encodeSignatureV2: expected ${NUM_HASHES_V2} slots, got ${sig.length}`);
|
|
152
|
+
}
|
|
153
|
+
const buf = Buffer.allocUnsafe(SIGNATURE_BYTES_V2);
|
|
154
|
+
for (let i = 0; i < NUM_HASHES_V2; i++) {
|
|
155
|
+
buf.writeBigUInt64LE(sig[i], i * U64_BYTES);
|
|
156
|
+
}
|
|
157
|
+
return new Uint8Array(buf);
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Estimated Jaccard similarity of two v2 signatures (fraction of equal slots).
|
|
161
|
+
*
|
|
162
|
+
* THIS IS v2-ONLY. Cross-version compare is rejected INTERNALLY with
|
|
163
|
+
* `MINHASH_VERSION_MISMATCH`: both inputs must be exact 256-slot v2 signatures,
|
|
164
|
+
* i.e. every slot a `bigint` in [0, P_V2] (the empty sentinel is exactly P_V2).
|
|
165
|
+
* A v1 signature is a `number[]` of u31 values, so passing a v1 array fails the
|
|
166
|
+
* bigint type/range check and is rejected — a mixed v1/v2 compare can never
|
|
167
|
+
* produce a similarity number. Never call this with v1 arrays.
|
|
168
|
+
*/
|
|
169
|
+
export function signatureSimilarityV2(a, b) {
|
|
170
|
+
assertV2Signature(a);
|
|
171
|
+
assertV2Signature(b);
|
|
172
|
+
const n = Math.min(a.length, b.length);
|
|
173
|
+
if (n === 0)
|
|
174
|
+
return 0;
|
|
175
|
+
let equal = 0;
|
|
176
|
+
for (let i = 0; i < n; i++)
|
|
177
|
+
if (a[i] === b[i])
|
|
178
|
+
equal++;
|
|
179
|
+
return equal / n;
|
|
180
|
+
}
|
|
181
|
+
/** Reject any array that is not an exact 256-slot v2 signature (bigint in [0, P_V2]). */
|
|
182
|
+
function assertV2Signature(sig) {
|
|
183
|
+
if (sig.length !== NUM_HASHES_V2) {
|
|
184
|
+
throw new Error(MINHASH_VERSION_MISMATCH_MSG);
|
|
185
|
+
}
|
|
186
|
+
for (const s of sig) {
|
|
187
|
+
if (typeof s !== "bigint" || s < 0n || s > P_V2) {
|
|
188
|
+
throw new Error(MINHASH_VERSION_MISMATCH_MSG);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
/** Frozen cross-version-reject message shared by the v2-only function guards. */
|
|
193
|
+
const MINHASH_VERSION_MISMATCH_MSG = "MINHASH_VERSION_MISMATCH";
|
|
@@ -30,7 +30,7 @@ export function normalize(text) {
|
|
|
30
30
|
return "";
|
|
31
31
|
let out = stripAnsi(text);
|
|
32
32
|
out = out.normalize("NFC");
|
|
33
|
-
out = out.
|
|
33
|
+
out = out.toLowerCase(); // case-fold so "Foo"/"FOO" collapse to one key (locale-independent: byte-exact across languages)
|
|
34
34
|
out = out.replace(/\r\n?/g, "\n"); // CRLF / CR → LF
|
|
35
35
|
out = out.replace(/\s+/g, " ").trim();
|
|
36
36
|
if (out.length > MAX_CHARS)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/conformance/emit.ts — VC1C observability seam.
|
|
3
|
+
*
|
|
4
|
+
* Emits the three VC1C structured events, all gated on `MEGACOMPACT_VC1C`
|
|
5
|
+
* (mode C parity: flag OFF => zero emissions). Every event is a JSON line with
|
|
6
|
+
* `ts` + `event` (ENGINEERING_PRACTICES §8); the emitters below capture the
|
|
7
|
+
* event name + fields and are never fatal on consumer failure (non-fatal
|
|
8
|
+
* observability, never breaks the agent loop).
|
|
9
|
+
*
|
|
10
|
+
* vector_cortex_minhash_v2_backfilled — M4 backfill wrote v2 rows
|
|
11
|
+
* vector_cortex_conformance_case_checked — a v2 conformance case dispatched
|
|
12
|
+
* vector_cortex_downgrade_copy_written — a downgrade legacy copy written
|
|
13
|
+
*
|
|
14
|
+
* No network, no side effects beyond the supplied emit callback
|
|
15
|
+
* (PREVENT-PI-004 / PREVENT-011).
|
|
16
|
+
*/
|
|
17
|
+
import { VC1C_ENABLED } from "../../config/vector-cortex.js";
|
|
18
|
+
/** A flag-gated no-op reporter (zero emissions, default when none injected). */
|
|
19
|
+
export const NOOP_CONFORMANCE_REPORTER = {
|
|
20
|
+
backfilled: () => { },
|
|
21
|
+
caseChecked: () => { },
|
|
22
|
+
downgradeWritten: () => { },
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Flag-gated emit: no-op when VC1C is off or no emitter is supplied. The
|
|
26
|
+
* returned reporter is itself flag-gated (`VC1C_ENABLED`), so wiring it into a
|
|
27
|
+
* runtime seam yields zero emissions when `MEGACOMPACT_VC1C=0` (byte-identical
|
|
28
|
+
* to the predecessor).
|
|
29
|
+
*/
|
|
30
|
+
export function createConformanceReporter(emit) {
|
|
31
|
+
const fire = (event, fields) => {
|
|
32
|
+
if (!VC1C_ENABLED())
|
|
33
|
+
return;
|
|
34
|
+
try {
|
|
35
|
+
emit?.(event, { ...fields, ts: new Date().toISOString() });
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
/* non-fatal observability */
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
return {
|
|
42
|
+
backfilled: (fields) => fire("vector_cortex_minhash_v2_backfilled", fields),
|
|
43
|
+
caseChecked: (fields) => fire("vector_cortex_conformance_case_checked", fields),
|
|
44
|
+
downgradeWritten: (fields) => fire("vector_cortex_downgrade_copy_written", fields),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/conformance/manifest.ts — FixtureManifestV2 (VC1C).
|
|
3
|
+
*
|
|
4
|
+
* Owns the v2 conformance manifest: reads the authoritative
|
|
5
|
+
* `conformance/vector-cortex/v2/manifest.json` plus the on-disk fixture tree
|
|
6
|
+
* and validates it canonically. Each FixtureManifestV2 entry carries:
|
|
7
|
+
* id, domain, inputDigest, expectedDigest, failureCode, algorithmTuple.
|
|
8
|
+
*
|
|
9
|
+
* Canonical validation rejects a mismatched corpus with the frozen codes:
|
|
10
|
+
* CONF_EXTRA_FIXTURE — a file on disk that the manifest does not list;
|
|
11
|
+
* CONF_MISSING_FIXTURE — a manifest entry with no file on disk;
|
|
12
|
+
* CONF_DIGEST_DRIFT — on-disk bytes do not match the manifest SHA-256.
|
|
13
|
+
*
|
|
14
|
+
* Key ordering rule (canonical JSON): object keys are sorted by UTF-8 bytes;
|
|
15
|
+
* a manifest that is not canonical (or a fixture whose canonical re-serialization
|
|
16
|
+
* differs) is itself a digest-drift / noncanonical condition. The invariant:
|
|
17
|
+
* canonical valid manifests converge to ONE digest (the corpus is a single
|
|
18
|
+
* reproducible byte image).
|
|
19
|
+
*
|
|
20
|
+
* Pure FS reads + pure predicates; no network, no side effects on authority
|
|
21
|
+
* data (PREVENT-PI-004 / PREVENT-011).
|
|
22
|
+
*/
|
|
23
|
+
import { readFileSync, readdirSync, statSync } from "node:fs";
|
|
24
|
+
import { createHash as sha256 } from "node:crypto";
|
|
25
|
+
import { join, relative, sep } from "node:path";
|
|
26
|
+
/** Frozen VC1C conformance failure codes (the "reject unknown / extra / drifted"). */
|
|
27
|
+
export const CONF_FAIL = {
|
|
28
|
+
EXTRA_FIXTURE: "CONF_EXTRA_FIXTURE",
|
|
29
|
+
MISSING_FIXTURE: "CONF_MISSING_FIXTURE",
|
|
30
|
+
DIGEST_DRIFT: "CONF_DIGEST_DRIFT",
|
|
31
|
+
NONCANONICAL: "CONF_NONCANONICAL",
|
|
32
|
+
UNKNOWN_DOMAIN: "CONF_UNKNOWN_DOMAIN",
|
|
33
|
+
};
|
|
34
|
+
/** Walk a directory tree returning relative POSIX paths (sorted). */
|
|
35
|
+
function walk(dir, base, acc = []) {
|
|
36
|
+
for (const name of readdirSync(dir)) {
|
|
37
|
+
const p = join(dir, name);
|
|
38
|
+
const st = statSync(p);
|
|
39
|
+
if (st.isDirectory())
|
|
40
|
+
walk(p, base, acc);
|
|
41
|
+
else
|
|
42
|
+
acc.push(relative(base, p).split(sep).join("/"));
|
|
43
|
+
}
|
|
44
|
+
return acc.sort();
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Canonical JSON bytes for a value (UTF-8, NFC keys, sorted, shortest numbers).
|
|
48
|
+
* Matches the stricter `canonicalNumber` in scripts/vector-cortex-conformance.mjs:
|
|
49
|
+
* a non-finite number or -0 has no canonical shortest representation and is
|
|
50
|
+
* rejected, so the two "canonical" serializers can never diverge for the same
|
|
51
|
+
* value (thereby preserving the "converge to one digest" invariant).
|
|
52
|
+
*/
|
|
53
|
+
function canonicalValue(value) {
|
|
54
|
+
if (value === null || typeof value !== "object") {
|
|
55
|
+
if (typeof value === "number") {
|
|
56
|
+
if (!Number.isFinite(value) || Object.is(value, -0)) {
|
|
57
|
+
throw new Error(`non-canonical number: ${value}`);
|
|
58
|
+
}
|
|
59
|
+
if (Number.isInteger(value) && Number.isSafeInteger(value))
|
|
60
|
+
return String(value);
|
|
61
|
+
return JSON.stringify(value);
|
|
62
|
+
}
|
|
63
|
+
return JSON.stringify(value);
|
|
64
|
+
}
|
|
65
|
+
if (Array.isArray(value))
|
|
66
|
+
return `[${value.map((v) => canonicalValue(v)).join(",")}]`;
|
|
67
|
+
const keys = Object.keys(value)
|
|
68
|
+
.map((k) => k.normalize("NFC"))
|
|
69
|
+
.sort();
|
|
70
|
+
const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalValue(value[k])}`);
|
|
71
|
+
return `{${parts.join(",")}}`;
|
|
72
|
+
}
|
|
73
|
+
function canonicalJson(value) {
|
|
74
|
+
return Buffer.from(`${canonicalValue(value)}\n`, "utf8");
|
|
75
|
+
}
|
|
76
|
+
function sha256Hex(bytes) {
|
|
77
|
+
return sha256("sha256").update(bytes).digest("hex");
|
|
78
|
+
}
|
|
79
|
+
/** parse a manifest.json (must be a valid v2 manifest object). */
|
|
80
|
+
function parseManifest(raw) {
|
|
81
|
+
const obj = JSON.parse(raw);
|
|
82
|
+
if (obj === null || typeof obj !== "object" || !Array.isArray(obj.fixtures)) {
|
|
83
|
+
throw new Error("CONF_MANIFEST_INVALID: manifest.fixtures must be an array");
|
|
84
|
+
}
|
|
85
|
+
return {
|
|
86
|
+
version: typeof obj.version === "string" ? obj.version : "2",
|
|
87
|
+
owner: typeof obj.owner === "string" ? obj.owner : "",
|
|
88
|
+
fixtures: obj.fixtures,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Read + normalize the v2 manifest and on-disk tree into a FixtureManifestV2.
|
|
93
|
+
* `fixtureRoot` is the conformance v2 root directory (injected by callers so
|
|
94
|
+
* `src/` stays runtime-independent of a hardcoded path).
|
|
95
|
+
*/
|
|
96
|
+
export function readFixtureManifestV2(fixtureRoot) {
|
|
97
|
+
const manifestRaw = readFileSync(join(fixtureRoot, "manifest.json"), "utf8");
|
|
98
|
+
const manifest = parseManifest(manifestRaw);
|
|
99
|
+
const byId = new Map();
|
|
100
|
+
const entries = [];
|
|
101
|
+
for (const fx of manifest.fixtures) {
|
|
102
|
+
const expectedDigest = String(fx.sha256 ?? "");
|
|
103
|
+
const domain = fx.domain ?? domainOf(fx.path);
|
|
104
|
+
const inputDigest = canonicalFileDigest(join(fixtureRoot, fx.path));
|
|
105
|
+
const entry = {
|
|
106
|
+
id: fx.id,
|
|
107
|
+
domain,
|
|
108
|
+
path: fx.path,
|
|
109
|
+
algorithm: fx.algorithm,
|
|
110
|
+
expected: fx.expected,
|
|
111
|
+
inputDigest,
|
|
112
|
+
expectedDigest,
|
|
113
|
+
expectedOutputDigest: typeof fx.outputDigest === "string" && fx.outputDigest.length > 0
|
|
114
|
+
? fx.outputDigest
|
|
115
|
+
: undefined,
|
|
116
|
+
failureCode: fx.expected === "ok" ? undefined : fx.expected,
|
|
117
|
+
algorithmTuple: String(fx.algorithm).split(";").filter(Boolean).map((s) => s.trim()),
|
|
118
|
+
};
|
|
119
|
+
entries.push(entry);
|
|
120
|
+
byId.set(fx.id, entry);
|
|
121
|
+
}
|
|
122
|
+
return {
|
|
123
|
+
version: manifest.version,
|
|
124
|
+
owner: manifest.owner.split(",").map((s) => s.trim()).filter(Boolean),
|
|
125
|
+
fixtureEntries: entries,
|
|
126
|
+
byId,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/** The v2 domain a fixture path belongs to (first path segment). */
|
|
130
|
+
export function domainOf(path) {
|
|
131
|
+
return path.split("/")[0] ?? "";
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* SHA-256 over the canonical JSON bytes of a fixture file. A missing file
|
|
135
|
+
* returns "" so `validateCanonicalV2` can report CONF_MISSING_FIXTURE instead of
|
|
136
|
+
* throwing ENOENT part-way through manifest normalization.
|
|
137
|
+
*/
|
|
138
|
+
function canonicalFileDigest(absPath) {
|
|
139
|
+
let raw;
|
|
140
|
+
try {
|
|
141
|
+
raw = readFileSync(absPath, "utf8");
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
return "";
|
|
145
|
+
}
|
|
146
|
+
let parsed;
|
|
147
|
+
try {
|
|
148
|
+
parsed = JSON.parse(raw);
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
parsed = raw; // non-JSON (e.g. seeds as bytes) — digest raw
|
|
152
|
+
}
|
|
153
|
+
return sha256Hex(canonicalJson(parsed));
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Validate the on-disk v2 corpus against the manifest. Rejects an extra,
|
|
157
|
+
* missing, or digest-drifted fixture file with the frozen codes. Also verifies
|
|
158
|
+
* the manifest itself is canonical (sorted keys, shortest numbers) — a
|
|
159
|
+
* noncanonical manifest is reported as NONCANONICAL.
|
|
160
|
+
*/
|
|
161
|
+
export function validateCanonicalV2(fixtureRoot) {
|
|
162
|
+
const codes = [];
|
|
163
|
+
const issues = [];
|
|
164
|
+
const manifestRaw = readFileSync(join(fixtureRoot, "manifest.json"), "utf8");
|
|
165
|
+
try {
|
|
166
|
+
const canonicalBytes = canonicalJson(JSON.parse(manifestRaw));
|
|
167
|
+
if (Buffer.from(manifestRaw, "utf8").toString("hex") !== canonicalBytes.toString("hex")) {
|
|
168
|
+
codes.push(CONF_FAIL.NONCANONICAL);
|
|
169
|
+
issues.push("manifest.json is not canonical");
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
codes.push(CONF_FAIL.NONCANONICAL);
|
|
174
|
+
issues.push("manifest.json is not valid JSON");
|
|
175
|
+
}
|
|
176
|
+
const manifest = readFixtureManifestV2(fixtureRoot);
|
|
177
|
+
const listedPaths = new Set(manifest.fixtureEntries.map((e) => e.path));
|
|
178
|
+
// Extra-file detection: every file under the root (except manifest) listed.
|
|
179
|
+
const onDisk = walk(fixtureRoot, fixtureRoot).filter((p) => p !== "manifest.json");
|
|
180
|
+
for (const p of onDisk) {
|
|
181
|
+
if (!listedPaths.has(p)) {
|
|
182
|
+
codes.push(CONF_FAIL.EXTRA_FIXTURE);
|
|
183
|
+
issues.push(`extra fixture: ${p}`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
// Missing + digest-drift for every listed fixture.
|
|
187
|
+
for (const entry of manifest.fixtureEntries) {
|
|
188
|
+
const abs = join(fixtureRoot, entry.path);
|
|
189
|
+
if (!exists(abs)) {
|
|
190
|
+
codes.push(CONF_FAIL.MISSING_FIXTURE);
|
|
191
|
+
issues.push(`missing fixture: ${entry.path}`);
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
const raw = readFileSync(abs, "utf8");
|
|
195
|
+
let canonicalHex;
|
|
196
|
+
try {
|
|
197
|
+
canonicalHex = sha256Hex(canonicalJson(JSON.parse(raw)));
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
canonicalHex = sha256Hex(Buffer.from(raw, "utf8"));
|
|
201
|
+
}
|
|
202
|
+
if (canonicalHex !== entry.expectedDigest) {
|
|
203
|
+
codes.push(CONF_FAIL.DIGEST_DRIFT);
|
|
204
|
+
issues.push(`digest drift for ${entry.path}: expected ${entry.expectedDigest} got ${canonicalHex}`);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
// The canonical-key ordering invariant: a canonical valid manifest is a single
|
|
208
|
+
// reproducible digest. (Extra-file presence also breaks purity but is the
|
|
209
|
+
// extra/missing covers it.) Noncanonical keys are caught above.
|
|
210
|
+
if (codes.length === 0) {
|
|
211
|
+
return { ok: true, entryCount: manifest.fixtureEntries.length };
|
|
212
|
+
}
|
|
213
|
+
return { ok: false, codes: dedupeCodes(codes), issues };
|
|
214
|
+
}
|
|
215
|
+
/** True if the manifest+corpus is canonical (converges to one digest). */
|
|
216
|
+
export function canonicalManifestsConverge(fixtureRoot) {
|
|
217
|
+
const v = validateCanonicalV2(fixtureRoot);
|
|
218
|
+
return v.ok;
|
|
219
|
+
}
|
|
220
|
+
function exists(p) {
|
|
221
|
+
try {
|
|
222
|
+
statSync(p);
|
|
223
|
+
return true;
|
|
224
|
+
}
|
|
225
|
+
catch {
|
|
226
|
+
return false;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
function dedupeCodes(codes) {
|
|
230
|
+
const out = [];
|
|
231
|
+
for (const c of codes)
|
|
232
|
+
if (!out.includes(c))
|
|
233
|
+
out.push(c);
|
|
234
|
+
return out;
|
|
235
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/conformance/runner.ts — v2 conformance runner (VC1C).
|
|
3
|
+
*
|
|
4
|
+
* Dispatches a conformance case STRICTLY by domain/version (no inference, no
|
|
5
|
+
* partial output on unknown) and compares both the canonical success bytes and
|
|
6
|
+
* the expected failure code. Owns `DowngradeReport`.
|
|
7
|
+
*
|
|
8
|
+
* The runner is triad-shaped (mode A client):
|
|
9
|
+
* A = this manifest runner over the real algorithms;
|
|
10
|
+
* B = an independent exact fixture reader (invertible digest recheck);
|
|
11
|
+
* C = reject any unknown domain/version WITHOUT partial output.
|
|
12
|
+
*
|
|
13
|
+
* Downgrade export: `exporter` produces a new legacy copy that never edits
|
|
14
|
+
* authority data; the resulting `DowngradeReport` is deterministic — a second
|
|
15
|
+
* run yields a byte-identical report digest (CONF-DOWN-003).
|
|
16
|
+
*
|
|
17
|
+
* No network, no side effects on authority (PREVENT-PI-004 / PREVENT-011).
|
|
18
|
+
*/
|
|
19
|
+
import { CONF_FAIL } from "./manifest.js";
|
|
20
|
+
import { createConformanceReporter, } from "./emit.js";
|
|
21
|
+
const UNKNOWN_DOMAIN = "CONF_UNKNOWN_DOMAIN";
|
|
22
|
+
const UNKNOWN_VERSION = "CONF_UNKNOWN_VERSION";
|
|
23
|
+
/**
|
|
24
|
+
* The runner-pinned expectation mismatch code: a dispatched handler returned an
|
|
25
|
+
* outcome that contradicts the manifest entry's frozen expectation (the wrong
|
|
26
|
+
* failure code, or a success/failure parity error). This is the runner's own
|
|
27
|
+
* cross-check (task 5: "compare both success bytes and expected failure code"),
|
|
28
|
+
* distinct from the domain's own failure codes.
|
|
29
|
+
*/
|
|
30
|
+
const EXPECTATION_MISMATCH = "CONF_EXPECTATION_MISMATCH";
|
|
31
|
+
export { UNKNOWN_DOMAIN, UNKNOWN_VERSION, EXPECTATION_MISMATCH };
|
|
32
|
+
/**
|
|
33
|
+
* Run one conformance case by dispatching strictly on `(domain, algorithmTuple)`.
|
|
34
|
+
* If no handler is registered for the exact domain/algorithms, returns
|
|
35
|
+
* `CONF_UNKNOWN_DOMAIN`/`CONF_UNKNOWN_VERSION` WITHOUT partial output.
|
|
36
|
+
*
|
|
37
|
+
* After dispatch the runner cross-checks the handler's outcome against the
|
|
38
|
+
* manifest entry's frozen expectation (task 5):
|
|
39
|
+
* - failure expected (`entry.failureCode` set): the handler must return
|
|
40
|
+
* `ok:false` with that exact code, else `CONF_EXPECTATION_MISMATCH`;
|
|
41
|
+
* - success expected (`entry.expected === "ok"`): the handler must return
|
|
42
|
+
* `ok:true`, and when the manifest pins an `expectedOutputDigest` the
|
|
43
|
+
* handler's `outputDigest` must match it, else `CONF_DIGEST_DRIFT`.
|
|
44
|
+
*/
|
|
45
|
+
export function runConformanceCase(entry, handlers, fixture, reporter = createConformanceReporter()) {
|
|
46
|
+
const key = `${entry.domain}::${entry.algorithmTuple.join(";")}`;
|
|
47
|
+
const handler = handlers.get(key);
|
|
48
|
+
let result;
|
|
49
|
+
if (!handler) {
|
|
50
|
+
if (entry.domain === "") {
|
|
51
|
+
result = { ok: false, code: UNKNOWN_DOMAIN, algorithm: entry.algorithm };
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
result = { ok: false, code: UNKNOWN_VERSION, algorithm: entry.algorithm };
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
const out = handler.run(entry, fixture);
|
|
59
|
+
if (out.ok) {
|
|
60
|
+
// Success expected? The manifest entry pins the expected success bytes.
|
|
61
|
+
if (entry.expected !== "ok") {
|
|
62
|
+
result = { ok: false, code: EXPECTATION_MISMATCH, algorithm: entry.algorithm };
|
|
63
|
+
}
|
|
64
|
+
else if (entry.expectedOutputDigest !== undefined &&
|
|
65
|
+
out.outputDigest !== entry.expectedOutputDigest) {
|
|
66
|
+
result = { ok: false, code: CONF_FAIL.DIGEST_DRIFT, algorithm: entry.algorithm };
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
result = { ok: true, outputDigest: out.outputDigest, algorithm: entry.algorithm };
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
// Failure expected? The manifest entry pins the exact failure code.
|
|
74
|
+
if (entry.failureCode !== undefined && out.code === entry.failureCode) {
|
|
75
|
+
result = { ok: false, code: out.code, algorithm: entry.algorithm };
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
result = { ok: false, code: EXPECTATION_MISMATCH, algorithm: entry.algorithm };
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
reporter.caseChecked({
|
|
83
|
+
id: entry.id,
|
|
84
|
+
domain: entry.domain,
|
|
85
|
+
algorithm: entry.algorithm,
|
|
86
|
+
ok: result.ok,
|
|
87
|
+
...(result.ok ? { outputDigest: result.outputDigest } : { code: result.code }),
|
|
88
|
+
});
|
|
89
|
+
return result;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Build a dispatch key for a (domain, algorithmTuple) pair. Callers register
|
|
93
|
+
* handlers under this key.
|
|
94
|
+
*/
|
|
95
|
+
export function handlerKey(domain, algorithmTuple) {
|
|
96
|
+
return `${domain}::${algorithmTuple.join(";")}`;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Run a downgrade export and recompute its report digest deterministically.
|
|
100
|
+
* A second invocation produces a byte-identical report (CONF-DOWN-003).
|
|
101
|
+
*/
|
|
102
|
+
export function runDowngradeExport(exporter, reporter = createConformanceReporter()) {
|
|
103
|
+
const report = exporter.exportOnce();
|
|
104
|
+
reporter.downgradeWritten({
|
|
105
|
+
exportedCopyId: report.exportedCopyId,
|
|
106
|
+
copiedCount: report.copiedCount,
|
|
107
|
+
unrepresentableIds: report.unrepresentableIds,
|
|
108
|
+
reportDigest: report.reportDigest,
|
|
109
|
+
});
|
|
110
|
+
return report;
|
|
111
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* conformance/triadB-reader.ts — TRIAD-B independent exact reader (VC1C).
|
|
3
|
+
*
|
|
4
|
+
* Deliberately shares NO code and NO cached-seed instance with the runner
|
|
5
|
+
* modules (l1-minhash-v2.ts / l1-lsh-v2.ts). It re-derives the minhash-v2
|
|
6
|
+
* signature and 64 bucket keys from first principles, using only the PUBLISHED
|
|
7
|
+
* (a,b) seed pairs passed in (read by the caller from
|
|
8
|
+
* conformance/.../minhash/seeds-v2.json — the table authored independently by
|
|
9
|
+
* scripts/gen-fixtures/minhash.mjs, the same process that produced the frozen
|
|
10
|
+
* fixture bytes). Because B shares no subroutine with the runner, a systematic
|
|
11
|
+
* error in the runner's splitmix64 / seed-derivation / signature arithmetic
|
|
12
|
+
* cannot satisfy both triad A and triad B.
|
|
13
|
+
*
|
|
14
|
+
* Pure compute, no deps, no storage, no network (PREVENT-PI-004 / PREVENT-011).
|
|
15
|
+
*/
|
|
16
|
+
import { createHash } from "node:crypto";
|
|
17
|
+
const MASK64 = 0xffffffffffffffffn;
|
|
18
|
+
const P = (1n << 61n) - 1n; // 2^61 - 1
|
|
19
|
+
const SLOTS = 256;
|
|
20
|
+
const U64 = 8;
|
|
21
|
+
const BANDS = 64;
|
|
22
|
+
/** 64-bit FNV-1a over UTF-8 bytes or a raw byte buffer (own, independent loop). */
|
|
23
|
+
function fnv1a64(input) {
|
|
24
|
+
let h = 0xcbf29ce484222325n;
|
|
25
|
+
for (const b of Buffer.from(input)) {
|
|
26
|
+
h = ((h ^ BigInt(b)) * 0x100000001b3n) & MASK64;
|
|
27
|
+
}
|
|
28
|
+
return h;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Re-derive the 2048-byte minhash-v2 signature and (optionally) the 64 LSH
|
|
32
|
+
* bucket keys for `text` within `session`, from published seed pairs only —
|
|
33
|
+
* never the runner module.
|
|
34
|
+
*/
|
|
35
|
+
export function independentReaderV2(text, session, seedPairs, withBuckets) {
|
|
36
|
+
// Normalization: ANSI-strip, NFC, case-fold, CRLF->LF, whitespace collapse,
|
|
37
|
+
// 32K cap — mirrored from the frozen VC1C spec.
|
|
38
|
+
let norm = (text || "")
|
|
39
|
+
.replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "")
|
|
40
|
+
.normalize("NFC")
|
|
41
|
+
.toLowerCase()
|
|
42
|
+
.replace(/\r\n?/g, "\n")
|
|
43
|
+
.replace(/\s+/g, " ")
|
|
44
|
+
.trim();
|
|
45
|
+
if (norm.length > 32768)
|
|
46
|
+
norm = norm.slice(0, 32768);
|
|
47
|
+
// Five-code-point shingles (deduped, capped at 50_000).
|
|
48
|
+
const grams = [];
|
|
49
|
+
if (norm.length > 0) {
|
|
50
|
+
const cps = Array.from(norm);
|
|
51
|
+
const seen = new Set();
|
|
52
|
+
const short = cps.length < 5;
|
|
53
|
+
for (let i = 0; i < (short ? 1 : cps.length - 4); i++) {
|
|
54
|
+
const h = short ? fnv1a64(norm) : fnv1a64(cps.slice(i, i + 5).join(""));
|
|
55
|
+
if (!seen.has(h)) {
|
|
56
|
+
seen.add(h);
|
|
57
|
+
grams.push(h);
|
|
58
|
+
}
|
|
59
|
+
if (!short && seen.size >= 50000)
|
|
60
|
+
break;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
// Signature: slot i = min over shingles of (a_i*x + b_i) mod p (exact BigInt).
|
|
64
|
+
const sig = new Array(SLOTS).fill(P);
|
|
65
|
+
for (let i = 0; i < SLOTS && grams.length > 0; i++) {
|
|
66
|
+
const pair = seedPairs[i];
|
|
67
|
+
const a = BigInt(pair.a);
|
|
68
|
+
const b = BigInt(pair.b);
|
|
69
|
+
let min = P;
|
|
70
|
+
for (const x of grams) {
|
|
71
|
+
const h = (a * x + b) % P;
|
|
72
|
+
if (h < min)
|
|
73
|
+
min = h;
|
|
74
|
+
}
|
|
75
|
+
sig[i] = min;
|
|
76
|
+
}
|
|
77
|
+
// Own 256 x u64 little-endian encoder.
|
|
78
|
+
const buf = Buffer.allocUnsafe(SLOTS * U64);
|
|
79
|
+
for (let i = 0; i < SLOTS; i++)
|
|
80
|
+
buf.writeBigUInt64LE(sig[i], i * U64);
|
|
81
|
+
const bytes = new Uint8Array(buf);
|
|
82
|
+
const digest = createHash("sha256").update(bytes).digest("hex");
|
|
83
|
+
// 64 buckets: FNV-1a over `<session>|v2|` + each 32-byte band (LE hex).
|
|
84
|
+
const buckets = [];
|
|
85
|
+
if (withBuckets) {
|
|
86
|
+
const prefix = Buffer.from(`${session}|v2|`, "utf8");
|
|
87
|
+
for (let band = 0; band < BANDS; band++) {
|
|
88
|
+
const bandBytes = Buffer.from(bytes.slice(band * 32, band * 32 + 32));
|
|
89
|
+
const out = Buffer.allocUnsafe(U64);
|
|
90
|
+
out.writeBigUInt64LE(fnv1a64(Buffer.concat([prefix, bandBytes])), 0);
|
|
91
|
+
buckets.push(`b${band}:${out.toString("hex")}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return { bytes, digest, buckets };
|
|
95
|
+
}
|