pi-mega-compact 0.17.1 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/vector-cortex.js +19 -0
- package/dist/config.js +117 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +4 -0
- package/dist/extensions/mega-events/context-handler/dbMirrorAppend.js +63 -0
- package/dist/extensions/mega-events/context-handler/gateCheck.js +59 -0
- package/dist/extensions/mega-events/context-handler/liveTrim.js +178 -0
- package/dist/extensions/mega-events/context-handler/pipelineRun.js +37 -0
- package/dist/extensions/mega-events/context-handler.js +39 -305
- package/dist/log.js +47 -0
- package/dist/src/config/vector-cortex.js +19 -0
- package/dist/src/config.js +1 -1
- package/dist/src/vector-cortex/encoder/asset.js +142 -0
- package/dist/src/vector-cortex/encoder/emit-vc2b.js +63 -0
- package/dist/src/vector-cortex/encoder/emit.js +42 -0
- package/dist/src/vector-cortex/encoder/heads.js +113 -0
- package/dist/src/vector-cortex/encoder/lexical.js +104 -0
- package/dist/src/vector-cortex/encoder/router.js +115 -0
- package/dist/src/vector-cortex/encoder/runtime.js +228 -0
- package/dist/src/vector-cortex/encoder/trigram.js +75 -0
- package/dist/src/vector-cortex/encoder/types.js +138 -0
- package/dist/vector-cortex/encoder/asset.js +142 -0
- package/dist/vector-cortex/encoder/emit-vc2b.js +63 -0
- package/dist/vector-cortex/encoder/emit.js +42 -0
- package/dist/vector-cortex/encoder/heads.js +113 -0
- package/dist/vector-cortex/encoder/lexical.js +104 -0
- package/dist/vector-cortex/encoder/router.js +115 -0
- package/dist/vector-cortex/encoder/runtime.js +228 -0
- package/dist/vector-cortex/encoder/trigram.js +75 -0
- package/dist/vector-cortex/encoder/types.js +138 -0
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +14 -0
- package/extensions/mega-events/context-handler/dbMirrorAppend.ts +93 -0
- package/extensions/mega-events/context-handler/gateCheck.ts +101 -0
- package/extensions/mega-events/context-handler/liveTrim.ts +241 -0
- package/extensions/mega-events/context-handler/pipelineRun.ts +79 -0
- package/extensions/mega-events/context-handler.ts +45 -347
- package/package.json +1 -1
- package/src/config/vector-cortex.ts +21 -0
- package/src/config.ts +2 -0
- package/src/vector-cortex/encoder/asset.ts +155 -0
- package/src/vector-cortex/encoder/emit-vc2b.ts +82 -0
- package/src/vector-cortex/encoder/emit.ts +51 -0
- package/src/vector-cortex/encoder/heads.ts +142 -0
- package/src/vector-cortex/encoder/lexical.ts +123 -0
- package/src/vector-cortex/encoder/router.ts +163 -0
- package/src/vector-cortex/encoder/runtime.ts +283 -0
- package/src/vector-cortex/encoder/trigram.ts +85 -0
- package/src/vector-cortex/encoder/types.ts +275 -0
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/trigram.ts — VC2B mode B: asset-free trigram encoder.
|
|
3
|
+
*
|
|
4
|
+
* Trigram B is a deterministic, asset-free (no learned model, no manifest, no
|
|
5
|
+
* calibration) 512-dim fixed feature encoding of a token/phrase sequence. It is
|
|
6
|
+
* the mode-B fallback selected when the learned asset (mode A) is removed,
|
|
7
|
+
* missing, unsupported, or digest-bad — and it never imports the learned asset
|
|
8
|
+
* or learned calibration (task 4). It derives directly from textual authority:
|
|
9
|
+
* the same document hashed via its byte-level trigrams yields the same 512-dim
|
|
10
|
+
* vector regardless of the asset state.
|
|
11
|
+
*
|
|
12
|
+
* Width is fixed at `ENCODER_TRIGRAM_WIDTH = 512` (VC2B task 4 "trigram B at 512
|
|
13
|
+
* dimensions"). The vector is L2-normalized; a zero-norm (empty) input maps to
|
|
14
|
+
* the all-zero vector, matching the heads convention of the VectorSet.
|
|
15
|
+
*
|
|
16
|
+
* Failure-triad independence: B's algorithm/index is distinct from A (learned
|
|
17
|
+
* projections) and C (token/phrase lexical) — it is a deterministic hashed
|
|
18
|
+
* n-gram bag-of-hashes, computed purely in-process with no external asset.
|
|
19
|
+
*
|
|
20
|
+
* Pi-agnostic, zero network (PREVENT-PI-004), no `any` (PREVENT-011).
|
|
21
|
+
*/
|
|
22
|
+
import { createHash } from "node:crypto";
|
|
23
|
+
import { l2Normalize } from "./heads.js";
|
|
24
|
+
import { createEncoderHeadsReporter, } from "./emit-vc2b.js";
|
|
25
|
+
/** Fixed output width of trigram B (VC2B task 4). */
|
|
26
|
+
export const ENCODER_TRIGRAM_WIDTH = 512;
|
|
27
|
+
/** Tokenize a phrase into byte-level trigrams (3-byte sliding windows). For a
|
|
28
|
+
* short phrase with fewer than 3 bytes we still emit the available shingles. */
|
|
29
|
+
function trigrams(text) {
|
|
30
|
+
const bytes = Buffer.from(text, "utf8");
|
|
31
|
+
if (bytes.length === 0)
|
|
32
|
+
return [];
|
|
33
|
+
const out = [];
|
|
34
|
+
const n = bytes.length;
|
|
35
|
+
// `Math.max(1, n - 2)` already emits a single whole-string shingle for 1- and
|
|
36
|
+
// 2-byte phrases (slice(0,3) covers the whole buffer), so there is NO separate
|
|
37
|
+
// short-phrase block — adding one would hash the same shingle twice (Q05).
|
|
38
|
+
for (let i = 0; i < Math.max(1, n - 2); i++) {
|
|
39
|
+
const chunk = bytes.slice(i, i + 3);
|
|
40
|
+
out.push(chunk.toString("hex"));
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Encode a phrase into a 512-dim L2-normalized trigram vector (all-zero on
|
|
46
|
+
* empty input). Deterministic: the same text always yields the same vector
|
|
47
|
+
* (repeat drift == 0) — no asset, no calibration, no network.
|
|
48
|
+
*/
|
|
49
|
+
export function embedTrigram512(text) {
|
|
50
|
+
const width = ENCODER_TRIGRAM_WIDTH;
|
|
51
|
+
const out = new Float32Array(width);
|
|
52
|
+
// Feistel-style double hashing of each trigram into a bucket index + weight.
|
|
53
|
+
for (const tg of trigrams(text)) {
|
|
54
|
+
const h1 = createHash("sha256").update(tg).digest();
|
|
55
|
+
const bucket = h1.readUInt32BE(0) % width;
|
|
56
|
+
const weight = (h1.readUInt32BE(4) / 4294967295) * 2 - 1;
|
|
57
|
+
out[bucket] += weight;
|
|
58
|
+
}
|
|
59
|
+
return l2Normalize(out);
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The 512-dim vector is produced even when the learned asset is absent: this is
|
|
63
|
+
* the mode-B selection point. Returns `{ ok: true, dim, width }` always — there
|
|
64
|
+
* is no asset to consult (task 4 + ENC-FALLBACK-003). Selecting mode B also
|
|
65
|
+
* emits `vector_cortex_encoder_fallback_selected` via the flag-gated reporter
|
|
66
|
+
* (task 5) — the production seam that makes the fallback event live in the
|
|
67
|
+
* runtime, not dead test-only wiring.
|
|
68
|
+
*/
|
|
69
|
+
export function selectTrigramBFallback(options = {}) {
|
|
70
|
+
const reporter = options.reporter ?? createEncoderHeadsReporter();
|
|
71
|
+
const selection = { ok: true, mode: "B", dim: ENCODER_TRIGRAM_WIDTH, width: ENCODER_TRIGRAM_WIDTH };
|
|
72
|
+
reporter.fallbackSelected({ mode: selection.mode, dim: selection.dim, width: selection.width });
|
|
73
|
+
return selection;
|
|
74
|
+
}
|
|
75
|
+
export { l2Normalize };
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/types.ts — VC2A contract (ModelManifestV1 /
|
|
3
|
+
* EncoderRuntime).
|
|
4
|
+
*
|
|
5
|
+
* The offline encoder runtime owns the learned-asset path (triad mode A: a
|
|
6
|
+
* qualified local ONNX). MODEL_ASSET.md is the normative target. This
|
|
7
|
+
* sprint (VC2A) ships the manifest + verification + shaped-inference contract;
|
|
8
|
+
* the trained weights are packaged in VC2C (MODEL_ASSET: "package.json changes
|
|
9
|
+
* occur only in VC2C"), but the verification, digest-before-load, platform
|
|
10
|
+
* demotion, shape rejection and RSS/latency budget all land here so a later
|
|
11
|
+
* sprint only substitutes real weights.
|
|
12
|
+
*
|
|
13
|
+
* Contract-first (ENGINEERING_PRACTICES §3): this types file is the reviewed
|
|
14
|
+
* gate; implementations import from it; consumers import only types + factory.
|
|
15
|
+
*
|
|
16
|
+
* Pi-agnostic and dependency-free (PREVENT-PI-004 — local assets only, the
|
|
17
|
+
* runtime never fetches). No `any` (PREVENT-011).
|
|
18
|
+
*/
|
|
19
|
+
/** Supported matrix from MODEL_ASSET.md §qualification. */
|
|
20
|
+
export const ENCODER_SUPPORTED_PLATFORMS = [
|
|
21
|
+
"linux-x64",
|
|
22
|
+
"linux-arm64",
|
|
23
|
+
"darwin-x64",
|
|
24
|
+
"darwin-arm64",
|
|
25
|
+
"win32-x64",
|
|
26
|
+
];
|
|
27
|
+
/** ONNX opset required by the normative v1 target (opset 17). */
|
|
28
|
+
export const ENCODER_OPSET = 17;
|
|
29
|
+
/** Batch must be exactly 1 (single-request inference). */
|
|
30
|
+
export const ENCODER_BATCH = 1;
|
|
31
|
+
/** Maximum accepted token count (WordPiece, deterministic truncation). */
|
|
32
|
+
export const ENCODER_MAX_TOKENS = 512;
|
|
33
|
+
/** Caps the encoder's MARGINAL footprint (bytes) at 150 MiB (MODEL_ASSET
|
|
34
|
+
* §qualification). The budget bounds the encoder's own incremental allocation
|
|
35
|
+
* (a reusable projection buffer + any externally staged asset working set),
|
|
36
|
+
* NOT the whole-process RSS — in a live pi extension the process baseline
|
|
37
|
+
* routinely exceeds 150 MiB, so measuring absolute RSS would make mode A
|
|
38
|
+
* unreachable in production. This is the "RSS" figure the acceptance metric
|
|
39
|
+
* and ENC_FAIL.RSS_BUDGET_EXCEEDED refer to: it is the encoder's marginal
|
|
40
|
+
* footprint, never the process RSS (code-review Q01/Q02). */
|
|
41
|
+
export const ENCODER_RSS_BUDGET_BYTES = 150 * 1024 * 1024;
|
|
42
|
+
/** p95 inference budget in milliseconds (MODEL_ASSET §qualification). */
|
|
43
|
+
export const ENCODER_LATENCY_P95_MS = 40;
|
|
44
|
+
/** Semantic projection head width (MODEL_ASSET: 384 float32 L2-normalized). */
|
|
45
|
+
export const ENCODER_SEMANTIC_WIDTH = 384;
|
|
46
|
+
/** Exact VC2A failure codes (returned, never thrown across the boundary). */
|
|
47
|
+
export const ENC_FAIL = {
|
|
48
|
+
/** opset != 17. */
|
|
49
|
+
OPSET_INVALID: "ENC_OPSET_INVALID",
|
|
50
|
+
/** batch != 1. */
|
|
51
|
+
BATCH_INVALID: "ENC_BATCH_INVALID",
|
|
52
|
+
/** maxTokens > 512. */
|
|
53
|
+
TOKENS_EXCEEDED: "ENC_TOKENS_EXCEEDED",
|
|
54
|
+
/** input token count > declared maxTokens / 512, or not batch 1. */
|
|
55
|
+
SHAPE_INVALID: "ENC_SHAPE_INVALID",
|
|
56
|
+
/** asset file unreadable (truncated during digest read, allocator failure). */
|
|
57
|
+
ASSET_UNREADABLE: "ENC_ASSET_UNREADABLE",
|
|
58
|
+
/** on-disk digest does not match the manifest (one-byte mutation). */
|
|
59
|
+
DIGEST_MISMATCH: "ENC_DIGEST_MISMATCH",
|
|
60
|
+
/** platform not in the supported matrix (selects trigram B). */
|
|
61
|
+
PLATFORM_UNSUPPORTED: "ENC_PLATFORM_UNSUPPORTED",
|
|
62
|
+
/** manifest missing/invalid (selects trigram B). */
|
|
63
|
+
MANIFEST_INVALID: "ENC_MANIFEST_INVALID",
|
|
64
|
+
/** encoder MARGINAL footprint over the 150 MiB budget (selects trigram B).
|
|
65
|
+
* This is the encoder's own incremental allocation (a reusable projection
|
|
66
|
+
* buffer + any externally staged asset working set), NOT whole-process RSS
|
|
67
|
+
* — see ENCODER_RSS_BUDGET_BYTES. */
|
|
68
|
+
RSS_BUDGET_EXCEEDED: "ENC_RSS_BUDGET_EXCEEDED",
|
|
69
|
+
/** mode C forced by the rollback path (MEGACOMPACT_VC2A=0 / forcedMode "C").
|
|
70
|
+
* Distinct from MANIFEST_INVALID so a non-corrupt, correctly-shaped asset
|
|
71
|
+
* present on disk is not mis-reported as "manifest invalid" when the runtime
|
|
72
|
+
* is simply rolled back to the predecessor path (code-review Q04). */
|
|
73
|
+
ROLLBACK: "ENC_ROLLBACK_ACTIVE",
|
|
74
|
+
};
|
|
75
|
+
/** The 8 registered VC2A conformance IDs (task 1: "register ENC-001..008"). */
|
|
76
|
+
export const ENC_IDS = [
|
|
77
|
+
"ENC-001",
|
|
78
|
+
"ENC-002",
|
|
79
|
+
"ENC-003",
|
|
80
|
+
"ENC-004",
|
|
81
|
+
"ENC-005",
|
|
82
|
+
"ENC-006",
|
|
83
|
+
"ENC-007",
|
|
84
|
+
"ENC-008",
|
|
85
|
+
];
|
|
86
|
+
// ---------------------------------------------------------------------------
|
|
87
|
+
// VC2B — multi-head encoder (VectorSetV1 / HeadCalibrationDraft).
|
|
88
|
+
// ---------------------------------------------------------------------------
|
|
89
|
+
/** The five independent projection heads in STABLE order (MODEL_ASSET
|
|
90
|
+
* §decision record; VC2B task 2 "stable order"). The array order is the
|
|
91
|
+
* normative ordering consumed by consumers: semantic, dependency,
|
|
92
|
+
* contradiction, cache-stability, payload-routing. */
|
|
93
|
+
export const ENCODER_HEAD_ORDER = [
|
|
94
|
+
"semantic",
|
|
95
|
+
"dependency",
|
|
96
|
+
"contradiction",
|
|
97
|
+
"cacheStability",
|
|
98
|
+
"payloadRouting",
|
|
99
|
+
];
|
|
100
|
+
/** The ordered per-head output dimensions: semantic 384, dependency 128,
|
|
101
|
+
* contradiction 128, cacheStability 64, payloadRouting 32 (VC2B task 2). */
|
|
102
|
+
export const ENCODER_HEAD_DIMS = {
|
|
103
|
+
semantic: 384,
|
|
104
|
+
dependency: 128,
|
|
105
|
+
contradiction: 128,
|
|
106
|
+
cacheStability: 64,
|
|
107
|
+
payloadRouting: 32,
|
|
108
|
+
};
|
|
109
|
+
/** Ordered dimension list matching ENCODER_HEAD_ORDER (384/128/128/64/32). */
|
|
110
|
+
export const ENCODER_HEAD_DIM_ORDER = ENCODER_HEAD_ORDER.map((h) => ENCODER_HEAD_DIMS[h]);
|
|
111
|
+
/**
|
|
112
|
+
* Weighted training losses per head (MODEL_ASSET §data/losses/calibration):
|
|
113
|
+
* semantic .35, dependency .20, contradiction .20, cache .15, payload .10.
|
|
114
|
+
* These are normative (VC2B task 3: "losses exactly .35/.20/.20/.15/.10").
|
|
115
|
+
*/
|
|
116
|
+
export const ENCODER_HEAD_LOSS_WEIGHTS = {
|
|
117
|
+
semantic: 0.35,
|
|
118
|
+
dependency: 0.2,
|
|
119
|
+
contradiction: 0.2,
|
|
120
|
+
cacheStability: 0.15,
|
|
121
|
+
payloadRouting: 0.1,
|
|
122
|
+
};
|
|
123
|
+
/** Sum of the five loss weights must be exactly 1.0 (asserted in tests). */
|
|
124
|
+
export const ENCODER_HEAD_LOSS_SUM = 1.0;
|
|
125
|
+
/** Deterministic seed shared by Python/NumPy training and ONNX export (VC2B
|
|
126
|
+
* task 3: "seed ... at 1729"). */
|
|
127
|
+
export const ENCODER_SEED = 1729;
|
|
128
|
+
/** The 16 registered VC2B conformance IDs (task 1: "register ENC-009..016"). */
|
|
129
|
+
export const ENC2B_IDS = [
|
|
130
|
+
"ENC-009",
|
|
131
|
+
"ENC-010",
|
|
132
|
+
"ENC-011",
|
|
133
|
+
"ENC-012",
|
|
134
|
+
"ENC-013",
|
|
135
|
+
"ENC-014",
|
|
136
|
+
"ENC-015",
|
|
137
|
+
"ENC-016",
|
|
138
|
+
];
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/asset.ts — VC2A asset verification (task 2).
|
|
3
|
+
*
|
|
4
|
+
* Verifies a ModelManifestV1 before any allocation: SHA-256 the ONNX and
|
|
5
|
+
* tokenizer against the manifest digests, require opset 17, batch exactly 1 and
|
|
6
|
+
* maximum 512 tokens, and confirm the current platform is in the supported
|
|
7
|
+
* matrix. On ANY of these the caller demotes to mode B (asset-free trigram) —
|
|
8
|
+
* never a remote fetch (PREVENT-PI-004). A truncated/unreadable asset during
|
|
9
|
+
* the digest read demotes with ENC_ASSET_UNREADABLE.
|
|
10
|
+
*
|
|
11
|
+
* Pi-agnostic. Filesystem reads only, zero network (PREVENT-PI-004).
|
|
12
|
+
*/
|
|
13
|
+
import { createHash } from "node:crypto";
|
|
14
|
+
import { readFileSync, statSync } from "node:fs";
|
|
15
|
+
import { join } from "node:path";
|
|
16
|
+
import { ENC_FAIL, ENCODER_BATCH, ENCODER_MAX_TOKENS, ENCODER_OPSET, ENCODER_SUPPORTED_PLATFORMS, } from "./types.js";
|
|
17
|
+
/** True when `p` is a single basename: non-empty, no path separators, no "..",
|
|
18
|
+
* no leading dot-segment traversal. Keeps manifest-controlled asset paths
|
|
19
|
+
* confined to the asset directory (no path-traversal via join()). */
|
|
20
|
+
function isBasename(p) {
|
|
21
|
+
if (!p || p.length === 0 || p.includes("/") || p.includes("\\"))
|
|
22
|
+
return false;
|
|
23
|
+
if (p === "." || p === "..")
|
|
24
|
+
return false;
|
|
25
|
+
return true;
|
|
26
|
+
}
|
|
27
|
+
/** Digest the on-disk bytes of one asset file; "" on unreadable (truncated). */
|
|
28
|
+
function digestFile(path) {
|
|
29
|
+
try {
|
|
30
|
+
const buf = readFileSync(path);
|
|
31
|
+
return createHash("sha256").update(buf).digest("hex");
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
// Truncated / unreadable during the digest read -> ENC_ASSET_UNREADABLE.
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Detect the current platform (MODEL_ASSET supported matrix). Unrecognized
|
|
40
|
+
* hosts return null so verification demotes to mode B (unsupported platform).
|
|
41
|
+
*/
|
|
42
|
+
export function detectPlatform(host = process.platform, arch = process.arch) {
|
|
43
|
+
if (!host || !arch)
|
|
44
|
+
return null;
|
|
45
|
+
// Normalize "win32"/"win32"/"linux"/"darwin" + "x64"/"arm64".
|
|
46
|
+
const plat = host === "win32" ? "win32" : host === "darwin" ? "darwin" : host === "linux" ? "linux" : "";
|
|
47
|
+
const a = arch === "x64" ? "x64" : arch === "arm64" ? "arm64" : "";
|
|
48
|
+
if (!plat || !a)
|
|
49
|
+
return null;
|
|
50
|
+
const candidate = `${plat}-${a}`;
|
|
51
|
+
return ENCODER_SUPPORTED_PLATFORMS.includes(candidate)
|
|
52
|
+
? candidate
|
|
53
|
+
: null;
|
|
54
|
+
}
|
|
55
|
+
function isManifest(m) {
|
|
56
|
+
const o = m;
|
|
57
|
+
return (!!o &&
|
|
58
|
+
typeof o === "object" &&
|
|
59
|
+
o.schema === "model-manifest-v1" &&
|
|
60
|
+
typeof o.opset === "number" &&
|
|
61
|
+
typeof o.batch === "number" &&
|
|
62
|
+
typeof o.maxTokens === "number" &&
|
|
63
|
+
typeof o.platform === "string" &&
|
|
64
|
+
!!o.onnx &&
|
|
65
|
+
!!o.tokenizer &&
|
|
66
|
+
typeof o.onnx.path === "string" &&
|
|
67
|
+
typeof o.onnx.sha256 === "string" &&
|
|
68
|
+
typeof o.tokenizer.path === "string" &&
|
|
69
|
+
typeof o.tokenizer.sha256 === "string");
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Verify the asset manifest + digest + constraints BEFORE allocation.
|
|
73
|
+
*
|
|
74
|
+
* - manifest parse/shape failure -> ENC_MANIFEST_INVALID -> mode B
|
|
75
|
+
* - unsupported platform -> ENC_PLATFORM_UNSUPPORTED -> mode B
|
|
76
|
+
* - opset != 17 -> ENC_OPSET_INVALID -> mode B
|
|
77
|
+
* - batch != 1 -> ENC_BATCH_INVALID -> mode B
|
|
78
|
+
* - maxTokens > 512 -> ENC_TOKENS_EXCEEDED -> mode B
|
|
79
|
+
* - on-disk digest != manifest digest -> ENC_DIGEST_MISMATCH (one-byte mutation)
|
|
80
|
+
* - unreadable/truncated file -> ENC_ASSET_UNREADABLE
|
|
81
|
+
*
|
|
82
|
+
* Returns ok only when EVERY constraint passes and both files hash to the
|
|
83
|
+
* declared digests (the "only batch1/max512 verified assets reach inference"
|
|
84
|
+
* invariant). The ok result surfaces the verified manifest's `maxTokens`
|
|
85
|
+
* (<= 512) so the runtime can enforce the per-manifest token capacity at
|
|
86
|
+
* inference (Q03), not just the global 512 ceiling.
|
|
87
|
+
*/
|
|
88
|
+
export function verifyEncoderAsset(assetDir, manifest, platform = detectPlatform()) {
|
|
89
|
+
if (typeof manifest !== "object" || manifest === null || !isManifest(manifest)) {
|
|
90
|
+
return { ok: false, code: ENC_FAIL.MANIFEST_INVALID };
|
|
91
|
+
}
|
|
92
|
+
if (!platform)
|
|
93
|
+
return { ok: false, code: ENC_FAIL.PLATFORM_UNSUPPORTED };
|
|
94
|
+
// The manifest's declared platform must match the runtime host (per-platform
|
|
95
|
+
// asset pinning): a bundle cross-shipped to the wrong arch is not qualified.
|
|
96
|
+
if (manifest.platform !== platform)
|
|
97
|
+
return { ok: false, code: ENC_FAIL.PLATFORM_UNSUPPORTED };
|
|
98
|
+
if (manifest.opset !== ENCODER_OPSET)
|
|
99
|
+
return { ok: false, code: ENC_FAIL.OPSET_INVALID };
|
|
100
|
+
if (manifest.batch !== ENCODER_BATCH)
|
|
101
|
+
return { ok: false, code: ENC_FAIL.BATCH_INVALID };
|
|
102
|
+
if (manifest.maxTokens > ENCODER_MAX_TOKENS)
|
|
103
|
+
return { ok: false, code: ENC_FAIL.TOKENS_EXCEEDED };
|
|
104
|
+
// Constrain asset paths to basenames (no separators / no '..') so a forged
|
|
105
|
+
// manifest cannot read digests from arbitrary paths off the asset dir.
|
|
106
|
+
if (!isBasename(manifest.onnx.path) || !isBasename(manifest.tokenizer.path)) {
|
|
107
|
+
return { ok: false, code: ENC_FAIL.MANIFEST_INVALID };
|
|
108
|
+
}
|
|
109
|
+
const onnxPath = join(assetDir, manifest.onnx.path);
|
|
110
|
+
const onnxDigest = digestFile(onnxPath);
|
|
111
|
+
if (onnxDigest === null)
|
|
112
|
+
return { ok: false, code: ENC_FAIL.ASSET_UNREADABLE };
|
|
113
|
+
if (onnxDigest !== manifest.onnx.sha256)
|
|
114
|
+
return { ok: false, code: ENC_FAIL.DIGEST_MISMATCH };
|
|
115
|
+
const tokPath = join(assetDir, manifest.tokenizer.path);
|
|
116
|
+
const tokDigest = digestFile(tokPath);
|
|
117
|
+
if (tokDigest === null)
|
|
118
|
+
return { ok: false, code: ENC_FAIL.ASSET_UNREADABLE };
|
|
119
|
+
if (tokDigest !== manifest.tokenizer.sha256)
|
|
120
|
+
return { ok: false, code: ENC_FAIL.DIGEST_MISMATCH };
|
|
121
|
+
let embeddedBytes = 0;
|
|
122
|
+
try {
|
|
123
|
+
embeddedBytes = statSync(onnxPath).size + statSync(tokPath).size;
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return { ok: false, code: ENC_FAIL.ASSET_UNREADABLE };
|
|
127
|
+
}
|
|
128
|
+
return { ok: true, embeddedBytes, maxTokens: manifest.maxTokens, onnxDigest, tokenizerDigest: tokDigest };
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Read + shape-check a committed ModelManifestV1 from an asset directory.
|
|
132
|
+
* Returns the parsed manifest or null when the file is absent/malformed.
|
|
133
|
+
*/
|
|
134
|
+
export function readEncoderManifest(assetDir) {
|
|
135
|
+
try {
|
|
136
|
+
const raw = JSON.parse(readFileSync(join(assetDir, "manifest.json"), "utf8"));
|
|
137
|
+
return isManifest(raw) ? raw : null;
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/emit-vc2b.ts — VC2B observability seam.
|
|
3
|
+
*
|
|
4
|
+
* Owns the two VC2B events (task 5), gated on `MEGACOMPACT_VC2B` so the
|
|
5
|
+
* flag-OFF path emits zero events (mode C parity, byte-identical predecessor):
|
|
6
|
+
*
|
|
7
|
+
* vector_cortex_encoder_heads_emitted — a multi-head VectorSetV1 produced
|
|
8
|
+
* vector_cortex_encoder_fallback_selected — a mode B/C fallback selected
|
|
9
|
+
*
|
|
10
|
+
* No dashboard or API change is necessary for this internal sprint (task 5).
|
|
11
|
+
* Every event is a JSON line with `ts` + `event` (ENGINEERING_PRACTICES §8); the
|
|
12
|
+
* emitters are non-fatal (never break the agent loop). Pi-agnostic, zero network
|
|
13
|
+
* (PREVENT-PI-004), no `any` (PREVENT-011).
|
|
14
|
+
*/
|
|
15
|
+
import { VC2B_ENABLED } from "../../config/vector-cortex.js";
|
|
16
|
+
import { Logger } from "../../log.js";
|
|
17
|
+
/** A flag-gated no-op reporter (zero emissions, structural no-op). */
|
|
18
|
+
export const NOOP_VC2B_REPORTER = {
|
|
19
|
+
headsEmitted: () => { },
|
|
20
|
+
fallbackSelected: () => { },
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* The default emitter: routes both VC2B events into the append-only structured
|
|
24
|
+
* logger (`src/log.ts`) as JSON lines with `ts` + `event`. Supplying `emit:` to
|
|
25
|
+
* `createEncoderHeadsReporter` replaces this with a caller-provided sink (used
|
|
26
|
+
* by tests and downstream consumers). Making the default a REAL producer means a
|
|
27
|
+
* caller that just invokes the producer seam (`encodeOrFallback`, `encodeVectorSet`,
|
|
28
|
+
* `selectTrigramBFallback`, `selectLexicalC`) without injecting an emitter still
|
|
29
|
+
* yields structured telemetry instead of silently dropping every event (task 5,
|
|
30
|
+
* code-review Q01). Best-effort: the logger swallows all I/O errors.
|
|
31
|
+
*/
|
|
32
|
+
function defaultEmitFor(logPath) {
|
|
33
|
+
const logger = new Logger(logPath === undefined ? {} : { path: logPath });
|
|
34
|
+
return (event, fields) => {
|
|
35
|
+
logger.info(event, fields);
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Flag-gated emit, defaulting to a real logger-backed sink. The returned
|
|
40
|
+
* reporter is itself flag-gated (`VC2B_ENABLED`), so wiring it into a producer
|
|
41
|
+
* seam yields zero emissions when `MEGACOMPACT_VC2B=0` (byte-identical to the
|
|
42
|
+
* predecessor). Pass an explicit `emit` to route elsewhere (tests, downstream
|
|
43
|
+
* consumers); omit it to emit real structured log lines (Q01: the default is a
|
|
44
|
+
* live producer, not a silent no-op). `opts.logPath` only redirects the default
|
|
45
|
+
* sink and is ignored when `emit` is supplied.
|
|
46
|
+
*/
|
|
47
|
+
export function createEncoderHeadsReporter(emit, opts = {}) {
|
|
48
|
+
const sink = emit ?? defaultEmitFor(opts.logPath);
|
|
49
|
+
const fire = (event, fields) => {
|
|
50
|
+
if (!VC2B_ENABLED())
|
|
51
|
+
return;
|
|
52
|
+
try {
|
|
53
|
+
sink(event, { ...fields, ts: new Date().toISOString() });
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
/* non-fatal observability */
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
return {
|
|
60
|
+
headsEmitted: (fields) => fire("vector_cortex_encoder_heads_emitted", fields),
|
|
61
|
+
fallbackSelected: (fields) => fire("vector_cortex_encoder_fallback_selected", fields),
|
|
62
|
+
};
|
|
63
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/emit.ts — VC2A observability seam.
|
|
3
|
+
*
|
|
4
|
+
* Emits the two VC2A structured events, gated on `MEGACOMPACT_VC2A` (mode C
|
|
5
|
+
* parity: flag OFF => zero emissions). Every event is a JSON line with `ts` +
|
|
6
|
+
* `event` (ENGINEERING_PRACTICES §8); the emitters are never fatal on consumer
|
|
7
|
+
* failure (non-fatal observability, never breaks the agent loop).
|
|
8
|
+
*
|
|
9
|
+
* vector_cortex_encoder_asset_verified — a qualified manifest+digest load
|
|
10
|
+
* vector_cortex_encoder_runtime_demoted — a demotion to mode B or C
|
|
11
|
+
*
|
|
12
|
+
* No network, no side effects beyond the supplied emit callback
|
|
13
|
+
* (PREVENT-PI-004 / PREVENT-011).
|
|
14
|
+
*/
|
|
15
|
+
import { VC2A_ENABLED } from "../../config/vector-cortex.js";
|
|
16
|
+
/** A flag-gated no-op reporter (zero emissions, default when none injected). */
|
|
17
|
+
export const NOOP_ENCODER_REPORTER = {
|
|
18
|
+
assetVerified: () => { },
|
|
19
|
+
runtimeDemoted: () => { },
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Flag-gated emit: no-op when VC2A is off or no emitter is supplied. The
|
|
23
|
+
* returned reporter is itself flag-gated (`VC2A_ENABLED`), so wiring it into a
|
|
24
|
+
* runtime seam yields zero emissions when `MEGACOMPACT_VC2A=0` (byte-identical
|
|
25
|
+
* to the predecessor).
|
|
26
|
+
*/
|
|
27
|
+
export function createEncoderReporter(emit) {
|
|
28
|
+
const fire = (event, fields) => {
|
|
29
|
+
if (!VC2A_ENABLED())
|
|
30
|
+
return;
|
|
31
|
+
try {
|
|
32
|
+
emit?.(event, { ...fields, ts: new Date().toISOString() });
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
/* non-fatal observability */
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
return {
|
|
39
|
+
assetVerified: (fields) => fire("vector_cortex_encoder_asset_verified", fields),
|
|
40
|
+
runtimeDemoted: (fields) => fire("vector_cortex_encoder_runtime_demoted", fields),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vector-cortex/encoder/heads.ts — VC2B multi-head encoder (tasks 1–2).
|
|
3
|
+
*
|
|
4
|
+
* Produces a `VectorSetV1`: five independent L2-normalized projection heads in
|
|
5
|
+
* STABLE order — semantic 384, dependency 128, contradiction 128, cacheStability
|
|
6
|
+
* 64, payloadRouting 32 (MODEL_ASSET §decision record). Each head L2-normalizes
|
|
7
|
+
* its raw projection; a zero-norm projection maps to an all-zero vector (task 2).
|
|
8
|
+
*
|
|
9
|
+
* The raw per-head projection is a deterministic seeded compression of the input
|
|
10
|
+
* token sequence (seeded by `ENCODER_SEED` and the head's stable index), which
|
|
11
|
+
* mirrors the VC2A `projectSemantic` placeholder pattern: the contract, shape
|
|
12
|
+
* gating, normalization, zero-norm mapping, ordering and loss/seed constants are
|
|
13
|
+
* all normative here; real trained weights are substituted in VC2C. This keeps
|
|
14
|
+
* the mode-A multi-head path fully testable end-to-end today with zero network.
|
|
15
|
+
*
|
|
16
|
+
* The VC2B emit seam (task 5) is wired: producing a VectorSetV1 emits
|
|
17
|
+
* `vector_cortex_encoder_heads_emitted`; selecting a mode B/C fallback emits
|
|
18
|
+
* `vector_cortex_encoder_fallback_selected` — both gated on MEGACOMPACT_VC2B.
|
|
19
|
+
*
|
|
20
|
+
* Pi-agnostic, zero network (PREVENT-PI-004), no `any` (PREVENT-011).
|
|
21
|
+
*/
|
|
22
|
+
import { ENCODER_HEAD_DIMS, ENCODER_HEAD_ORDER, ENCODER_HEAD_LOSS_WEIGHTS, ENCODER_HEAD_LOSS_SUM, ENCODER_SEED, } from "./types.js";
|
|
23
|
+
import { createEncoderHeadsReporter, NOOP_VC2B_REPORTER, } from "./emit-vc2b.js";
|
|
24
|
+
/** The stable head index of a head name (its position in ENCODER_HEAD_ORDER). */
|
|
25
|
+
const HEAD_INDEX = {
|
|
26
|
+
semantic: 0,
|
|
27
|
+
dependency: 1,
|
|
28
|
+
contradiction: 2,
|
|
29
|
+
cacheStability: 3,
|
|
30
|
+
payloadRouting: 4,
|
|
31
|
+
};
|
|
32
|
+
/** Deterministic 32-bit LCG step (matches runtime.ts projectSemantic). */
|
|
33
|
+
function nextState(state) {
|
|
34
|
+
return (state * 1664525 + 1013904223) >>> 0;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* L2-normalize a float vector in place semantics (returns a new Float32Array).
|
|
38
|
+
* A zero-norm (or empty) input maps to an all-zero vector of the same length
|
|
39
|
+
* (task 2: "mapping zero norm to an all-zero vector"). All finite.
|
|
40
|
+
*/
|
|
41
|
+
export function l2Normalize(values) {
|
|
42
|
+
const out = new Float32Array(values.length);
|
|
43
|
+
let sum = 0;
|
|
44
|
+
for (const v of values)
|
|
45
|
+
sum += v * v;
|
|
46
|
+
const norm = Math.sqrt(sum);
|
|
47
|
+
if (!(norm > 0))
|
|
48
|
+
return out; // zero norm -> all-zero
|
|
49
|
+
for (let i = 0; i < values.length; i++)
|
|
50
|
+
out[i] = values[i] / norm;
|
|
51
|
+
return out;
|
|
52
|
+
}
|
|
53
|
+
/** L2 norm of a Float32Array (0 for empty/all-zero). */
|
|
54
|
+
export function l2Norm(values) {
|
|
55
|
+
let sum = 0;
|
|
56
|
+
for (const v of values)
|
|
57
|
+
sum += v * v;
|
|
58
|
+
return Math.sqrt(sum);
|
|
59
|
+
}
|
|
60
|
+
/** A deterministic per-head projection over the token sequence, pre-normalization. */
|
|
61
|
+
function projectRaw(head, tokens, seed) {
|
|
62
|
+
const dim = ENCODER_HEAD_DIMS[head];
|
|
63
|
+
const out = new Float32Array(dim);
|
|
64
|
+
// An EMPTY token sequence has no signal: the raw projection is the zero vector,
|
|
65
|
+
// so after L2 normalization it maps to the all-zero vector (task 2: "mapping
|
|
66
|
+
// zero norm to an all-zero vector"; ENC-ZERO-002). This keeps empty input
|
|
67
|
+
// finite and zero-norm instead of seeding spurious unit-norm noise.
|
|
68
|
+
if (tokens.length === 0)
|
|
69
|
+
return out;
|
|
70
|
+
// Mix the stable head index + ENCODER_SEED + seed into a per-head state so
|
|
71
|
+
// each head is a distinct independent projection (failure-triad independence).
|
|
72
|
+
let state = (((ENCODER_SEED ^ HEAD_INDEX[head]) >>> 0) ^ (seed >>> 0)) ^ 0x9e3779b9;
|
|
73
|
+
for (const t of tokens)
|
|
74
|
+
state = nextState(state ^ ((t >>> 0) * 2654435761));
|
|
75
|
+
for (let i = 0; i < dim; i++) {
|
|
76
|
+
state = nextState(state ^ seed);
|
|
77
|
+
out[i] = (state / 4294967296) * 2 - 1;
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Compute one head's L2-normalized vector (all-zero on zero norm) for a token
|
|
83
|
+
* sequence. Deterministic for a given seed (repeat drift == 0).
|
|
84
|
+
*/
|
|
85
|
+
export function projectHead(head, tokens, seed = ENCODER_SEED) {
|
|
86
|
+
const raw = projectRaw(head, tokens, seed);
|
|
87
|
+
return { head, dim: ENCODER_HEAD_DIMS[head], values: l2Normalize(raw) };
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Encode a token sequence into a `VectorSetV1`: the five heads in stable order,
|
|
91
|
+
* each L2-normalized (all-zero on zero norm). Emits `heads_emitted` via the
|
|
92
|
+
* reporter (non-fatal, flag-gated). Deterministic for a given seed.
|
|
93
|
+
*/
|
|
94
|
+
export function encodeVectorSet(tokens, options = {}) {
|
|
95
|
+
const seed = options.seed ?? ENCODER_SEED;
|
|
96
|
+
const reporter = options.reporter ?? createEncoderHeadsReporter();
|
|
97
|
+
const heads = ENCODER_HEAD_ORDER.map((h) => projectHead(h, tokens, seed));
|
|
98
|
+
reporter.headsEmitted({
|
|
99
|
+
heads: heads.length,
|
|
100
|
+
dims: heads.map((h) => h.dim).join("/"),
|
|
101
|
+
normalized: true,
|
|
102
|
+
tokens: tokens.length,
|
|
103
|
+
});
|
|
104
|
+
return { schema: "vector-set-v1", inputTokens: [...tokens], heads, normalized: true };
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* The per-head loss weights (must sum to ENCODER_HEAD_LOSS_SUM exactly).
|
|
108
|
+
* Exposed for training/tests to assert the normative .35/.20/.20/.15/.10 split.
|
|
109
|
+
*/
|
|
110
|
+
export function headLossWeights() {
|
|
111
|
+
return { ...ENCODER_HEAD_LOSS_WEIGHTS };
|
|
112
|
+
}
|
|
113
|
+
export { ENCODER_HEAD_ORDER, ENCODER_HEAD_DIMS, ENCODER_HEAD_LOSS_SUM, ENCODER_SEED, NOOP_VC2B_REPORTER };
|