pi-mega-compact 0.4.4 → 0.4.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/extensions/dashboard-server.js +450 -0
  2. package/dist/extensions/dashboard-server.test.js +111 -0
  3. package/dist/extensions/error-patterns.js +115 -0
  4. package/dist/extensions/mega-compact.js +782 -0
  5. package/dist/extensions/mega-compact.test.js +328 -0
  6. package/dist/extensions/openclaw-mega-compact.js +291 -0
  7. package/dist/src/adapt.js +106 -0
  8. package/dist/src/boundary.js +88 -0
  9. package/dist/src/boundary.test.js +53 -0
  10. package/dist/src/canary.js +118 -0
  11. package/dist/src/compact.js +250 -0
  12. package/dist/src/compact.test.js +78 -0
  13. package/dist/src/config/dedup.js +81 -0
  14. package/dist/src/config.js +12 -0
  15. package/dist/src/dedup/dedup.test.js +41 -0
  16. package/dist/src/dedup/digest.js +30 -0
  17. package/dist/src/dedup/l1-lsh.js +52 -0
  18. package/dist/src/dedup/l1-minhash.js +91 -0
  19. package/dist/src/dedup/l1-verify.js +54 -0
  20. package/dist/src/dedup/l1.test.js +50 -0
  21. package/dist/src/dedup/mmr.js +45 -0
  22. package/dist/src/dedup/normalize.js +39 -0
  23. package/dist/src/dedup/raptor/guardrails.js +83 -0
  24. package/dist/src/dedup/raptor/index.js +94 -0
  25. package/dist/src/dedup/raptor/kmeans.js +152 -0
  26. package/dist/src/dedup/raptor/raptor.test.js +205 -0
  27. package/dist/src/dedup/raptor/retrieval.js +81 -0
  28. package/dist/src/dedup/raptor/summarizer.js +85 -0
  29. package/dist/src/dedup/raptor/tree.js +177 -0
  30. package/dist/src/dedup/sprint12.test.js +219 -0
  31. package/dist/src/dedup/topk.js +60 -0
  32. package/dist/src/dedup-engine.test.js +447 -0
  33. package/dist/src/e2e.test.js +698 -0
  34. package/dist/src/embedder.js +102 -0
  35. package/dist/src/engine.js +137 -0
  36. package/dist/src/engine.test.js +111 -0
  37. package/dist/src/extractive.js +209 -0
  38. package/dist/src/extractive.test.js +130 -0
  39. package/dist/src/httpEmbedder.js +143 -0
  40. package/dist/src/log.js +47 -0
  41. package/dist/src/log.test.js +42 -0
  42. package/dist/src/minilm.js +92 -0
  43. package/dist/src/monitoring.js +131 -0
  44. package/dist/src/ratio.bench.test.js +897 -0
  45. package/dist/src/recall.integration.test.js +77 -0
  46. package/dist/src/recall.js +60 -0
  47. package/dist/src/recall.test.js +50 -0
  48. package/dist/src/sprint14.test.js +219 -0
  49. package/dist/src/store/backfill.js +189 -0
  50. package/dist/src/store/bloom.js +114 -0
  51. package/dist/src/store/compression.js +177 -0
  52. package/dist/src/store/compression.test.js +67 -0
  53. package/dist/src/store/integrity.js +44 -0
  54. package/dist/src/store/migrate.js +79 -0
  55. package/dist/src/store/migrate.test.js +139 -0
  56. package/dist/src/store/sprint10.test.js +186 -0
  57. package/dist/src/store/sqlite.js +574 -0
  58. package/dist/src/store.js +115 -0
  59. package/dist/src/store.test.js +142 -0
  60. package/dist/src/supersede.js +68 -0
  61. package/dist/src/supersede.test.js +36 -0
  62. package/dist/src/tokens.js +31 -0
  63. package/dist/src/types.js +8 -0
  64. package/dist/src/types.test.js +9 -0
  65. package/dist/src/vectorStore.js +465 -0
  66. package/dist/src/vectorStore.test.js +479 -0
  67. package/dist/src/wordpiece.js +129 -0
  68. package/extensions/mega-compact.ts +9 -3
  69. package/package.json +4 -2
@@ -0,0 +1,130 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { extractiveSummarize } from "./extractive.js";
4
+ function msg(role, text, toolName) {
5
+ return toolName ? { role, text, toolName, input: text, output: text } : { role, text };
6
+ }
7
+ // ---- Determinism -----------------------------------------------------------
8
+ test("extractive summary is deterministic", () => {
9
+ const messages = [
10
+ msg("user", "please write src/index.ts"),
11
+ msg("assistant", "I'll write src/index.ts now."),
12
+ msg("tool", '{"file_path":"src/index.ts"}', "write"),
13
+ msg("assistant", "Done."),
14
+ ];
15
+ const s1 = extractiveSummarize(messages);
16
+ const s2 = extractiveSummarize(messages);
17
+ assert.deepStrictEqual(s1, s2);
18
+ });
19
+ // ---- Compression -----------------------------------------------------------
20
+ test("extractive summary produces small output", () => {
21
+ // Build 70 messages (simulating a session)
22
+ const messages = [];
23
+ for (let i = 0; i < 70; i++) {
24
+ messages.push(msg("user", `request ${i}: please help with feature ${i}`));
25
+ messages.push(msg("assistant", `working on feature ${i} in src/file${i % 5}.ts`));
26
+ messages.push(msg("tool", `{"file_path":"src/file${i % 5}.ts","content":"..."}`, "write"));
27
+ messages.push(msg("assistant", `done with feature ${i}`));
28
+ }
29
+ const rawText = messages.map((m) => m.text).join("\n");
30
+ const rawTokens = Math.ceil(rawText.length / 4);
31
+ const summary = extractiveSummarize(messages);
32
+ const ratio = rawTokens / summary.tokenEstimate;
33
+ // Compression should be at least 5:1 (target is 35:1)
34
+ assert.ok(ratio >= 5, `compression ratio ${ratio.toFixed(1)}:1 is less than 5:1`);
35
+ assert.ok(summary.tokenEstimate < 5000, `summary is ${summary.tokenEstimate} tokens (expected < 5000)`);
36
+ });
37
+ // ---- Empty input ------------------------------------------------------------
38
+ test("empty messages returns minimal summary", () => {
39
+ const summary = extractiveSummarize([]);
40
+ assert.equal(summary.topicSummary, "(empty)");
41
+ assert.equal(summary.keyDecisions.length, 0);
42
+ assert.equal(summary.nextSteps.length, 0);
43
+ assert.equal(summary.filesModified.length, 0);
44
+ assert.equal(summary.tokenEstimate, 0);
45
+ });
46
+ // ---- Key decisions ----------------------------------------------------------
47
+ test("extracts decisions from assistant messages", () => {
48
+ const messages = [
49
+ msg("user", "which database should we use?"),
50
+ msg("assistant", "I recommend using better-sqlite3 for the local vector store."),
51
+ msg("assistant", "Let's go with the Trident pipeline architecture."),
52
+ ];
53
+ const summary = extractiveSummarize(messages);
54
+ assert.ok(summary.keyDecisions.length >= 1, "should extract at least 1 decision");
55
+ assert.ok(summary.keyDecisions.some((d) => d.includes("better-sqlite3")), `decisions: ${JSON.stringify(summary.keyDecisions)}`);
56
+ });
57
+ test("no decisions in tool messages", () => {
58
+ const messages = [
59
+ msg("tool", "I recommend something", "bash"),
60
+ ];
61
+ const summary = extractiveSummarize(messages);
62
+ // Tool messages should not be checked for decisions
63
+ assert.equal(summary.keyDecisions.length, 0);
64
+ });
65
+ // ---- Files modified ---------------------------------------------------------
66
+ test("extracts files from write/edit tool calls", () => {
67
+ const messages = [
68
+ msg("tool", '{"file_path":"/home/user/project/src/index.ts","content":"..."}', "write"),
69
+ msg("tool", '{"file_path":"/home/user/project/README.md","content":"..."}', "edit"),
70
+ ];
71
+ const summary = extractiveSummarize(messages);
72
+ assert.ok(summary.filesModified.includes("/home/user/project/src/index.ts"));
73
+ assert.ok(summary.filesModified.includes("/home/user/project/README.md"));
74
+ });
75
+ test("extracts files from git commands in bash", () => {
76
+ const messages = [
77
+ msg("tool", "git add src/index.ts src/types.ts", "bash"),
78
+ ];
79
+ const summary = extractiveSummarize(messages);
80
+ assert.ok(summary.filesModified.some((f) => f.includes("index.ts")));
81
+ });
82
+ // ---- Pending work -----------------------------------------------------------
83
+ test("extracts pending work markers", () => {
84
+ const messages = [
85
+ msg("user", "run the tests"),
86
+ msg("assistant", "Tests pass. TODO: add integration tests for the dedup path."),
87
+ ];
88
+ const summary = extractiveSummarize(messages);
89
+ assert.ok(summary.nextSteps.length >= 1, "should find TODO");
90
+ assert.ok(summary.nextSteps.some((s) => /integration tests/i.test(s)));
91
+ });
92
+ // ---- topicSummary structure -------------------------------------------------
93
+ test("topicSummary contains scope line", () => {
94
+ const messages = [
95
+ msg("user", "hello"),
96
+ msg("assistant", "hi there"),
97
+ ];
98
+ const summary = extractiveSummarize(messages);
99
+ assert.ok(summary.topicSummary.includes("Conversation: 2 messages"));
100
+ assert.ok(summary.topicSummary.includes("1 user"));
101
+ assert.ok(summary.topicSummary.includes("1 assistant"));
102
+ });
103
+ test("topicSummary includes tools when present", () => {
104
+ const messages = [
105
+ msg("tool", "ok", "write"),
106
+ msg("tool", "ok", "bash"),
107
+ ];
108
+ const summary = extractiveSummarize(messages);
109
+ assert.ok(summary.topicSummary.includes("Tools:"));
110
+ assert.ok(summary.topicSummary.includes("write"));
111
+ assert.ok(summary.topicSummary.includes("bash"));
112
+ });
113
+ // ---- Same messages produce same summary ------------------------------------
114
+ test("deterministic across invocations with complex input", () => {
115
+ const messages = [
116
+ msg("user", "help me refactor the auth module"),
117
+ msg("assistant", "I'll look at the current auth implementation."),
118
+ msg("tool", '{"file_path":"src/auth.ts"}', "read"),
119
+ msg("assistant", "I recommend splitting auth.ts into separate files."),
120
+ msg("user", "sounds good, go ahead"),
121
+ msg("assistant", "I'll create src/auth/login.ts and src/auth/register.ts."),
122
+ msg("tool", '{"file_path":"src/auth/login.ts","content":"..."}', "write"),
123
+ msg("tool", '{"file_path":"src/auth/register.ts","content":"..."}', "write"),
124
+ msg("assistant", "Done. TODO: update the import paths in main.ts."),
125
+ ];
126
+ const results = Array.from({ length: 5 }, () => extractiveSummarize(messages));
127
+ for (let i = 1; i < results.length; i++) {
128
+ assert.deepStrictEqual(results[i], results[0], `run ${i} differs from run 0`);
129
+ }
130
+ });
@@ -0,0 +1,143 @@
1
+ /**
2
+ * httpEmbedder.ts — pluggable LOCALHOST embeddings client (Sprint 12, BYO).
3
+ *
4
+ * Lets the user bring their own embedding backend WITHOUT this extension
5
+ * shipping a model, a native dependency, or a remote call. The backend is a
6
+ * localhost HTTP server the user runs themselves (local ONNX/TEI/llamafile/
7
+ * Ollama-embeddings/…) and points us at via MEGACOMPACT_EMBEDDING_URL.
8
+ *
9
+ * This honors PREVENT-PI-004 (critical: local-only, zero remote network): the
10
+ * only allowed network is a user-spawned localhost endpoint, in the same
11
+ * exception class as the optional /dashboard UI server. It is NOT a remote
12
+ * provider call — compacted conversation content never leaves the machine.
13
+ *
14
+ * The endpoint contract (OpenAI-style, tolerant parser):
15
+ * request: POST { url } body { "input": ["<text>"] }
16
+ * response: { "data": [ { "embedding": [0.1, …] } ] } (also accepts
17
+ * { "embeddings": [...] } and { "data": [[...]] })
18
+ *
19
+ * VectorStore is deliberately synchronous, so embed() runs the network call in
20
+ * a short-lived child process (its own event loop) and blocks the parent with
21
+ * spawnSync. We deliberately do NOT use Atomics.wait on the main thread — that
22
+ * would deadlock fetch (the blocked main thread can't pump the socket, so the
23
+ * promise never settles). A child process has its own event loop, so spawnSync
24
+ * blocks without that deadlock. Only used when this embedder is selected; the
25
+ * default TrigramEmbedder path stays pure-sync, zero-network, zero-native.
26
+ */
27
+ import { l2Normalize } from "./embedder.js";
28
+ import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned embedding server (BYO backend, never remote)
29
+ /** Read + validate the localhost embeddings config from the environment. */
30
+ export function embeddingConfigFromEnv() {
31
+ const url = process.env.MEGACOMPACT_EMBEDDING_URL;
32
+ if (!url)
33
+ return null;
34
+ if (!/^https?:\/\/localhost[:/]/.test(url) && !/^https?:\/\/127\.0\.0\.1[:/]/.test(url)) {
35
+ // Only loopback is permitted — a remote host would violate PREVENT-PI-004.
36
+ throw new Error(`MEGACOMPACT_EMBEDDING_URL must be a localhost/127.0.0.1 endpoint (got ${url}). ` +
37
+ `Remote embedding endpoints are not allowed (PREVENT-PI-004).`);
38
+ }
39
+ const headers = {};
40
+ if (process.env.MEGACOMPACT_EMBEDDING_HEADERS) {
41
+ try {
42
+ Object.assign(headers, JSON.parse(process.env.MEGACOMPACT_EMBEDDING_HEADERS));
43
+ }
44
+ catch {
45
+ throw new Error("MEGACOMPACT_EMBEDDING_HEADERS must be valid JSON");
46
+ }
47
+ }
48
+ const dim = process.env.MEGACOMPACT_EMBEDDING_DIM
49
+ ? Number(process.env.MEGACOMPACT_EMBEDDING_DIM)
50
+ : undefined;
51
+ return {
52
+ url,
53
+ apiKey: process.env.MEGACOMPACT_EMBEDDING_KEY,
54
+ headers,
55
+ dim: Number.isFinite(dim) ? dim : undefined,
56
+ };
57
+ }
58
+ /** Extract a single embedding vector from a tolerant OpenAI-style response. */
59
+ function parseEmbedding(body) {
60
+ if (body && typeof body === "object") {
61
+ const b = body;
62
+ if (Array.isArray(b.data) && b.data[0] && typeof b.data[0] === "object") {
63
+ const first = b.data[0];
64
+ if (Array.isArray(first.embedding))
65
+ return first.embedding;
66
+ if (Array.isArray(first))
67
+ return first;
68
+ }
69
+ if (Array.isArray(b.embeddings))
70
+ return b.embeddings[0];
71
+ if (Array.isArray(b.data))
72
+ return b.data;
73
+ }
74
+ throw new Error("embeddings response missing a recognized vector shape");
75
+ }
76
+ // Inline worker script: performs the async fetch in a child process that has
77
+ // its own event loop (no main-thread deadlock), writes the JSON response to
78
+ // stdout. Reads request from env to avoid shell-quoting the body.
79
+ const WORKER = String.raw `
80
+ const u = process.env.MC_URL, b = process.env.MC_BODY, h = JSON.parse(process.env.MC_HEADERS || "{}");
81
+ try {
82
+ const r = await fetch(u, { method: "POST", headers: h, body: b }); // guardrails-allow PREVENT-PI-004: localhost-only user-spawned embedding server (BYO backend, never remote)
83
+ const out = JSON.stringify({ status: r.status, ok: r.ok, json: await r.json() });
84
+ process.stdout.write(out);
85
+ } catch (e) {
86
+ process.stdout.write(JSON.stringify({ error: String(e && e.message ? e.message : e) }));
87
+ }
88
+ `;
89
+ export class HttpEmbedder {
90
+ url;
91
+ apiKey;
92
+ headers;
93
+ resolvedDim;
94
+ constructor(opts) {
95
+ this.url = opts.url;
96
+ this.apiKey = opts.apiKey;
97
+ this.headers = opts.headers ?? {};
98
+ this.resolvedDim = opts.dim ?? 0; // resolved after the first embed
99
+ }
100
+ get dim() {
101
+ return this.resolvedDim;
102
+ }
103
+ embed(text) {
104
+ const body = JSON.stringify({ input: [text] });
105
+ const headers = {
106
+ "content-type": "application/json",
107
+ ...this.headers,
108
+ };
109
+ if (this.apiKey)
110
+ headers["authorization"] = `Bearer ${this.apiKey}`;
111
+ // localhost-only fetch — audited PREVENT-PI-004 exception (user-spawned
112
+ // local embedding server, same class as the /dashboard localhost UI). The
113
+ // child has its own event loop, so spawnSync blocks without deadlocking.
114
+ const res = spawnSync(process.execPath, ["-e", WORKER], {
115
+ encoding: "utf8",
116
+ env: {
117
+ ...process.env,
118
+ MC_URL: this.url,
119
+ MC_BODY: body,
120
+ MC_HEADERS: JSON.stringify(headers),
121
+ },
122
+ });
123
+ if (res.error || typeof res.stdout !== "string" || res.stdout.length === 0) {
124
+ const detail = res.error ? String(res.error) : res.stderr || "empty response";
125
+ throw new Error(`embedding server ${this.url} unreachable: ${detail}`);
126
+ }
127
+ let parsed;
128
+ try {
129
+ parsed = JSON.parse(res.stdout);
130
+ }
131
+ catch {
132
+ throw new Error(`embedding server ${this.url} returned non-JSON: ${res.stdout.slice(0, 200)}`);
133
+ }
134
+ if (parsed.error)
135
+ throw new Error(`embedding server ${this.url} failed: ${parsed.error}`);
136
+ if (!parsed.ok)
137
+ throw new Error(`embedding server ${this.url} returned ${parsed.status}`);
138
+ const vec = parseEmbedding(parsed.json);
139
+ if (this.resolvedDim === 0)
140
+ this.resolvedDim = vec.length;
141
+ return l2Normalize(vec);
142
+ }
143
+ }
@@ -0,0 +1,47 @@
1
+ /**
2
+ * log.ts — tiny append-only structured logger.
3
+ *
4
+ * Writes one JSON object per line to a log file (default:
5
+ * ~/.pi/agent/extensions/mega-compact.log). Best-effort: logging never throws
6
+ * into the extension. Pi-agnostic and dependency-free so it can be unit-tested.
7
+ */
8
+ import { appendFileSync, mkdirSync } from "node:fs";
9
+ import { dirname, join } from "node:path";
10
+ import { STATE_DIR_DEFAULT } from "./config.js";
11
+ /** Default log path lives alongside the state dir. */
12
+ export function defaultLogPath() {
13
+ return join(STATE_DIR_DEFAULT, "mega-compact.log");
14
+ }
15
+ export class Logger {
16
+ path;
17
+ enabled;
18
+ /** Monotonic clock injected by the caller so the module stays deterministic. */
19
+ now;
20
+ constructor(opts = {}) {
21
+ this.path = opts.path ?? defaultLogPath();
22
+ this.enabled = opts.enabled ?? true;
23
+ this.now = opts.now ?? (() => Date.now());
24
+ }
25
+ /** Append one structured line. Swallows all I/O errors. */
26
+ log(level, event, fields = {}) {
27
+ if (!this.enabled)
28
+ return;
29
+ const entry = { ts: this.now(), level, event, ...fields };
30
+ try {
31
+ mkdirSync(dirname(this.path), { recursive: true });
32
+ appendFileSync(this.path, `${JSON.stringify(entry)}\n`);
33
+ }
34
+ catch {
35
+ /* best-effort: never break the extension on a log failure */
36
+ }
37
+ }
38
+ info(event, fields) {
39
+ this.log("info", event, fields);
40
+ }
41
+ warn(event, fields) {
42
+ this.log("warn", event, fields);
43
+ }
44
+ error(event, fields) {
45
+ this.log("error", event, fields);
46
+ }
47
+ }
@@ -0,0 +1,42 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { mkdtempSync, rmSync, readFileSync, existsSync } from "node:fs";
4
+ import { tmpdir } from "node:os";
5
+ import { join } from "node:path";
6
+ import { Logger } from "./log.js";
7
+ const baseTmp = mkdtempSync(join(tmpdir(), "mc-log-"));
8
+ let counter = 0;
9
+ function logPath() {
10
+ return join(baseTmp, `run-${counter++}`, "mega-compact.log");
11
+ }
12
+ test("logger appends one JSON line per entry", () => {
13
+ const path = logPath();
14
+ let clock = 1000;
15
+ const log = new Logger({ path, now: () => clock++ });
16
+ log.info("compact", { checkpointId: "chkpt_001" });
17
+ log.warn("recall-empty", { query: "x" });
18
+ const lines = readFileSync(path, "utf8").trim().split("\n");
19
+ assert.equal(lines.length, 2);
20
+ const first = JSON.parse(lines[0]);
21
+ assert.equal(first.level, "info");
22
+ assert.equal(first.event, "compact");
23
+ assert.equal(first.checkpointId, "chkpt_001");
24
+ assert.equal(first.ts, 1000);
25
+ const second = JSON.parse(lines[1]);
26
+ assert.equal(second.level, "warn");
27
+ assert.equal(second.ts, 1001);
28
+ });
29
+ test("disabled logger writes nothing", () => {
30
+ const path = logPath();
31
+ const log = new Logger({ path, enabled: false });
32
+ log.info("compact", { a: 1 });
33
+ assert.equal(existsSync(path), false);
34
+ });
35
+ test("logger never throws on a bad path", () => {
36
+ // A path whose parent cannot be created (null byte) — must be swallowed.
37
+ const log = new Logger({ path: "/\0/nope.log" });
38
+ assert.doesNotThrow(() => log.error("boom", { x: 1 }));
39
+ });
40
+ test("cleanup", () => {
41
+ rmSync(baseTmp, { recursive: true, force: true });
42
+ });
@@ -0,0 +1,92 @@
1
+ /**
2
+ * minilm.ts — local MiniLM (all-MiniLM-L6-v2) sentence embedder (Sprint 12).
3
+ *
4
+ * Implements the `Embedder` interface so it drops into the existing VectorStore
5
+ * dedup cascade and search with no call-site changes. Inference is 100% local:
6
+ * the ONNX model + WordPiece vocab are on-disk artifacts fetched once by
7
+ * scripts/setup-minilm.mjs. There is NO network call at runtime (PREVENT-PI-004).
8
+ *
9
+ * Inputs (dynamic): input_ids, attention_mask, token_type_ids (int64).
10
+ * Output: last_hidden_state (batch, seq, 384). We mean-pool over non-padded
11
+ * tokens (attention_mask == 1) and L2-normalize → 384-dim unit vector.
12
+ *
13
+ * The ONNX session + tokenizer are loaded LAZILY on first embed() so the default
14
+ * TrigramEmbedder path (and its zero native-init cost) is untouched unless
15
+ * MEGACOMPACT_EMBEDDER=minilm is selected.
16
+ */
17
+ import { join } from "node:path";
18
+ import { homedir } from "node:os";
19
+ import { existsSync } from "node:fs";
20
+ import { l2Normalize, awaitSync } from "./embedder.js";
21
+ import { WordPieceTokenizer } from "./wordpiece.js";
22
+ export const MINILM_DIM = 384;
23
+ export const MINILM_MAX_LEN = 256;
24
+ /** Resolve the model directory: MEGACOMPACT_MINILM_DIR > ./models/minilm > ~/.pi … */
25
+ function resolveModelDir() {
26
+ if (process.env.MEGACOMPACT_MINILM_DIR)
27
+ return process.env.MEGACOMPACT_MINILM_DIR;
28
+ // Repo-local vendored path (gitignored).
29
+ const local = join(process.cwd(), "models", "minilm");
30
+ if (existsSync(local))
31
+ return local;
32
+ return join(homedir(), ".pi", "agent", "extensions", "mega-compact", "models", "minilm");
33
+ }
34
+ export class MiniLMEmbedder {
35
+ dim = MINILM_DIM;
36
+ session = null;
37
+ tokenizer = null;
38
+ modelDir;
39
+ loadPromise = null;
40
+ constructor(modelDir = resolveModelDir()) {
41
+ this.modelDir = modelDir;
42
+ }
43
+ async ensureLoaded() {
44
+ if (this.session && this.tokenizer)
45
+ return;
46
+ if (this.loadPromise)
47
+ return this.loadPromise;
48
+ this.loadPromise = (async () => {
49
+ const ort = await import("onnxruntime-node");
50
+ const modelPath = join(this.modelDir, "model_quantized.onnx");
51
+ const vocabPath = join(this.modelDir, "vocab.txt");
52
+ if (!existsSync(modelPath) || !existsSync(vocabPath)) {
53
+ throw new Error(`MiniLM artifacts missing in ${this.modelDir}. Run: node scripts/setup-minilm.mjs`);
54
+ }
55
+ // 1 thread is plenty for a single short-region embed and bounds CPU.
56
+ this.session = await ort.InferenceSession.create(modelPath, {
57
+ executionProviders: ["cpu"],
58
+ graphOptimizationLevel: "all",
59
+ });
60
+ this.tokenizer = WordPieceTokenizer.fromVocabFile(vocabPath);
61
+ })();
62
+ return this.loadPromise;
63
+ }
64
+ embed(text) {
65
+ awaitSync(this.ensureLoaded());
66
+ const enc = this.tokenizer.encode(text, MINILM_MAX_LEN);
67
+ const n = enc.inputIds.length;
68
+ const BigInt64 = (arr) => arr.map((x) => BigInt(x));
69
+ const ort = awaitSync(import("onnxruntime-node"));
70
+ const tensors = {
71
+ input_ids: new ort.Tensor("int64", BigInt64(enc.inputIds), [1, n]),
72
+ attention_mask: new ort.Tensor("int64", BigInt64(enc.attentionMask), [1, n]),
73
+ token_type_ids: new ort.Tensor("int64", BigInt64(enc.tokenTypeIds), [1, n]),
74
+ };
75
+ const out = awaitSync(this.session.run(tensors));
76
+ const hidden = out.last_hidden_state.data;
77
+ // hidden shape: [1, n, 384]. Mean-pool over non-padded positions.
78
+ const pooled = new Array(MINILM_DIM).fill(0);
79
+ let count = 0;
80
+ for (let i = 0; i < n; i++) {
81
+ if (enc.attentionMask[i] === 0)
82
+ continue;
83
+ const base = i * MINILM_DIM;
84
+ for (let d = 0; d < MINILM_DIM; d++)
85
+ pooled[d] += hidden[base + d];
86
+ count++;
87
+ }
88
+ if (count === 0)
89
+ return l2Normalize(new Array(MINILM_DIM).fill(0));
90
+ return l2Normalize(pooled.map((x) => x / count));
91
+ }
92
+ }
@@ -0,0 +1,131 @@
1
+ /**
2
+ * monitoring.ts — local dedup monitoring + alerting (Sprint 14, Phase 7).
3
+ *
4
+ * Per-decision structured events go to `events.log` (append-only JSON).
5
+ * Aggregate metrics (hit rate, FP rate, per-tier p95 latency, storage) go to
6
+ * `dashboard.json` — the SAME local-only file the /dashboard UI reads. There is
7
+ * NO Prometheus port and NO network listener (PREVENT-PI-004). Alerting is local
8
+ * only: an FP-rate breach flips the tier to MARK_ONLY and writes a warning.
9
+ *
10
+ * Best-effort: logging/metrics never throw into the add()/search() path.
11
+ */
12
+ import { readFileSync, writeFileSync, existsSync, mkdirSync, appendFileSync } from "node:fs";
13
+ import { dirname, join } from "node:path";
14
+ import { STATE_DIR_DEFAULT } from "./config.js";
15
+ const TIERS = ["L0", "L1", "L2", "RAPTOR"];
16
+ function emptyMetrics() {
17
+ const dec = {};
18
+ const dp = {};
19
+ const fp = {};
20
+ const lat = {};
21
+ for (const t of TIERS) {
22
+ dec[t] = 0;
23
+ dp[t] = 0;
24
+ fp[t] = 0;
25
+ lat[t] = [];
26
+ }
27
+ return { decisions: dec, deduped: dp, falsePositives: fp, latency: lat, storageBytes: 0 };
28
+ }
29
+ /** Append a structured decision event to events.log (best-effort). */
30
+ export function logDecision(path, ev) {
31
+ try {
32
+ mkdirSync(dirname(path), { recursive: true });
33
+ appendFileSync(path, `${JSON.stringify(ev)}\n`);
34
+ }
35
+ catch {
36
+ /* never break the extension on a log failure */
37
+ }
38
+ }
39
+ /**
40
+ * Load metrics from dashboard.json, or return a fresh empty snapshot.
41
+ * Kept simple + synchronous (no network).
42
+ */
43
+ export function loadMetrics(path) {
44
+ try {
45
+ if (existsSync(path)) {
46
+ const raw = readFileSync(path, "utf-8");
47
+ const parsed = JSON.parse(raw);
48
+ const base = emptyMetrics();
49
+ return {
50
+ decisions: { ...base.decisions, ...(parsed.decisions ?? {}) },
51
+ deduped: { ...base.deduped, ...(parsed.deduped ?? {}) },
52
+ falsePositives: { ...base.falsePositives, ...(parsed.falsePositives ?? {}) },
53
+ latency: { ...base.latency, ...(parsed.latency ?? {}) },
54
+ storageBytes: parsed.storageBytes ?? 0,
55
+ };
56
+ }
57
+ }
58
+ catch {
59
+ /* corrupt metrics → fresh */
60
+ }
61
+ return emptyMetrics();
62
+ }
63
+ /** Persist metrics to dashboard.json (best-effort). */
64
+ export function saveMetrics(path, m) {
65
+ try {
66
+ mkdirSync(dirname(path), { recursive: true });
67
+ writeFileSync(path, JSON.stringify(m));
68
+ }
69
+ catch {
70
+ /* never break the extension */
71
+ }
72
+ }
73
+ /** Compute the p95 latency (ms) for a tier from its samples. */
74
+ export function p95(samples) {
75
+ if (samples.length === 0)
76
+ return 0;
77
+ const sorted = [...samples].sort((a, b) => a - b);
78
+ const idx = Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95));
79
+ return sorted[idx];
80
+ }
81
+ /** FP rate for a tier over the current window (0..1). */
82
+ export function fpRate(m, tier) {
83
+ const decisions = m.decisions[tier] ?? 0;
84
+ if (decisions === 0)
85
+ return 0;
86
+ return (m.falsePositives[tier] ?? 0) / decisions;
87
+ }
88
+ /**
89
+ * Evaluate FP-rate breaches against the config thresholds. A breached fuzzy tier
90
+ * (L0 vs L1/L2 have different thresholds) is auto-downgraded to MARK_ONLY — the
91
+ * local re-map of "alertmanager" (QA #18/#19): record but don't collapse, no
92
+ * remote alert. Returns the tiers flipped so the caller can mutate its config.
93
+ */
94
+ export function evaluateAlerts(m, cfg) {
95
+ const breached = [];
96
+ const warnings = [];
97
+ for (const tier of TIERS) {
98
+ const rate = fpRate(m, tier);
99
+ const limit = tier === "L0" ? cfg.FP_RATE_L0 : cfg.FP_RATE_L1L2;
100
+ if (rate > limit) {
101
+ breached.push(tier);
102
+ warnings.push(`DEDUP FP BREACH tier=${tier} rate=${rate.toFixed(4)} > ${limit}`);
103
+ }
104
+ }
105
+ return { breached, warnings };
106
+ }
107
+ /**
108
+ * Record one decision into the metrics snapshot (mutates `m` in place) and
109
+ * returns the updated snapshot. Caps stored latency samples to keep memory
110
+ * bounded (last 1000 per tier).
111
+ */
112
+ export function recordDecision(m, tier, result, latencyMs, falsePositive = false) {
113
+ m.decisions[tier] = (m.decisions[tier] ?? 0) + 1;
114
+ if (result === "deduped")
115
+ m.deduped[tier] = (m.deduped[tier] ?? 0) + 1;
116
+ if (falsePositive)
117
+ m.falsePositives[tier] = (m.falsePositives[tier] ?? 0) + 1;
118
+ const arr = m.latency[tier] ?? (m.latency[tier] = []);
119
+ arr.push(latencyMs);
120
+ if (arr.length > 1000)
121
+ arr.shift();
122
+ return m;
123
+ }
124
+ /** Default metrics path alongside the state dir. */
125
+ export function defaultMetricsPath(stateDir = STATE_DIR_DEFAULT) {
126
+ return join(stateDir, "dashboard.json");
127
+ }
128
+ /** Default events-log path alongside the state dir. */
129
+ export function defaultEventsPath(stateDir = STATE_DIR_DEFAULT) {
130
+ return join(stateDir, "events.log");
131
+ }