pi-mega-compact 0.8.25 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -9
- package/dist/extensions/mega-pipeline/compact.js +2 -1
- package/dist/src/dedup/raptor/tree.js +11 -0
- package/dist/src/memory.test.js +29 -0
- package/dist/src/memoryOps.js +4 -19
- package/dist/src/memoryRecall.test.js +27 -0
- package/dist/src/memoryRoundtrip.test.js +137 -0
- package/dist/src/raptor-inject-summaries.test.js +10 -3
- package/dist/src/recall.js +29 -13
- package/dist/src/sprint4x-rag-verification.test.js +93 -0
- package/dist/src/store/repoKey.js +45 -0
- package/dist/src/store/sqlite/turns.js +1 -3
- package/dist/src/store/vectorIndex.test.js +25 -1
- package/dist/src/vector-search.js +11 -5
- package/dist/src/vectorStore.js +4 -1
- package/extensions/mega-pipeline/compact.ts +2 -1
- package/package.json +1 -1
- package/src/dedup/raptor/tree.ts +11 -0
- package/src/memory.test.ts +47 -1
- package/src/memoryOps.ts +4 -19
- package/src/memoryRecall.test.ts +36 -0
- package/src/memoryRoundtrip.test.ts +155 -0
- package/src/raptor-inject-summaries.test.ts +22 -6
- package/src/recall.ts +439 -368
- package/src/sprint4x-rag-verification.test.ts +119 -0
- package/src/store/repoKey.ts +50 -0
- package/src/store/sqlite/turns.ts +177 -167
- package/src/store/vectorIndex.test.ts +25 -1
- package/src/vector-search.ts +10 -5
- package/src/vectorStore.ts +4 -1
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sprint4x-rag-verification.test.ts — run the S40–S47 RAG suite once against
|
|
3
|
+
* the current implementation and assert the documented "default enable" claims.
|
|
4
|
+
*
|
|
5
|
+
* This is the verification pass demanded by the roadmap/backlog: each spec
|
|
6
|
+
* claims its feature flags DEFAULT ON. The honest check (not a re-statement of
|
|
7
|
+
* the spec) is: which flags actually exist in code, what do they default to,
|
|
8
|
+
* and is the consuming path wired?
|
|
9
|
+
*
|
|
10
|
+
* Findings recorded here, per spec:
|
|
11
|
+
* S40 importance-scoring — module src/importance.ts exists (S40A) but is
|
|
12
|
+
* NOT consumed by compactSession/vector-paths;
|
|
13
|
+
* no shipped flag, no adapter wiring.
|
|
14
|
+
* S41 self-rag-quality-gate — spec-only; NO flag, NO consumer.
|
|
15
|
+
* S42 raptor-multilevel — PARTIALLY SHIPPED: RAPTOR_MULTILEVEL_ENABLED
|
|
16
|
+
* and RAPTOR_LEAF_EXPANSION both default true
|
|
17
|
+
* with real consumers; the spec's claim holds.
|
|
18
|
+
* S43 hyde-vague-queries — spec-only (QUERY_REFORMULATION_ENABLED absent).
|
|
19
|
+
* S44 three-tier-latency-routing — spec-only (TIERED_ROUTING_ENABLED absent).
|
|
20
|
+
* S45 crag-quality-metrics — spec-only (CRAG_ENABLED absent).
|
|
21
|
+
* S46 visual-memory-map — spec-only (MEMORY_MAP_ENABLED absent;
|
|
22
|
+
* memory-graph endpoint not in dashboard-server).
|
|
23
|
+
* S47 auto-categorizing-wiki — spec-only (AUTO_WIKI_ENABLED absent).
|
|
24
|
+
*
|
|
25
|
+
* Policy: the suite runs against the flags that EXIST today. Unimplemented spec
|
|
26
|
+
* claims are pinned by the S4X_SPEC_ONLY tests so regressions can't silently
|
|
27
|
+
* add them in the wrong state, and this file records exactly which
|
|
28
|
+
* spec-vs-implementation gaps remain.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import { test } from "node:test";
|
|
32
|
+
import assert from "node:assert/strict";
|
|
33
|
+
import { loadDedupConfig } from "./config/dedup.js";
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Clear any env pollution so the default is measured, not the override.
|
|
37
|
+
*/
|
|
38
|
+
function fresh<T extends string>(envKey: T, get: () => unknown): unknown {
|
|
39
|
+
delete process.env[envKey];
|
|
40
|
+
try {
|
|
41
|
+
return get();
|
|
42
|
+
} finally {
|
|
43
|
+
delete process.env[envKey];
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// ---- S42: shipped flags default ON (claim holds) ----------------------------
|
|
48
|
+
|
|
49
|
+
test("S42 RAPTOR_MULTILEVEL_ENABLED defaults ON and honors env off", () => {
|
|
50
|
+
const d = fresh("MEGACOMPACT_RAPTOR_MULTILEVEL", () => loadDedupConfig());
|
|
51
|
+
assert.ok(
|
|
52
|
+
(d as { RAPTOR_MULTILEVEL_ENABLED: boolean }).RAPTOR_MULTILEVEL_ENABLED,
|
|
53
|
+
"ship claim: multi-level on by default",
|
|
54
|
+
);
|
|
55
|
+
process.env.MEGACOMPACT_RAPTOR_MULTILEVEL = "false";
|
|
56
|
+
const off = loadDedupConfig();
|
|
57
|
+
assert.ok(
|
|
58
|
+
!off.RAPTOR_MULTILEVEL_ENABLED,
|
|
59
|
+
"env override: MEGACOMPACT_RAPTOR_MULTILEVEL=false turns it off",
|
|
60
|
+
);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("S42 RAPTOR_LEAF_EXPANSION defaults ON and honors env off", () => {
|
|
64
|
+
const d = fresh("MEGACOMPACT_RAPTOR_LEAF_EXPANSION", () => loadDedupConfig());
|
|
65
|
+
assert.ok(
|
|
66
|
+
(d as { RAPTOR_LEAF_EXPANSION: boolean }).RAPTOR_LEAF_EXPANSION,
|
|
67
|
+
"ship claim: leaf-expansion on by default",
|
|
68
|
+
);
|
|
69
|
+
process.env.MEGACOMPACT_RAPTOR_LEAF_EXPANSION = "false";
|
|
70
|
+
const off = loadDedupConfig();
|
|
71
|
+
assert.ok(!off.RAPTOR_LEAF_EXPANSION, "env override off");
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
// ---- S40: module exists, claims do NOT yet hold at the flag/consumer level --
|
|
75
|
+
|
|
76
|
+
test("S40 importance module exists in-tree (S40A artifact)", async () => {
|
|
77
|
+
// Load the module and confirm it exports the scoring surface the spec
|
|
78
|
+
// describes. This proves the S40A implementation exists even though nothing
|
|
79
|
+
// wires it into compaction/vector paths yet (documented gap).
|
|
80
|
+
const mod = await import("./importance.js");
|
|
81
|
+
assert.ok(mod && typeof mod === "object", "importance.ts loads");
|
|
82
|
+
// Exports per spec: score() plus item-type enum.
|
|
83
|
+
assert.ok(typeof (mod as Record<string, unknown>).score === "function",
|
|
84
|
+
"importance.ts exports score() function",
|
|
85
|
+
);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test("S40 has no shipped consumer flag in the current codebase (gap pinned)", async () => {
|
|
89
|
+
// The S40 spec claims an IMPORTANCE_SCORING flag defaults ON. In the
|
|
90
|
+
// current code no such flag exists, and no vector/compact path reads
|
|
91
|
+
// importance scores. This test documents the gap so a future wiring does
|
|
92
|
+
// not silently flip it on in the wrong shape.
|
|
93
|
+
const cfg = loadDedupConfig();
|
|
94
|
+
assert.ok(
|
|
95
|
+
!("IMPORTANCE_SCORING" in cfg),
|
|
96
|
+
"no IMPORTANCE_SCORING flag yet (S40 consumer wiring missing)",
|
|
97
|
+
);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
// ---- S41/S43–S47: spec-only modules — pin the absence ------------------------
|
|
101
|
+
|
|
102
|
+
test("S4X spec-only flags absent from DedupConfig", () => {
|
|
103
|
+
const cfg = loadDedupConfig();
|
|
104
|
+
const specFlags = [
|
|
105
|
+
"CRITIQUE_ENABLED", // S41
|
|
106
|
+
"QUERY_REFORMULATION_ENABLED", // S43
|
|
107
|
+
"TIERED_ROUTING_ENABLED", // S44
|
|
108
|
+
"CRAG_ENABLED", // S45
|
|
109
|
+
"CRAG_EXPANSION_ENABLED", // S45
|
|
110
|
+
"MEMORY_MAP_ENABLED", // S46
|
|
111
|
+
"AUTO_WIKI_ENABLED", // S47
|
|
112
|
+
] as const;
|
|
113
|
+
for (const f of specFlags) {
|
|
114
|
+
assert.ok(
|
|
115
|
+
!(f in cfg),
|
|
116
|
+
`${f} is spec-only — should not be present in DedupConfig`,
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
});
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* repoKey.ts — S25-B: the single repo-scope key for BOTH global PGlite indexes.
|
|
3
|
+
*
|
|
4
|
+
* Before S25 the checkpoint index (vector_index) keyed on stateDir while the
|
|
5
|
+
* memory index (memory_index) keyed on the git root — two scopes that meant
|
|
6
|
+
* cross-repo checkpoint hydration had no way back from repo_id → the repo's
|
|
7
|
+
* store. This helper unifies both indexes on ONE key: the resolved git root,
|
|
8
|
+
* falling back to stateDir outside git worktrees.
|
|
9
|
+
*
|
|
10
|
+
* stateDirForRepo() reverses the mapping via the machine-wide
|
|
11
|
+
* repo_registry (src/store/sqlite/global-index.ts): a repo_id hit from the
|
|
12
|
+
* index resolves to that repo's stateDir so getCheckpoint() can hydrate from
|
|
13
|
+
* the authoritative node:sqlite store. Returns undefined when unresolvable —
|
|
14
|
+
* the caller skips the hit (degrade, never crash).
|
|
15
|
+
*
|
|
16
|
+
* PREVENT-PI-004: `git rev-parse` is local + read-only (annotated below).
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { execSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: read-only `git rev-parse` to scope the vector index per-repo
|
|
20
|
+
import { getRepoRegistry } from "./sqlite/global-index.js";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Resolve the canonical repo key for a state dir. Git root when inside a
|
|
24
|
+
* worktree, stateDir otherwise. Two repos sharing a git root (e.g. nested
|
|
25
|
+
* checkouts pointing at the same repo) collapse to one scope — intended.
|
|
26
|
+
*/
|
|
27
|
+
export function repoKey(stateDir: string): string {
|
|
28
|
+
try {
|
|
29
|
+
const out = execSync("git rev-parse --show-toplevel", {
|
|
30
|
+
cwd: stateDir,
|
|
31
|
+
encoding: "utf-8",
|
|
32
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
33
|
+
}).trim();
|
|
34
|
+
return out || stateDir;
|
|
35
|
+
} catch {
|
|
36
|
+
return stateDir;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Reverse map: repo_id → stateDir. Registry hit wins (git-root scope, S25);
|
|
42
|
+
* otherwise the key is assumed to be a legacy/ungit-scoped stateDir and
|
|
43
|
+
* returned verbatim (callers treat undefined-unopenable dirs as skip/degrade).
|
|
44
|
+
*/
|
|
45
|
+
export function stateDirForRepo(
|
|
46
|
+
repoId: string,
|
|
47
|
+
indexDir?: string,
|
|
48
|
+
): string | undefined {
|
|
49
|
+
return getRepoRegistry(repoId, indexDir)?.stateDir ?? repoId;
|
|
50
|
+
}
|
|
@@ -25,28 +25,28 @@ import type { DatabaseSync } from "node:sqlite";
|
|
|
25
25
|
|
|
26
26
|
/** A row in `turns`. */
|
|
27
27
|
export interface TurnRow {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
28
|
+
id: number;
|
|
29
|
+
conversationId: string;
|
|
30
|
+
sessionId: string;
|
|
31
|
+
turnIndex: number;
|
|
32
|
+
role: string | null;
|
|
33
|
+
startedAt: number;
|
|
34
|
+
endedAt: number | null;
|
|
35
|
+
ctxTokens: number | null;
|
|
36
|
+
ctxPercent: number | null;
|
|
37
|
+
pressureBand: string | null;
|
|
38
|
+
modelId: string | null;
|
|
39
|
+
epochId: string | null;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
/** A row in `turn_recall`. */
|
|
43
43
|
export interface TurnRecallRow {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
44
|
+
id: number;
|
|
45
|
+
turnId: number;
|
|
46
|
+
checkpointId: string;
|
|
47
|
+
score: number;
|
|
48
|
+
source: string;
|
|
49
|
+
raptorLevel: number | null;
|
|
50
50
|
}
|
|
51
51
|
|
|
52
52
|
/** Where a recalled hit came from (recorded on turn_recall.source). */
|
|
@@ -54,24 +54,24 @@ export type RecallSource = "flat" | "raptor" | "cross-repo" | "memory";
|
|
|
54
54
|
|
|
55
55
|
/** Generate a new conversation id (`conv_` + 16 hex). */
|
|
56
56
|
export function newConversationId(): string {
|
|
57
|
-
|
|
57
|
+
return `conv_${randomBytes(8).toString("hex")}`;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
60
|
function rowToTurn(r: Record<string, unknown>): TurnRow {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
61
|
+
return {
|
|
62
|
+
id: r.id as number,
|
|
63
|
+
conversationId: r.conversation_id as string,
|
|
64
|
+
sessionId: r.session_id as string,
|
|
65
|
+
turnIndex: r.turn_index as number,
|
|
66
|
+
role: (r.role as string | null) ?? null,
|
|
67
|
+
startedAt: r.started_at as number,
|
|
68
|
+
endedAt: (r.ended_at as number | null) ?? null,
|
|
69
|
+
ctxTokens: (r.ctx_tokens as number | null) ?? null,
|
|
70
|
+
ctxPercent: (r.ctx_percent as number | null) ?? null,
|
|
71
|
+
pressureBand: (r.pressure_band as string | null) ?? null,
|
|
72
|
+
modelId: (r.model_id as string | null) ?? null,
|
|
73
|
+
epochId: (r.epoch_id as string | null) ?? null,
|
|
74
|
+
};
|
|
75
75
|
}
|
|
76
76
|
|
|
77
77
|
/**
|
|
@@ -80,26 +80,26 @@ function rowToTurn(r: Record<string, unknown>): TurnRow {
|
|
|
80
80
|
* Idempotent on (session_id, turn_index) — re-upserting overwrites metrics.
|
|
81
81
|
*/
|
|
82
82
|
export function recordTurn(
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
83
|
+
input: {
|
|
84
|
+
conversationId: string;
|
|
85
|
+
sessionId: string;
|
|
86
|
+
turnIndex: number;
|
|
87
|
+
role?: string;
|
|
88
|
+
startedAt?: number;
|
|
89
|
+
endedAt?: number;
|
|
90
|
+
ctxTokens?: number;
|
|
91
|
+
ctxPercent?: number;
|
|
92
|
+
pressureBand?: string;
|
|
93
|
+
modelId?: string;
|
|
94
|
+
epochId?: string;
|
|
95
|
+
},
|
|
96
|
+
stateDir: string = getStateDir(),
|
|
97
97
|
): number {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
98
|
+
const db = openStore(stateDir);
|
|
99
|
+
const sid = normalizeSessionId(input.sessionId);
|
|
100
|
+
const startedAt = input.startedAt ?? Date.now();
|
|
101
|
+
db.prepare(
|
|
102
|
+
`INSERT INTO turns (conversation_id, session_id, turn_index, role, started_at,
|
|
103
103
|
ended_at, ctx_tokens, ctx_percent, pressure_band, model_id, epoch_id)
|
|
104
104
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
105
105
|
ON CONFLICT(session_id, turn_index) DO UPDATE SET
|
|
@@ -112,23 +112,23 @@ export function recordTurn(
|
|
|
112
112
|
pressure_band = COALESCE(excluded.pressure_band, pressure_band),
|
|
113
113
|
model_id = COALESCE(excluded.model_id, model_id),
|
|
114
114
|
epoch_id = COALESCE(excluded.epoch_id, epoch_id)`,
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
115
|
+
).run(
|
|
116
|
+
input.conversationId,
|
|
117
|
+
sid,
|
|
118
|
+
input.turnIndex,
|
|
119
|
+
input.role ?? null,
|
|
120
|
+
startedAt,
|
|
121
|
+
input.endedAt ?? null,
|
|
122
|
+
input.ctxTokens ?? null,
|
|
123
|
+
input.ctxPercent ?? null,
|
|
124
|
+
input.pressureBand ?? null,
|
|
125
|
+
input.modelId ?? null,
|
|
126
|
+
input.epochId ?? null,
|
|
127
|
+
);
|
|
128
|
+
const row = db
|
|
129
|
+
.prepare("SELECT id FROM turns WHERE session_id = ? AND turn_index = ?")
|
|
130
|
+
.get(sid, input.turnIndex) as { id: number };
|
|
131
|
+
return row.id;
|
|
132
132
|
}
|
|
133
133
|
|
|
134
134
|
/**
|
|
@@ -137,108 +137,116 @@ export function recordTurn(
|
|
|
137
137
|
* path + score. RAPTOR cluster hits carry raptorLevel. Best-effort + non-fatal.
|
|
138
138
|
*/
|
|
139
139
|
export function recordTurnRecall(
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
140
|
+
turnId: number,
|
|
141
|
+
hits: {
|
|
142
|
+
checkpointId: string;
|
|
143
|
+
score: number;
|
|
144
|
+
source: RecallSource;
|
|
145
|
+
raptorLevel?: number;
|
|
146
|
+
}[],
|
|
147
|
+
stateDir: string = getStateDir(),
|
|
148
148
|
): void {
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
149
|
+
if (hits.length === 0) return;
|
|
150
|
+
const db = openStore(stateDir);
|
|
151
|
+
const stmt = db.prepare(
|
|
152
|
+
`INSERT INTO turn_recall (turn_id, checkpoint_id, score, source, raptor_level)
|
|
153
153
|
VALUES (?, ?, ?, ?, ?)
|
|
154
154
|
ON CONFLICT(turn_id, checkpoint_id) DO UPDATE SET
|
|
155
155
|
score = excluded.score, source = excluded.source,
|
|
156
156
|
raptor_level = excluded.raptor_level`,
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
157
|
+
);
|
|
158
|
+
withTx(db, () => {
|
|
159
|
+
for (const h of hits) {
|
|
160
|
+
stmt.run(
|
|
161
|
+
turnId,
|
|
162
|
+
h.checkpointId,
|
|
163
|
+
h.score,
|
|
164
|
+
h.source,
|
|
165
|
+
h.raptorLevel ?? null,
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
});
|
|
163
169
|
}
|
|
164
170
|
|
|
165
171
|
/** Get a turn by conversation id + turn index (the lookup a fork uses). */
|
|
166
172
|
export function getTurn(
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
173
|
+
conversationId: string,
|
|
174
|
+
turnIndex: number,
|
|
175
|
+
stateDir: string = getStateDir(),
|
|
170
176
|
): TurnRow | null {
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
.get(conversationId, turnIndex) as Record<string, unknown> | undefined;
|
|
177
|
-
return row ? rowToTurn(row) : null;
|
|
177
|
+
const db = openStore(stateDir);
|
|
178
|
+
const row = db
|
|
179
|
+
.prepare(`SELECT * FROM turns WHERE conversation_id = ? AND turn_index = ?`)
|
|
180
|
+
.get(conversationId, turnIndex) as Record<string, unknown> | undefined;
|
|
181
|
+
return row ? rowToTurn(row) : null;
|
|
178
182
|
}
|
|
179
183
|
|
|
180
184
|
/** Get a turn by its global id. */
|
|
181
185
|
export function getTurnById(
|
|
182
|
-
|
|
183
|
-
|
|
186
|
+
turnId: number,
|
|
187
|
+
stateDir: string = getStateDir(),
|
|
184
188
|
): TurnRow | null {
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
189
|
+
const db = openStore(stateDir);
|
|
190
|
+
const row = db.prepare("SELECT * FROM turns WHERE id = ?").get(turnId) as
|
|
191
|
+
| Record<string, unknown>
|
|
192
|
+
| undefined;
|
|
193
|
+
return row ? rowToTurn(row) : null;
|
|
190
194
|
}
|
|
191
195
|
|
|
192
196
|
/** All turn_recall rows for a turn (what was injected at that turn). */
|
|
193
197
|
export function listTurnRecall(
|
|
194
|
-
|
|
195
|
-
|
|
198
|
+
turnId: number,
|
|
199
|
+
stateDir: string = getStateDir(),
|
|
196
200
|
): TurnRecallRow[] {
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
+
const db = openStore(stateDir);
|
|
202
|
+
const rows = db
|
|
203
|
+
.prepare(
|
|
204
|
+
`SELECT id, turn_id, checkpoint_id, score, source, raptor_level
|
|
201
205
|
FROM turn_recall WHERE turn_id = ? ORDER BY score DESC`,
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
206
|
+
)
|
|
207
|
+
.all(turnId) as Array<Record<string, unknown>>;
|
|
208
|
+
return rows.map((r) => ({
|
|
209
|
+
id: r.id as number,
|
|
210
|
+
turnId: r.turn_id as number,
|
|
211
|
+
checkpointId: r.checkpoint_id as string,
|
|
212
|
+
score: r.score as number,
|
|
213
|
+
source: r.source as string,
|
|
214
|
+
raptorLevel: (r.raptor_level as number | null) ?? null,
|
|
215
|
+
}));
|
|
212
216
|
}
|
|
213
217
|
|
|
214
218
|
/** All turns in a conversation, ascending by turn_index. */
|
|
215
219
|
export function listConversationTurns(
|
|
216
|
-
|
|
217
|
-
|
|
220
|
+
conversationId: string,
|
|
221
|
+
stateDir: string = getStateDir(),
|
|
218
222
|
): TurnRow[] {
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
223
|
+
const db = openStore(stateDir);
|
|
224
|
+
const rows = db
|
|
225
|
+
.prepare(
|
|
226
|
+
`SELECT * FROM turns WHERE conversation_id = ? ORDER BY turn_index ASC`,
|
|
227
|
+
)
|
|
228
|
+
.all(conversationId) as Array<Record<string, unknown>>;
|
|
229
|
+
return rows.map(rowToTurn);
|
|
226
230
|
}
|
|
227
231
|
|
|
228
232
|
/** Resolve a session's conversation id, generating + persisting one if none.
|
|
229
233
|
* A resumed session inherits its existing conversationId from session_state. */
|
|
230
234
|
export function ensureConversationId(
|
|
231
|
-
|
|
232
|
-
|
|
235
|
+
sessionId: string,
|
|
236
|
+
stateDir: string = getStateDir(),
|
|
233
237
|
): string {
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
238
|
+
const st = loadSessionState(sessionId, stateDir);
|
|
239
|
+
if (st.conversationId) return st.conversationId;
|
|
240
|
+
const conv = newConversationId();
|
|
241
|
+
saveSessionState(
|
|
242
|
+
sessionId,
|
|
243
|
+
{
|
|
244
|
+
...st,
|
|
245
|
+
conversationId: conv,
|
|
246
|
+
},
|
|
247
|
+
stateDir,
|
|
248
|
+
);
|
|
249
|
+
return conv;
|
|
242
250
|
}
|
|
243
251
|
|
|
244
252
|
/**
|
|
@@ -252,41 +260,43 @@ export function ensureConversationId(
|
|
|
252
260
|
* injected in the new session's session_state so they're not re-recalled.
|
|
253
261
|
*/
|
|
254
262
|
export function forkConversation(
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
263
|
+
parentConversationId: string,
|
|
264
|
+
forkTurnId: number,
|
|
265
|
+
stateDir: string = getStateDir(),
|
|
258
266
|
): { conversationId: string; recalled: TurnRecallRow[] } {
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
267
|
+
const childId = newConversationId();
|
|
268
|
+
const db: DatabaseSync = openStore(stateDir);
|
|
269
|
+
withTx(db, () => {
|
|
270
|
+
db.prepare(
|
|
271
|
+
`INSERT INTO conversation_branches
|
|
264
272
|
(conversation_id, parent_conversation_id, fork_turn_id, created_at)
|
|
265
273
|
VALUES (?, ?, ?, ?)
|
|
266
274
|
ON CONFLICT(conversation_id) DO NOTHING`,
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
275
|
+
).run(childId, parentConversationId, forkTurnId, Date.now());
|
|
276
|
+
});
|
|
277
|
+
// Replay set: the parent's injected checkpoints at the fork turn.
|
|
278
|
+
const recalled = listTurnRecall(forkTurnId, stateDir);
|
|
279
|
+
return { conversationId: childId, recalled };
|
|
272
280
|
}
|
|
273
281
|
|
|
274
282
|
/** Clear turn tracking rows for a session (tests / DR). */
|
|
275
283
|
export function clearTurns(
|
|
276
|
-
|
|
277
|
-
|
|
284
|
+
sessionId: string,
|
|
285
|
+
stateDir: string = getStateDir(),
|
|
278
286
|
): void {
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
287
|
+
const db: DatabaseSync = openStore(stateDir);
|
|
288
|
+
const sid = normalizeSessionId(sessionId);
|
|
289
|
+
withTx(db, () => {
|
|
290
|
+
const turnIds = db
|
|
291
|
+
.prepare("SELECT id FROM turns WHERE session_id = ?")
|
|
292
|
+
.all(sid) as Array<{ id: number }>;
|
|
293
|
+
const ids = turnIds.map((t) => t.id);
|
|
294
|
+
if (ids.length > 0) {
|
|
295
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
296
|
+
db.prepare(
|
|
297
|
+
`DELETE FROM turn_recall WHERE turn_id IN (${placeholders})`,
|
|
298
|
+
).run(...ids);
|
|
299
|
+
}
|
|
300
|
+
db.prepare("DELETE FROM turns WHERE session_id = ?").run(sid);
|
|
301
|
+
});
|
|
292
302
|
}
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import { test } from "node:test";
|
|
14
14
|
import assert from "node:assert/strict";
|
|
15
|
-
import { mkdtempSync, rmSync } from "node:fs";
|
|
15
|
+
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
16
16
|
import { tmpdir } from "node:os";
|
|
17
17
|
import { join } from "node:path";
|
|
18
18
|
import {
|
|
@@ -95,6 +95,30 @@ test("dimension guard: non-512 vectors are skipped, never corrupt the index", as
|
|
|
95
95
|
}
|
|
96
96
|
});
|
|
97
97
|
|
|
98
|
+
test("corrupt-dir self-heal: torn PGlite dir is deleted + retried, never crashes", async () => {
|
|
99
|
+
delete process.env.MEGACOMPACT_PGLITE_DISABLED;
|
|
100
|
+
const dir = isolateIndexDir();
|
|
101
|
+
try {
|
|
102
|
+
await closeVectorIndex();
|
|
103
|
+
// Tear the dir: garbage bytes where PGlite expects its data/ files.
|
|
104
|
+
mkdirSync(dir, { recursive: true });
|
|
105
|
+
writeFileSync(join(dir, "data"), Buffer.from([0xde, 0xad, 0xbe, 0xef, 0xde, 0xad]));
|
|
106
|
+
// openPgLite(retryOnCorrupt=true) deletes + retries; if that still fails it
|
|
107
|
+
// disables gracefully. Either path must NOT throw.
|
|
108
|
+
const pg = await initVectorIndex();
|
|
109
|
+
assert.ok(pg, "index self-healed after corruption ");
|
|
110
|
+
// And the index works after heal: upsert + search round-trip.
|
|
111
|
+
await upsertEmbedding("/repoE/.pi/mega-compact", "sessE", "chkpt_001", spikeVec(7));
|
|
112
|
+
const hits = await searchAsync(spikeVec(7), { k: 1 });
|
|
113
|
+
assert.equal(hits.length, 1, "search works on the healed index");
|
|
114
|
+
assert.equal(hits[0].checkpointId, "chkpt_001");
|
|
115
|
+
} finally {
|
|
116
|
+
await closeVectorIndex();
|
|
117
|
+
rmSync(dir, { recursive: true, force: true });
|
|
118
|
+
delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
|
|
98
122
|
test("kill-switch: MEGACOMPACT_PGLITE_DISABLED disables the index gracefully", async () => {
|
|
99
123
|
const dir = isolateIndexDir();
|
|
100
124
|
process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
|
package/src/vector-search.ts
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
import { cosineSimilarity } from "./embedder.js";
|
|
15
15
|
import { normalizeSessionId } from "./store.js";
|
|
16
16
|
import { mmrRerank, type MmrItem } from "./dedup/mmr.js";
|
|
17
|
+
import { stateDirForRepo } from "./store/repoKey.js";
|
|
17
18
|
import { topK } from "./dedup/topk.js";
|
|
18
19
|
import {
|
|
19
20
|
listCheckpoints,
|
|
@@ -320,13 +321,17 @@ export async function vectorSearchAsync(
|
|
|
320
321
|
// Index empty/unavailable → synchronous per-session fallback (this repo).
|
|
321
322
|
return vectorSearch(store, sid, query, k);
|
|
322
323
|
}
|
|
323
|
-
// Hydrate each index hit from the authoritative node:sqlite store.
|
|
324
|
-
//
|
|
325
|
-
//
|
|
326
|
-
//
|
|
324
|
+
// Hydrate each index hit from the authoritative node:sqlite store. The
|
|
325
|
+
// index keys on repo_id (S25: git root via repoKey; legacy rows keyed by
|
|
326
|
+
// stateDir) — resolve repo_id → stateDir via stateDirForRepo, and skip
|
|
327
|
+
// unresolvable/foreign hits (degrade, never crash). Cross-repo hits carry
|
|
328
|
+
// their source repoId so the recall block can label them; same-repo stays
|
|
329
|
+
// unlabeled.
|
|
327
330
|
const hydrated: SearchHit[] = [];
|
|
328
331
|
for (const h of indexHits) {
|
|
329
|
-
const
|
|
332
|
+
const hitStateDir = stateDirForRepo(h.repoId);
|
|
333
|
+
if (!hitStateDir) continue;
|
|
334
|
+
const cp = getCheckpoint(h.sessionId, h.checkpointId, hitStateDir);
|
|
330
335
|
if (cp && cp.dedupStatus !== "removed") {
|
|
331
336
|
const crossRepo =
|
|
332
337
|
opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
|