pi-mega-compact 0.8.26 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -9
- package/dist/extensions/mega-compact.js +16 -1
- package/dist/extensions/mega-config.js +1 -0
- package/dist/extensions/mega-pipeline/compact.js +2 -1
- package/dist/extensions/mega-runtime/reset-runtime.js +8 -0
- package/dist/extensions/mega-runtime/runtime.js +12 -50
- package/dist/extensions/mega-shutdown-widget.test.js +121 -0
- package/dist/src/compact.js +4 -2
- package/dist/src/dedup/raptor/tree.js +11 -0
- package/dist/src/memory.test.js +29 -0
- package/dist/src/memoryOps.js +4 -19
- package/dist/src/memoryRecall.test.js +27 -0
- package/dist/src/memoryRoundtrip.test.js +137 -0
- package/dist/src/recall.js +5 -4
- package/dist/src/sprint4x-rag-verification.test.js +93 -0
- package/dist/src/store/memoryIndex.js +29 -7
- package/dist/src/store/pgOpenGuard.js +83 -0
- package/dist/src/store/pgOpenGuard.test.js +74 -0
- package/dist/src/store/repoKey.js +45 -0
- package/dist/src/store/vectorIndex.js +30 -8
- package/dist/src/store/vectorIndex.test.js +25 -1
- package/dist/src/vector-search.js +11 -5
- package/dist/src/vectorStore.js +4 -1
- package/extensions/mega-compact.ts +16 -1
- package/extensions/mega-config.ts +8 -0
- package/extensions/mega-pipeline/compact.ts +2 -1
- package/extensions/mega-runtime/reset-runtime.ts +88 -0
- package/extensions/mega-runtime/runtime.ts +27 -59
- package/extensions/mega-shutdown-widget.test.ts +141 -0
- package/package.json +1 -1
- package/src/compact.ts +209 -174
- package/src/dedup/raptor/tree.ts +11 -0
- package/src/memory.test.ts +47 -1
- package/src/memoryOps.ts +4 -19
- package/src/memoryRecall.test.ts +36 -0
- package/src/memoryRoundtrip.test.ts +155 -0
- package/src/recall.ts +6 -3
- package/src/sprint4x-rag-verification.test.ts +119 -0
- package/src/store/memoryIndex.ts +34 -8
- package/src/store/pgOpenGuard.test.ts +89 -0
- package/src/store/pgOpenGuard.ts +93 -0
- package/src/store/repoKey.ts +50 -0
- package/src/store/vectorIndex.test.ts +25 -1
- package/src/store/vectorIndex.ts +35 -9
- package/src/vector-search.ts +10 -5
- package/src/vectorStore.ts +4 -1
package/src/store/vectorIndex.ts
CHANGED
|
@@ -40,6 +40,8 @@ export interface VectorIndexHit {
|
|
|
40
40
|
score: number;
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
+
import { withOpenTimeout } from "./pgOpenGuard.js";
|
|
44
|
+
|
|
43
45
|
let db: PGliteInstance | undefined;
|
|
44
46
|
let initPromise: Promise<PGliteInstance | undefined> | undefined;
|
|
45
47
|
let disabled = false;
|
|
@@ -131,12 +133,19 @@ async function openPgLite(
|
|
|
131
133
|
if (!mod) return undefined;
|
|
132
134
|
const dir = indexDir();
|
|
133
135
|
mkdirSync(dir, { recursive: true });
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
await
|
|
136
|
+
// Bounded open: PGlite is single-writer over a shared dataDir, so a second
|
|
137
|
+
// pi process on the same dir can block here forever. Without the ceiling the
|
|
138
|
+
// never-settling promise gets cached in initPromise and every later caller
|
|
139
|
+
// awaits it — which is how a stalled index wedged a whole pi turn.
|
|
140
|
+
let openTimedOut = false;
|
|
141
|
+
const pg = await withOpenTimeout(
|
|
142
|
+
(async () => {
|
|
143
|
+
const inst = await new mod.PGlite({
|
|
144
|
+
dataDir: dir,
|
|
145
|
+
extensions: { vector: mod.vector },
|
|
146
|
+
});
|
|
147
|
+
await inst.exec("CREATE EXTENSION IF NOT EXISTS vector;");
|
|
148
|
+
await inst.exec(`
|
|
140
149
|
CREATE TABLE IF NOT EXISTS vector_index (
|
|
141
150
|
repo_id TEXT NOT NULL,
|
|
142
151
|
session_id TEXT NOT NULL,
|
|
@@ -145,10 +154,27 @@ async function openPgLite(
|
|
|
145
154
|
PRIMARY KEY (repo_id, session_id, checkpoint_id)
|
|
146
155
|
);
|
|
147
156
|
`);
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
157
|
+
// HNSW index over cosine distance for fast NN. Created idempotently.
|
|
158
|
+
await inst.exec(
|
|
159
|
+
"CREATE INDEX IF NOT EXISTS vector_index_hnsw ON vector_index USING hnsw (embedding vector_cosine_ops);",
|
|
160
|
+
);
|
|
161
|
+
return inst;
|
|
162
|
+
})(),
|
|
163
|
+
(reason) => {
|
|
164
|
+
openTimedOut = true;
|
|
165
|
+
logWarn(`init ${reason}`);
|
|
166
|
+
},
|
|
151
167
|
);
|
|
168
|
+
if (!pg) {
|
|
169
|
+
if (openTimedOut) {
|
|
170
|
+
// Don't leave the dead open cached, and don't retry on the next call —
|
|
171
|
+
// a contended dataDir would just burn another full timeout per caller.
|
|
172
|
+
// Same terminal state as any other init failure: fall back to the scan.
|
|
173
|
+
initPromise = undefined;
|
|
174
|
+
disabled = true;
|
|
175
|
+
}
|
|
176
|
+
return undefined;
|
|
177
|
+
}
|
|
152
178
|
db = pg;
|
|
153
179
|
return pg;
|
|
154
180
|
} catch (err) {
|
package/src/vector-search.ts
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
import { cosineSimilarity } from "./embedder.js";
|
|
15
15
|
import { normalizeSessionId } from "./store.js";
|
|
16
16
|
import { mmrRerank, type MmrItem } from "./dedup/mmr.js";
|
|
17
|
+
import { stateDirForRepo } from "./store/repoKey.js";
|
|
17
18
|
import { topK } from "./dedup/topk.js";
|
|
18
19
|
import {
|
|
19
20
|
listCheckpoints,
|
|
@@ -320,13 +321,17 @@ export async function vectorSearchAsync(
|
|
|
320
321
|
// Index empty/unavailable → synchronous per-session fallback (this repo).
|
|
321
322
|
return vectorSearch(store, sid, query, k);
|
|
322
323
|
}
|
|
323
|
-
// Hydrate each index hit from the authoritative node:sqlite store.
|
|
324
|
-
//
|
|
325
|
-
//
|
|
326
|
-
//
|
|
324
|
+
// Hydrate each index hit from the authoritative node:sqlite store. The
|
|
325
|
+
// index keys on repo_id (S25: git root via repoKey; legacy rows keyed by
|
|
326
|
+
// stateDir) — resolve repo_id → stateDir via stateDirForRepo, and skip
|
|
327
|
+
// unresolvable/foreign hits (degrade, never crash). Cross-repo hits carry
|
|
328
|
+
// their source repoId so the recall block can label them; same-repo stays
|
|
329
|
+
// unlabeled.
|
|
327
330
|
const hydrated: SearchHit[] = [];
|
|
328
331
|
for (const h of indexHits) {
|
|
329
|
-
const
|
|
332
|
+
const hitStateDir = stateDirForRepo(h.repoId);
|
|
333
|
+
if (!hitStateDir) continue;
|
|
334
|
+
const cp = getCheckpoint(h.sessionId, h.checkpointId, hitStateDir);
|
|
330
335
|
if (cp && cp.dedupStatus !== "removed") {
|
|
331
336
|
const crossRepo =
|
|
332
337
|
opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
|
package/src/vectorStore.ts
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
type DedupTier,
|
|
18
18
|
} from "./config/dedup.js";
|
|
19
19
|
import { logDecision } from "./monitoring.js";
|
|
20
|
+
import { repoKey } from "./store/repoKey.js";
|
|
20
21
|
import type { StoredCheckpoint } from "./store.js";
|
|
21
22
|
import { getStateDir, normalizeSessionId, compressSmart } from "./store.js";
|
|
22
23
|
import { computeContentDigest } from "./dedup/digest.js";
|
|
@@ -149,7 +150,9 @@ export class VectorStore {
|
|
|
149
150
|
) {
|
|
150
151
|
this.embedder = opts.embedder ?? defaultEmbedder();
|
|
151
152
|
this.stateDir = opts.stateDir ?? getStateDir();
|
|
152
|
-
|
|
153
|
+
// S25: single repo-scope key shared with the memory index (git-root
|
|
154
|
+
// scoped; falls back to stateDir outside git).
|
|
155
|
+
this.repoId = opts.repoId ?? repoKey(this.stateDir);
|
|
153
156
|
// Sprint 14: all tier flags/thresholds flow from the single config source
|
|
154
157
|
// (DedupConfig). The legacy opts.dedupSim / opts.l2Enabled remain accepted
|
|
155
158
|
// for backward-compat callers but flags are authoritative via `cfg`.
|