pi-mega-compact 0.8.26 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +12 -9
  2. package/dist/extensions/mega-compact.js +16 -1
  3. package/dist/extensions/mega-config.js +1 -0
  4. package/dist/extensions/mega-pipeline/compact.js +2 -1
  5. package/dist/extensions/mega-runtime/reset-runtime.js +8 -0
  6. package/dist/extensions/mega-runtime/runtime.js +12 -50
  7. package/dist/extensions/mega-shutdown-widget.test.js +121 -0
  8. package/dist/src/compact.js +4 -2
  9. package/dist/src/dedup/raptor/tree.js +11 -0
  10. package/dist/src/memory.test.js +29 -0
  11. package/dist/src/memoryOps.js +4 -19
  12. package/dist/src/memoryRecall.test.js +27 -0
  13. package/dist/src/memoryRoundtrip.test.js +137 -0
  14. package/dist/src/recall.js +5 -4
  15. package/dist/src/sprint4x-rag-verification.test.js +93 -0
  16. package/dist/src/store/memoryIndex.js +29 -7
  17. package/dist/src/store/pgOpenGuard.js +83 -0
  18. package/dist/src/store/pgOpenGuard.test.js +74 -0
  19. package/dist/src/store/repoKey.js +45 -0
  20. package/dist/src/store/vectorIndex.js +30 -8
  21. package/dist/src/store/vectorIndex.test.js +25 -1
  22. package/dist/src/vector-search.js +11 -5
  23. package/dist/src/vectorStore.js +4 -1
  24. package/extensions/mega-compact.ts +16 -1
  25. package/extensions/mega-config.ts +8 -0
  26. package/extensions/mega-pipeline/compact.ts +2 -1
  27. package/extensions/mega-runtime/reset-runtime.ts +88 -0
  28. package/extensions/mega-runtime/runtime.ts +27 -59
  29. package/extensions/mega-shutdown-widget.test.ts +141 -0
  30. package/package.json +1 -1
  31. package/src/compact.ts +209 -174
  32. package/src/dedup/raptor/tree.ts +11 -0
  33. package/src/memory.test.ts +47 -1
  34. package/src/memoryOps.ts +4 -19
  35. package/src/memoryRecall.test.ts +36 -0
  36. package/src/memoryRoundtrip.test.ts +155 -0
  37. package/src/recall.ts +6 -3
  38. package/src/sprint4x-rag-verification.test.ts +119 -0
  39. package/src/store/memoryIndex.ts +34 -8
  40. package/src/store/pgOpenGuard.test.ts +89 -0
  41. package/src/store/pgOpenGuard.ts +93 -0
  42. package/src/store/repoKey.ts +50 -0
  43. package/src/store/vectorIndex.test.ts +25 -1
  44. package/src/store/vectorIndex.ts +35 -9
  45. package/src/vector-search.ts +10 -5
  46. package/src/vectorStore.ts +4 -1
@@ -40,6 +40,8 @@ export interface VectorIndexHit {
40
40
  score: number;
41
41
  }
42
42
 
43
+ import { withOpenTimeout } from "./pgOpenGuard.js";
44
+
43
45
  let db: PGliteInstance | undefined;
44
46
  let initPromise: Promise<PGliteInstance | undefined> | undefined;
45
47
  let disabled = false;
@@ -131,12 +133,19 @@ async function openPgLite(
131
133
  if (!mod) return undefined;
132
134
  const dir = indexDir();
133
135
  mkdirSync(dir, { recursive: true });
134
- const pg = await new mod.PGlite({
135
- dataDir: dir,
136
- extensions: { vector: mod.vector },
137
- });
138
- await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
139
- await pg.exec(`
136
+ // Bounded open: PGlite is single-writer over a shared dataDir, so a second
137
+ // pi process on the same dir can block here forever. Without the ceiling the
138
+ // never-settling promise gets cached in initPromise and every later caller
139
+ // awaits it — which is how a stalled index wedged a whole pi turn.
140
+ let openTimedOut = false;
141
+ const pg = await withOpenTimeout(
142
+ (async () => {
143
+ const inst = await new mod.PGlite({
144
+ dataDir: dir,
145
+ extensions: { vector: mod.vector },
146
+ });
147
+ await inst.exec("CREATE EXTENSION IF NOT EXISTS vector;");
148
+ await inst.exec(`
140
149
  CREATE TABLE IF NOT EXISTS vector_index (
141
150
  repo_id TEXT NOT NULL,
142
151
  session_id TEXT NOT NULL,
@@ -145,10 +154,27 @@ async function openPgLite(
145
154
  PRIMARY KEY (repo_id, session_id, checkpoint_id)
146
155
  );
147
156
  `);
148
- // HNSW index over cosine distance for fast NN. Created idempotently.
149
- await pg.exec(
150
- "CREATE INDEX IF NOT EXISTS vector_index_hnsw ON vector_index USING hnsw (embedding vector_cosine_ops);",
157
+ // HNSW index over cosine distance for fast NN. Created idempotently.
158
+ await inst.exec(
159
+ "CREATE INDEX IF NOT EXISTS vector_index_hnsw ON vector_index USING hnsw (embedding vector_cosine_ops);",
160
+ );
161
+ return inst;
162
+ })(),
163
+ (reason) => {
164
+ openTimedOut = true;
165
+ logWarn(`init ${reason}`);
166
+ },
151
167
  );
168
+ if (!pg) {
169
+ if (openTimedOut) {
170
+ // Don't leave the dead open cached, and don't retry on the next call —
171
+ // a contended dataDir would just burn another full timeout per caller.
172
+ // Same terminal state as any other init failure: fall back to the scan.
173
+ initPromise = undefined;
174
+ disabled = true;
175
+ }
176
+ return undefined;
177
+ }
152
178
  db = pg;
153
179
  return pg;
154
180
  } catch (err) {
@@ -14,6 +14,7 @@
14
14
  import { cosineSimilarity } from "./embedder.js";
15
15
  import { normalizeSessionId } from "./store.js";
16
16
  import { mmrRerank, type MmrItem } from "./dedup/mmr.js";
17
+ import { stateDirForRepo } from "./store/repoKey.js";
17
18
  import { topK } from "./dedup/topk.js";
18
19
  import {
19
20
  listCheckpoints,
@@ -320,13 +321,17 @@ export async function vectorSearchAsync(
320
321
  // Index empty/unavailable → synchronous per-session fallback (this repo).
321
322
  return vectorSearch(store, sid, query, k);
322
323
  }
323
- // Hydrate each index hit from the authoritative node:sqlite store. repoId is
324
- // that repo's stateDir, so cross-repo hits resolve against their own store.
325
- // Tag cross-repo hits with their source repoId so the recall block can label
326
- // them ("from repo <name>"); same-repo hits stay unlabeled.
324
+ // Hydrate each index hit from the authoritative node:sqlite store. The
325
+ // index keys on repo_id (S25: git root via repoKey; legacy rows keyed by
326
+ // stateDir) resolve repo_id stateDir via stateDirForRepo, and skip
327
+ // unresolvable/foreign hits (degrade, never crash). Cross-repo hits carry
328
+ // their source repoId so the recall block can label them; same-repo stays
329
+ // unlabeled.
327
330
  const hydrated: SearchHit[] = [];
328
331
  for (const h of indexHits) {
329
- const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
332
+ const hitStateDir = stateDirForRepo(h.repoId);
333
+ if (!hitStateDir) continue;
334
+ const cp = getCheckpoint(h.sessionId, h.checkpointId, hitStateDir);
330
335
  if (cp && cp.dedupStatus !== "removed") {
331
336
  const crossRepo =
332
337
  opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
@@ -17,6 +17,7 @@ import {
17
17
  type DedupTier,
18
18
  } from "./config/dedup.js";
19
19
  import { logDecision } from "./monitoring.js";
20
+ import { repoKey } from "./store/repoKey.js";
20
21
  import type { StoredCheckpoint } from "./store.js";
21
22
  import { getStateDir, normalizeSessionId, compressSmart } from "./store.js";
22
23
  import { computeContentDigest } from "./dedup/digest.js";
@@ -149,7 +150,9 @@ export class VectorStore {
149
150
  ) {
150
151
  this.embedder = opts.embedder ?? defaultEmbedder();
151
152
  this.stateDir = opts.stateDir ?? getStateDir();
152
- this.repoId = opts.repoId ?? this.stateDir;
153
+ // S25: single repo-scope key shared with the memory index (git-root
154
+ // scoped; falls back to stateDir outside git).
155
+ this.repoId = opts.repoId ?? repoKey(this.stateDir);
153
156
  // Sprint 14: all tier flags/thresholds flow from the single config source
154
157
  // (DedupConfig). The legacy opts.dedupSim / opts.l2Enabled remain accepted
155
158
  // for backward-compat callers but flags are authoritative via `cfg`.