pi-memoria 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/runtime.ts DELETED
@@ -1,1469 +0,0 @@
1
- /**
2
- * MemoriaRuntime: binds configuration, store roots, in-memory indexes, file
3
- * watchers and write paths into one object that tools and commands share.
4
- *
5
- * Responsibilities beyond the store layer:
6
- *
7
- * - Serialize every read-modify-write per root (`KeyedMutex`).
8
- * - Keep an LRU of recent searches, invalidated by any mutation.
9
- * - Track hits/writes/last-used (`UsageStore`) outside the search path.
10
- * - Merge several roots with rank normalization so no store is drowned.
11
- * - Report health: duplicate topics, contradictions, staleness, promotion.
12
- */
13
-
14
- import { existsSync } from "node:fs";
15
- import { watch, type FSWatcher } from "node:fs";
16
- import { homedir } from "node:os";
17
- import { basename, join, relative } from "node:path";
18
- import { DEFAULT_CONFIG, INDEX_DIR, LIBRARY_DIR, loadConfig, resolveRoots, resolveStorePath, type PathContext, type ResolvedRoots } from "./config.ts";
19
- import { MemoryIndex, createIndexSaver, isIgnoredWatchPath } from "./index-engine.ts";
20
- import {
21
- addHotEntry,
22
- categorySummary,
23
- createMemory,
24
- deleteMemory,
25
- ensureStore,
26
- extractFactUnits,
27
- hotTemplate,
28
- looksAtomicTopic,
29
- moveMemory,
30
- priorityRank,
31
- readHot,
32
- recoverJournal,
33
- removeHotMatches,
34
- renderLibraryIndexes,
35
- resolveMemoryRef,
36
- touchLastUsed,
37
- updateMemory,
38
- writeHot,
39
- validateHot,
40
- type CreateMemoryInput,
41
- type CreateMemoryResult,
42
- type IndexListingDoc,
43
- type MoveMemoryInput,
44
- type MoveMemoryResult,
45
- type UpdateMemoryInput,
46
- } from "./store.ts";
47
- import { SessionStore } from "./sessions.ts";
48
- import { UsageStore } from "./usage.ts";
49
- import { compareDocs, detectContradictions, isComparableTerm, suggestMerges, type SimilarityDoc } from "./similarity.ts";
50
- import { expandSynonymTable, loadSynonymsFile, mergeSynonymTables } from "./synonyms.ts";
51
- import { KeyedMutex, atomicWriteFile, debounce, normalizePath, oneLine, readFileOrUndefined } from "./util.ts";
52
- import type {
53
- Contradiction,
54
- HotFileState,
55
- MemoriaConfig,
56
- MemoryDoc,
57
- MergeSuggestion,
58
- Scope,
59
- SearchHit,
60
- SearchOptions,
61
- SearchResult,
62
- SessionHit,
63
- SessionReadResult,
64
- SessionScanStats,
65
- SessionSearchOptions,
66
- SessionSearchResult,
67
- StoreStats,
68
- TimeWindow,
69
- } from "./types.ts";
70
-
71
- interface StoreState {
72
- root: string;
73
- index: MemoryIndex;
74
- saver: ReturnType<typeof createIndexSaver>;
75
- usage?: UsageStore;
76
- usageSaver: ReturnType<typeof debounce<[]>>;
77
- /** Merged per-store synonym table (config + synonyms.json). */
78
- synonyms?: { table: Record<string, string[]>; loadedAt: number };
79
- /** Coalesces watcher bursts into one refresh. */
80
- settleTimer?: ReturnType<typeof setTimeout>;
81
- /** Notes whose `last_used` frontmatter still needs refreshing. */
82
- pendingTouch: Set<string>;
83
- watcher?: FSWatcher;
84
- /** Description of a crash-recovered move, surfaced by stats/doctor. */
85
- recovered?: string;
86
- }
87
-
88
- /** Orders candidate ids after an optional model rerank. */
89
- export type Reranker = (query: string, hits: SearchHit[], options: { topK: number; timeoutMs: number; model: string }) => Promise<string[] | undefined>;
90
-
91
- export interface RuntimeSearchResult extends SearchResult {
92
- /** Per-root breakdown, useful for diagnostics. */
93
- byRoot: Array<{ root: string; hits: number; tookMs: number }>;
94
- /** True when the result came from the LRU instead of the index. */
95
- cached?: boolean;
96
- /** True when a model reranker reordered the hits. */
97
- reranked?: boolean;
98
- /**
99
- * Excerpts from past conversations, attached only when the library returned
100
- * nothing. These are evidence, not curated memory: the caller must label
101
- * them and point at `memoria_sessions` for verification.
102
- */
103
- sessionHits?: SessionHit[];
104
- sessionStats?: SessionScanStats;
105
- }
106
-
107
- export interface TopicReportEntry {
108
- id: string;
109
- title: string;
110
- relPath: string;
111
- root: string;
112
- category: string;
113
- /** Number of fact units in the body. */
114
- facts: number;
115
- bytes: number;
116
- updated: number;
117
- lastUsed: number;
118
- hits: number;
119
- writes: number;
120
- /** True when the note has not been retrieved within `staleAfterDays`. */
121
- stale: boolean;
122
- }
123
-
124
- const SEARCH_CACHE_TTL_MS = 120_000;
125
- /** Most notes touched with `last_used` in one flush, to bound write bursts. */
126
- const MAX_TOUCH_PER_FLUSH = 20;
127
-
128
- /**
129
- * Cheap `{id, path, aliases}` view of the index, used to resolve references
130
- * without touching the filesystem.
131
- */
132
- function liveLookup(index: MemoryIndex): Array<{ id: string; path: string; aliases: string[] }> {
133
- return index.liveDocs().map((entry) => ({ id: entry.meta.id, path: entry.meta.path, aliases: entry.meta.aliases ?? [] }));
134
- }
135
-
136
- export class MemoriaRuntime {
137
- config: MemoriaConfig = { ...DEFAULT_CONFIG };
138
- roots: ResolvedRoots;
139
- private stores = new Map<string, StoreState>();
140
- /** Share one first load per root; callers never see a half-loaded index. */
141
- private loadingStores = new Map<string, Promise<StoreState>>();
142
- /** Serializes read-modify-write sequences per store root. */
143
- private locks = new KeyedMutex();
144
- private initialized = false;
145
- private cwd: string;
146
- private pathContext: PathContext;
147
- private reranker?: Reranker;
148
- private searchCache = new Map<string, { result: RuntimeSearchResult; generation: number; at: number }>();
149
- private cacheGeneration = 0;
150
- private sessionStore?: SessionStore;
151
- private sessionStoreKey = "";
152
-
153
- /**
154
- * @param cwd Working directory of the session (only used for relative paths).
155
- * @param options.home Home directory for `~` expansion.
156
- * @param options.agentDir Pi's agent config directory. The extension passes
157
- * pi's own `getAgentDir()` so `$AGENT_DIR` follows
158
- * `PI_CODING_AGENT_DIR`; it defaults to `~/.pi/agent`.
159
- */
160
- constructor(cwd: string, options: { home?: string; agentDir?: string } = {}) {
161
- this.cwd = cwd;
162
- const home = options.home ?? homedir();
163
- this.pathContext = { home, agentDir: options.agentDir ?? join(home, ".pi", "agent") };
164
- this.roots = resolveRoots(cwd, this.config, this.pathContext);
165
- }
166
-
167
- setCwd(cwd: string): void {
168
- if (cwd === this.cwd) return;
169
- this.cwd = cwd;
170
- this.roots = resolveRoots(cwd, this.config, this.pathContext);
171
- }
172
-
173
- /** Install (or clear) the optional model reranker. */
174
- setReranker(reranker: Reranker | undefined): void {
175
- this.reranker = reranker;
176
- }
177
-
178
- /** Load config, create stores and start watching. Idempotent. */
179
- async init(force = false, reloadConfig = true): Promise<void> {
180
- if (this.initialized && !force) return;
181
- this.initialized = true;
182
- this.roots = resolveRoots(this.cwd, this.config, this.pathContext);
183
- const projectExists = existsSync(this.roots.primary);
184
- if (reloadConfig) {
185
- // Only auto-create the project store once memoria is initialized by use;
186
- // reads fall back to an empty store when it does not exist yet.
187
- this.config = await safeLoadConfig(this.roots.primary);
188
- }
189
- this.roots = resolveRoots(this.cwd, this.config, this.pathContext);
190
- if (force || projectExists) await ensureStore(this.roots.primary, this.config.hotLimit, this.config.defaultCategory);
191
- const active = this.activeRoots("all");
192
- for (const root of active) {
193
- await this.storeFor(root, true);
194
- }
195
- await this.primeSynonyms();
196
- }
197
-
198
- /** Replace the active configuration and rebuild stores (used when config changes). */
199
- async reconfigure(config: MemoriaConfig): Promise<void> {
200
- await this.dispose();
201
- this.clearSessionCache();
202
- this.config = config;
203
- this.roots = resolveRoots(this.cwd, this.config, this.pathContext);
204
- await this.init(true, false);
205
- }
206
-
207
- /** Load config + ensure store without starting watchers (used by commands). */
208
- async prepare(create = true): Promise<void> {
209
- const projectExists = existsSync(this.roots.primary);
210
- if (projectExists || !create) this.config = await safeLoadConfig(this.roots.primary);
211
- this.roots = resolveRoots(this.cwd, this.config, this.pathContext);
212
- if (create) await ensureStore(this.roots.primary, this.config.hotLimit, this.config.defaultCategory);
213
- }
214
-
215
- configure(config: MemoriaConfig): void {
216
- this.config = config;
217
- this.roots = resolveRoots(this.cwd, this.config, this.pathContext);
218
- }
219
-
220
- /* ---------------------------------------------------------------- */
221
- /* Stores */
222
- /* ---------------------------------------------------------------- */
223
-
224
- private activeRoots(scope: Scope = "all"): string[] {
225
- const normalized = normalizeScope(scope);
226
- const available: string[] = [];
227
- const push = (root: string | undefined): void => {
228
- if (root && existsSync(root) && !available.includes(root)) available.push(root);
229
- };
230
- if (normalized === "project") {
231
- push(this.roots.project);
232
- return available;
233
- }
234
- push(this.roots.primary);
235
- if (normalized === "all") {
236
- push(this.roots.project);
237
- for (const extra of this.roots.extras) push(extra);
238
- }
239
- return available;
240
- }
241
-
242
- /**
243
- * The root a write should target.
244
- *
245
- * `primary` is the user-level store; `project` is the optional writable
246
- * per-project store; `all` writes to the primary store. `extraRoots` are
247
- * read-only additions by design.
248
- */
249
- rootForScope(scope: Scope): string {
250
- const normalized = normalizeScope(scope);
251
- if (normalized === "project" && this.roots.project) return this.roots.project;
252
- return this.roots.primary;
253
- }
254
-
255
- private async storeFor(root: string, startWatcher = false): Promise<StoreState> {
256
- let state = this.stores.get(root);
257
- if (state) {
258
- if (startWatcher) this.ensureWatcher(state);
259
- return state;
260
- }
261
- let loading = this.loadingStores.get(root);
262
- if (!loading) {
263
- loading = (async (): Promise<StoreState> => {
264
- const index = new MemoryIndex(root, {
265
- bodyCacheBytes: this.config.bodyCacheBytes,
266
- maxScanIntervalMs: this.config.scanIntervalMs,
267
- exclude: this.config.exclude,
268
- indexFormat: this.config.indexFormat,
269
- });
270
- const saver = createIndexSaver(index);
271
- const loaded: StoreState = {
272
- root,
273
- index,
274
- saver,
275
- usageSaver: debounce(() => {
276
- void this.flushUsage(root, loaded).catch(() => {});
277
- }, 4000),
278
- pendingTouch: new Set(),
279
- };
280
- const recovered = await recoverJournal(root).catch(() => undefined);
281
- if (recovered) loaded.recovered = recovered;
282
- await index.load();
283
- if (this.config.usageTracking) loaded.usage = await UsageStore.load(root).catch(() => undefined);
284
- this.stores.set(root, loaded);
285
- return loaded;
286
- })();
287
- this.loadingStores.set(root, loading);
288
- }
289
- try {
290
- state = await loading;
291
- } finally {
292
- if (this.loadingStores.get(root) === loading) this.loadingStores.delete(root);
293
- }
294
- if (startWatcher) this.ensureWatcher(state);
295
- return state;
296
- }
297
-
298
- private ensureWatcher(state: StoreState): void {
299
- if (state.watcher) return;
300
- try {
301
- state.watcher = watch(state.root, { recursive: true }, (_event, filename) => {
302
- const name = filename ? normalizePath(String(filename)) : "";
303
- if (isIgnoredWatchPath(name)) return;
304
- state!.index.dirty = true;
305
- // Coalesce bursts (editors write several files at once, memories are
306
- // moved in pairs): refresh once the filesystem settles instead of on
307
- // every event.
308
- this.scheduleSettle(state!);
309
- });
310
- state.watcher.on("error", () => {
311
- state.watcher?.close();
312
- state.watcher = undefined;
313
- state.index.watcherActive = false;
314
- });
315
- state.index.watcherActive = true;
316
- } catch {
317
- state.index.watcherActive = false;
318
- }
319
- }
320
-
321
- private scheduleSettle(state: StoreState): void {
322
- const delay = this.config.watcherSettleMs;
323
- if (delay <= 0) return;
324
- if (state.settleTimer) clearTimeout(state.settleTimer);
325
- state.settleTimer = setTimeout(() => {
326
- state.settleTimer = undefined;
327
- void state.index.refresh(false).then((changed) => {
328
- if (changed) {
329
- this.bumpCacheGeneration();
330
- state.saver.schedule();
331
- }
332
- });
333
- }, delay);
334
- state.settleTimer.unref?.();
335
- }
336
-
337
- /** Get an index for a root, loading it lazily. */
338
- async indexFor(root: string): Promise<MemoryIndex> {
339
- const state = await this.storeFor(root);
340
- return state.index;
341
- }
342
-
343
- /** Invalidate the search LRU (any mutation or external change). */
344
- private bumpCacheGeneration(): void {
345
- this.cacheGeneration += 1;
346
- this.searchCache.clear();
347
- }
348
-
349
- /** Every refresh that changes an index must also invalidate cached recall. */
350
- private async refreshIndex(index: MemoryIndex, force = false): Promise<void> {
351
- if (await index.refresh(force)) this.bumpCacheGeneration();
352
- }
353
-
354
- /* ---------------------------------------------------------------- */
355
- /* Search */
356
- /* ---------------------------------------------------------------- */
357
-
358
- /**
359
- * Search every active root and merge the results.
360
- *
361
- * Scores are rank-normalized *upward* only: a store whose best hit scores
362
- * lower purely because its corpus is small is lifted to the global best
363
- * (`scale = globalMax / rootMax`, capped at 4). Nothing is ever demoted, so
364
- * the raw scale that `autoRecallMinScore` is calibrated against survives.
365
- */
366
- async search(query: string, options: SearchOptions & { scope?: Scope } = {}): Promise<RuntimeSearchResult> {
367
- const started = performance.now();
368
- const scope = options.scope ?? "all";
369
- const roots = this.activeRoots(scope);
370
- if (roots.length === 0) {
371
- return { hits: [], total: 0, tookMs: 0, truncated: false, missing: [], byRoot: [] };
372
- }
373
- const limit = Math.max(1, options.limit ?? 8);
374
- // A cached result is only valid after each root has had a chance to observe
375
- // watcher events or its periodic freshness scan. Both checks are cheap when
376
- // the index is already fresh.
377
- const states = await Promise.all(roots.map((root) => this.storeFor(root)));
378
- await Promise.all(states.map((state) => this.refreshIndex(state.index, options.refresh === true)));
379
- const synonymTables = await Promise.all(states.map((state) => this.synonymsFor(state.root)));
380
- const cacheKey = this.searchCacheKey(query, options);
381
- const cached = this.lookupCache(cacheKey);
382
- if (cached) {
383
- if (cached.timeWindow) options.onTimeWindow?.(cached.timeWindow);
384
- this.recordHits(cached.hits);
385
- return { ...cached, cached: true, tookMs: Math.round((performance.now() - started) * 1000) / 1000 };
386
- }
387
-
388
- const perRoot = await Promise.all(
389
- states.map(async (state, indexInRoots) => {
390
- const root = state.root;
391
- const index = state.index;
392
- const result = index.searchWarm(query, {
393
- ...options,
394
- limit: Math.max(limit, 8),
395
- synonyms: synonymTables[indexInRoots],
396
- synonymWeight: this.config.synonymWeight,
397
- snippetChars: this.config.snippetChars,
398
- relatedHits: options.relatedHits ?? this.config.relatedHits,
399
- relatedBoost: options.relatedBoost ?? this.config.relatedBoost,
400
- timeHints: options.timeHints ?? this.config.timeHints,
401
- });
402
- return { root, result };
403
- }),
404
- );
405
-
406
- const globalMax = Math.max(0, ...perRoot.map((entry) => entry.result.hits[0]?.score ?? 0));
407
- const merged: SearchHit[] = [];
408
- const seenIds = new Set<string>();
409
- let total = 0;
410
- let truncated = false;
411
- let timeWindow: TimeWindow | undefined;
412
- const missing = new Set<string>();
413
- const byRoot: RuntimeSearchResult["byRoot"] = [];
414
- for (const entry of perRoot) {
415
- total += entry.result.total;
416
- truncated = truncated || entry.result.truncated;
417
- for (const term of entry.result.missing) missing.add(term);
418
- if (entry.result.timeWindow) timeWindow = entry.result.timeWindow;
419
- const rootMax = entry.result.hits[0]?.score ?? 0;
420
- const scale = rootMax > 0 && rootMax < globalMax ? Math.min(4, globalMax / rootMax) : 1;
421
- for (const hit of entry.result.hits) {
422
- // Ids are unique per store, not across stores: a note that exists in
423
- // two roots is returned once (the first root wins).
424
- if (seenIds.has(hit.doc.id)) continue;
425
- seenIds.add(hit.doc.id);
426
- merged.push(scale === 1 ? hit : { ...hit, score: Math.round(hit.score * scale * 1000) / 1000 });
427
- }
428
- byRoot.push({ root: entry.root, hits: entry.result.hits.length, tookMs: entry.result.tookMs });
429
- }
430
- merged.sort((a, b) => b.score - a.score);
431
- const result: RuntimeSearchResult = {
432
- hits: merged.slice(0, limit),
433
- total,
434
- tookMs: Math.round((performance.now() - started) * 1000) / 1000,
435
- truncated,
436
- missing: [...missing].slice(0, 8),
437
- timeWindow,
438
- byRoot,
439
- };
440
- this.storeCache(cacheKey, result);
441
- this.recordHits(result.hits);
442
- return result;
443
- }
444
-
445
- /** Search, then optionally reorder the top candidates with the model. */
446
- async recall(query: string, options: SearchOptions & { scope?: Scope; rerank?: boolean; sessionFallback?: boolean } = {}): Promise<RuntimeSearchResult> {
447
- const result = await this.search(query, options);
448
- // "Not in memory" is the case session transcripts exist for: attach a
449
- // small, clearly-labelled set of excerpts so the model can check what was
450
- // actually said before claiming it does not know.
451
- if (result.hits.length === 0 && options.sessionFallback !== false && this.config.sessionFallback) {
452
- const fallback = await this.sessionSearch(query, {
453
- limit: 2,
454
- // The fallback runs inside a prompt, so it gets a small slice of the
455
- // search budget; the explicit tool call can take longer.
456
- budgetMs: Math.min(this.config.sessionScanMs, 400),
457
- }).catch(() => undefined);
458
- if (fallback) {
459
- // Stats are kept even with no hits: they carry the ripgrep status the
460
- // caller warns about, and the renderers only show the excerpts when
461
- // `sessionHits` is non-empty.
462
- result.sessionStats = fallback.stats;
463
- if (fallback.hits.length > 0) result.sessionHits = fallback.hits;
464
- }
465
- }
466
- const wanted = options.rerank ?? this.config.rerank;
467
- if (!wanted || !this.reranker || result.hits.length < 2) return result;
468
- const topK = Math.max(2, Math.min(this.config.rerankTopK, result.hits.length));
469
- const candidates = result.hits.slice(0, topK);
470
- try {
471
- const order = await this.reranker(query, candidates, {
472
- topK,
473
- timeoutMs: this.config.rerankTimeoutMs,
474
- model: this.config.rerankModel,
475
- });
476
- if (!order || order.length === 0) return result;
477
- const position = new Map(order.map((id, index) => [id, index]));
478
- const reordered = [...candidates].sort((a, b) => {
479
- const pa = position.get(a.doc.id) ?? Number.MAX_SAFE_INTEGER;
480
- const pb = position.get(b.doc.id) ?? Number.MAX_SAFE_INTEGER;
481
- return pa - pb || b.score - a.score;
482
- });
483
- return { ...result, hits: [...reordered, ...result.hits.slice(topK)], reranked: true };
484
- } catch {
485
- // Reranking is best-effort: a failure must never break recall.
486
- return result;
487
- }
488
- }
489
-
490
- private searchCacheKey(query: string, options: SearchOptions & { scope?: Scope }): string {
491
- const normalized = {
492
- q: query,
493
- parts: options.parts ?? [],
494
- limit: options.limit ?? 8,
495
- root: options.root ?? "",
496
- category: options.category ?? "",
497
- tags: options.tags ?? [],
498
- anyTags: options.anyTags ?? [],
499
- minPriority: options.minPriority ?? "",
500
- since: options.since ?? 0,
501
- includeUnfiled: options.includeUnfiled !== false,
502
- minScore: options.minScore ?? 0,
503
- prefix: options.prefix !== false,
504
- fuzzy: options.fuzzy !== false,
505
- expandRelated: options.expandRelated !== false,
506
- relatedHits: options.relatedHits ?? this.config.relatedHits,
507
- relatedBoost: options.relatedBoost ?? this.config.relatedBoost,
508
- dropSuperseded: options.dropSuperseded === true,
509
- timeHints: options.timeHints ?? this.config.timeHints,
510
- explain: Boolean(options.explain),
511
- scope: options.scope ?? "all",
512
- };
513
- return `${this.activeRoots(options.scope ?? "all").join("|")}::${JSON.stringify(normalized)}`;
514
- }
515
-
516
- private lookupCache(key: string): RuntimeSearchResult | undefined {
517
- if (this.config.searchCacheSize <= 0 || !key) return undefined;
518
- const entry = this.searchCache.get(key);
519
- if (!entry) return undefined;
520
- if (entry.generation !== this.cacheGeneration || Date.now() - entry.at > SEARCH_CACHE_TTL_MS) {
521
- this.searchCache.delete(key);
522
- return undefined;
523
- }
524
- // Refresh recency.
525
- this.searchCache.delete(key);
526
- this.searchCache.set(key, entry);
527
- return entry.result;
528
- }
529
-
530
- private storeCache(key: string, result: RuntimeSearchResult): void {
531
- if (this.config.searchCacheSize <= 0 || !key) return;
532
- if (this.searchCache.size >= this.config.searchCacheSize) {
533
- const oldest = this.searchCache.keys().next();
534
- if (!oldest.done) this.searchCache.delete(oldest.value);
535
- }
536
- this.searchCache.set(key, { result, generation: this.cacheGeneration, at: Date.now() });
537
- }
538
-
539
- /** Count retrieved notes for promotion/staleness tracking (never blocks). */
540
- private recordHits(hits: SearchHit[]): void {
541
- if (!this.config.usageTracking) return;
542
- const now = Date.now();
543
- for (const hit of hits) {
544
- const state = this.stores.get(hit.doc.root);
545
- if (!state?.usage) continue;
546
- state.usage.record(hit.doc.id, "hit", now);
547
- state.pendingTouch.add(hit.doc.id);
548
- state.usageSaver();
549
- }
550
- }
551
-
552
- /** Persist usage and refresh `last_used` frontmatter for recently used notes. */
553
- private async flushUsage(root: string, state: StoreState): Promise<void> {
554
- const usage = state.usage;
555
- if (!usage) return;
556
- await this.locks.run(root, async () => {
557
- const validIds = new Set(state.index.liveDocs().map((entry) => entry.meta.id));
558
- if (usage.size > validIds.size * 1.2) usage.prune(validIds);
559
- let touched = 0;
560
- for (const id of [...state.pendingTouch]) {
561
- if (touched >= MAX_TOUCH_PER_FLUSH) break;
562
- const entry = usage.get(id);
563
- if (!entry) continue;
564
- const idx = state.index.idxForId(id);
565
- if (idx === undefined) continue;
566
- const doc = await resolveMemoryRef(root, id, liveLookup(state.index));
567
- state.pendingTouch.delete(id);
568
- if (!doc) continue;
569
- if (entry.lastUsed - (doc.lastUsed || 0) < this.config.lastUsedWriteIntervalMs) continue;
570
- const updated = await touchLastUsed(root, doc, entry.lastUsed).catch(() => undefined);
571
- if (!updated) continue;
572
- state.index.removeDoc(idx);
573
- state.index.addDoc(updated);
574
- state.saver.schedule();
575
- touched += 1;
576
- }
577
- await usage.save().catch(() => {});
578
- });
579
- }
580
-
581
- /** Flush every pending `last_used` update immediately (shutdown). */
582
- async flushUsageNow(): Promise<void> {
583
- for (const state of this.stores.values()) {
584
- state.usageSaver.flush();
585
- await this.flushUsage(state.root, state).catch(() => {});
586
- }
587
- }
588
-
589
- /* ---------------------------------------------------------------- */
590
- /* Past sessions */
591
- /* ---------------------------------------------------------------- */
592
-
593
- /** Roots searched for saved transcripts: config extras first, then pi's own. */
594
- sessionRoots(): string[] {
595
- const roots: string[] = [];
596
- const push = (root: string): void => {
597
- if (root && !roots.includes(root)) roots.push(root);
598
- };
599
- for (const extra of this.config.sessionRoots) {
600
- const resolved = resolveStorePath(extra, this.cwd, this.pathContext);
601
- if (resolved) push(resolved);
602
- }
603
- push(join(this.pathContext.agentDir, "sessions"));
604
- push(join(this.pathContext.agentDir, "sessions-archive"));
605
- return roots;
606
- }
607
-
608
- /** Lazily create the transcript store (keys on config + agent dir). */
609
- private sessions(): SessionStore {
610
- const roots = this.sessionRoots();
611
- const key = `${roots.join("|")}::${this.config.sessionCacheBytes}::${this.config.sessionExcerptChars}::${this.config.sessionRipgrep}`;
612
- if (!this.sessionStore || this.sessionStoreKey !== key) {
613
- this.sessionStore = new SessionStore({
614
- roots,
615
- cacheBytes: this.config.sessionCacheBytes,
616
- excerptChars: this.config.sessionExcerptChars,
617
- rgPath: this.config.sessionRipgrep ? undefined : null,
618
- // pi ships ripgrep under <agent dir>/bin and puts it on PATH.
619
- rgCandidates: [join(this.pathContext.agentDir, "bin", process.platform === "win32" ? "rg.exe" : "rg")],
620
- });
621
- this.sessionStoreKey = key;
622
- }
623
- return this.sessionStore;
624
- }
625
-
626
- /**
627
- * Search saved session transcripts. Read-only, bounded and always safe to
628
- * call: a missing sessions directory yields an empty result.
629
- */
630
- async sessionSearch(query: string, options: SessionSearchOptions = {}): Promise<SessionSearchResult> {
631
- const empty: SessionSearchResult = {
632
- hits: [],
633
- stats: { roots: [], files: 0, messages: 0, bytes: 0, cachedFiles: 0, skipped: 0, partial: false },
634
- tookMs: 0,
635
- };
636
- if (!this.config.sessionSearch) return empty;
637
- const store = this.sessions();
638
- const haveRoots = store.roots.some((root) => existsSync(root));
639
- if (!haveRoots) return { ...empty, stats: { ...empty.stats, roots: store.roots } };
640
- const result = await store.search(query, {
641
- ...options,
642
- includeTools: options.includeTools ?? this.config.sessionIncludeTools,
643
- budgetMs: options.budgetMs ?? this.config.sessionScanMs,
644
- });
645
- this.lastSessionSearchMs = result.tookMs;
646
- return result;
647
- }
648
-
649
- /** Read a window of a transcript around a hit line (verification step). */
650
- async sessionRead(path: string, line: number, window = 6): Promise<SessionReadResult | undefined> {
651
- if (!this.config.sessionSearch) return undefined;
652
- return this.sessions().readWindow(path, line, window);
653
- }
654
-
655
- /** Drop the transcript cache (used when config changes). */
656
- clearSessionCache(): void {
657
- this.sessionStore?.clear();
658
- this.sessionStore = undefined;
659
- this.sessionStoreKey = "";
660
- }
661
-
662
- /** Milliseconds of the last transcript scan, for status reporting. */
663
- lastSessionSearchMs = 0;
664
-
665
- /* ---------------------------------------------------------------- */
666
- /* Writes */
667
- /* ---------------------------------------------------------------- */
668
-
669
- async write(input: CreateMemoryInput & { scope?: Scope }): Promise<{ result: CreateMemoryResult; root: string }> {
670
- const root = this.rootForScope(input.scope ?? "primary");
671
- await ensureStore(root, this.config.hotLimit, this.config.defaultCategory);
672
- const written = await this.locks.run(root, () => this.writeLocked(root, input));
673
- this.bumpCacheGeneration();
674
- const state = this.stores.get(root);
675
- if (state?.usage) {
676
- state.usage.record(written.result.doc.id, "write");
677
- state.usageSaver();
678
- }
679
- return written;
680
- }
681
-
682
- private async writeLocked(root: string, input: CreateMemoryInput): Promise<{ result: CreateMemoryResult; root: string }> {
683
- const state = await this.storeFor(root);
684
- const existingIdx = input.id ? state.index.idxForId(input.id) ?? -1 : -1;
685
- const existingDoc =
686
- existingIdx >= 0
687
- ? await resolveMemoryRef(root, input.id!, liveLookup(state.index))
688
- : undefined;
689
- const previousByPath = state.index.idxForRelPath(input.id && existingDoc ? existingDoc.relPath : "");
690
- const result = await createMemory(root, { ...input, existingDoc }, this.config.defaultCategory, {
691
- topicMaxWords: this.config.topicMaxWords,
692
- topicMaxChars: this.config.topicMaxChars,
693
- });
694
- if (existingIdx >= 0) state.index.removeDoc(existingIdx);
695
- if (previousByPath >= 0 && previousByPath !== existingIdx) state.index.removeDoc(previousByPath);
696
- const stale = state.index.idxForRelPath(result.relPath);
697
- if (stale >= 0) state.index.removeDoc(stale);
698
- state.index.addDoc(result.doc);
699
- state.index.builtAt = Date.now();
700
- state.saver.schedule();
701
- return { result, root };
702
- }
703
-
704
- async updateMemoryById(
705
- id: string,
706
- input: UpdateMemoryInput & { scope?: Scope },
707
- ): Promise<{ doc: MemoryDoc; root: string } | undefined> {
708
- const candidates = this.activeRoots(input.scope ?? "primary");
709
- for (const root of candidates) {
710
- const locked = await this.locks.run(root, async () => this.updateMemoryByIdLocked(root, id, input));
711
- if (locked) {
712
- this.bumpCacheGeneration();
713
- return locked;
714
- }
715
- }
716
- return undefined;
717
- }
718
-
719
- private async updateMemoryByIdLocked(root: string, id: string, input: UpdateMemoryInput): Promise<{ doc: MemoryDoc; root: string } | undefined> {
720
- const state = await this.storeFor(root);
721
- const idx = state.index.idxForId(id);
722
- if (idx === undefined) return undefined;
723
- const doc = await resolveMemoryRef(root, id, liveLookup(state.index));
724
- if (!doc) return undefined;
725
- const updated = await updateMemory(root, doc, input);
726
- state.index.removeDoc(idx);
727
- state.index.addDoc(updated);
728
- state.index.builtAt = Date.now();
729
- state.saver.schedule();
730
- return { doc: updated, root };
731
- }
732
-
733
- /**
734
- * Re-file a note under a better broad topic, merging into the target note
735
- * when one already exists there.
736
- */
737
- async move(
738
- from: string,
739
- input: MoveMemoryInput,
740
- scope: Scope = "primary",
741
- ): Promise<{ result: MoveMemoryResult; root: string } | undefined> {
742
- const candidates = this.activeRoots(scope);
743
- for (const root of candidates) {
744
- const locked = await this.locks.run(root, async () => this.moveLocked(root, from, input));
745
- if (locked) {
746
- this.bumpCacheGeneration();
747
- return locked;
748
- }
749
- }
750
- return undefined;
751
- }
752
-
753
- private async moveLocked(root: string, from: string, input: MoveMemoryInput): Promise<{ result: MoveMemoryResult; root: string } | undefined> {
754
- const state = await this.storeFor(root);
755
- const doc = await resolveMemoryRef(root, from, liveLookup(state.index));
756
- if (!doc) return undefined;
757
- const sourceIdx = state.index.idxForRelPath(doc.relPath);
758
- const result = await moveMemory(root, doc, input, this.config.defaultCategory, {
759
- topicMaxWords: this.config.topicMaxWords,
760
- topicMaxChars: this.config.topicMaxChars,
761
- });
762
- if (sourceIdx >= 0) state.index.removeDoc(sourceIdx);
763
- // Replacing the target's index entry must happen after removing the source:
764
- // merging keeps the target's id but the source's path may have been indexed.
765
- const targetIdx = state.index.idxForRelPath(result.doc.relPath);
766
- if (targetIdx >= 0) state.index.removeDoc(targetIdx);
767
- const staleIdx = state.index.idxForId(result.doc.id);
768
- if (staleIdx !== undefined) state.index.removeDoc(staleIdx);
769
- state.index.addDoc(result.doc);
770
- state.index.builtAt = Date.now();
771
- state.saver.schedule();
772
- return { result, root };
773
- }
774
-
775
- async forget(id: string, scope: Scope = "primary"): Promise<{ doc: MemoryDoc; trashPath: string; root: string } | undefined> {
776
- const candidates = this.activeRoots(scope);
777
- for (const root of candidates) {
778
- const locked = await this.locks.run(root, async () => this.forgetLocked(root, id));
779
- if (locked) {
780
- this.bumpCacheGeneration();
781
- return locked;
782
- }
783
- }
784
- return undefined;
785
- }
786
-
787
- private async forgetLocked(root: string, id: string): Promise<{ doc: MemoryDoc; trashPath: string; root: string } | undefined> {
788
- const state = await this.storeFor(root);
789
- const idx = state.index.idxForId(id);
790
- if (idx === undefined) return undefined;
791
- const doc = await resolveMemoryRef(root, id, liveLookup(state.index));
792
- if (!doc) return undefined;
793
- const trashPath = await deleteMemory(root, doc);
794
- state.index.removeDoc(idx);
795
- state.index.builtAt = Date.now();
796
- state.saver.schedule();
797
- if (state.usage) {
798
- state.usage.prune(new Set(state.index.liveDocs().map((entry) => entry.meta.id)));
799
- state.usageSaver();
800
- }
801
- return { doc, trashPath, root };
802
- }
803
-
804
- async readMemory(ref: string, scope: Scope = "all"): Promise<{ doc: MemoryDoc; root: string } | undefined> {
805
- const candidates = this.activeRoots(scope);
806
- for (const root of candidates) {
807
- const state = await this.storeFor(root);
808
- const lookup = liveLookup(state.index);
809
- const doc = await resolveMemoryRef(root, ref, lookup);
810
- if (doc) return { doc, root };
811
- }
812
- return undefined;
813
- }
814
-
815
- /**
816
- * Notes that declare this one superseded, plus the notes it relates to.
817
- * Used by `memoria_read` so the agent never acts on a replaced decision.
818
- */
819
- async linksFor(id: string, scope: Scope = "all"): Promise<{ supersededBy: Array<{ id: string; title: string }>; related: Array<{ id: string; title: string; relPath: string }> }> {
820
- const supersededBy: Array<{ id: string; title: string }> = [];
821
- const related: Array<{ id: string; title: string; relPath: string }> = [];
822
- for (const root of this.activeRoots(scope)) {
823
- const state = await this.storeFor(root);
824
- const index = state.index;
825
- const idx = index.idxForId(id);
826
- if (idx === undefined) continue;
827
- const meta = index.docs[idx];
828
- for (const [supersededId, superseding] of index.supersedeMap()) {
829
- if (supersededId !== id) continue;
830
- for (const supersedingId of superseding) {
831
- const at = index.idxForId(supersedingId);
832
- if (at === undefined) continue;
833
- supersededBy.push({ id: supersedingId, title: index.docs[at].title });
834
- }
835
- }
836
- for (const target of index.relatedDocs(idx)) {
837
- const targetMeta = index.docs[target];
838
- related.push({ id: targetMeta.id, title: targetMeta.title, relPath: targetMeta.relPath });
839
- }
840
- for (const ref of meta.supersedes ?? []) {
841
- const at = index.resolveLink(ref);
842
- if (at === undefined) continue;
843
- supersededBy.push({ id: index.docs[at].id, title: index.docs[at].title });
844
- }
845
- }
846
- return { supersededBy, related };
847
- }
848
-
849
- /* ---------------------------------------------------------------- */
850
- /* MEMORY.md */
851
- /* ---------------------------------------------------------------- */
852
-
853
- async hotState(root = this.roots.primary): Promise<HotFileState> {
854
- const state = await readHot(root, this.config.hotLimit);
855
- return { content: state.content, chars: state.chars, limit: state.limit, over: state.over, path: state.path, mtimeMs: 0 };
856
- }
857
-
858
- async hotAdd(
859
- text: string,
860
- topic?: string,
861
- root = this.roots.primary,
862
- ): Promise<{ before: number; after: number; limit: number; paragraph: string; created: boolean }> {
863
- return this.locks.run(root, async () => {
864
- await ensureStore(root, this.config.hotLimit, this.config.defaultCategory);
865
- const state = await readHot(root, this.config.hotLimit);
866
- const content = state.exists ? state.content : hotTemplate();
867
- const added = addHotEntry(content, text, topic);
868
- if (added.changed) await writeHot(root, added.content, this.config.hotLimit);
869
- return {
870
- before: state.chars,
871
- after: added.content.length,
872
- limit: this.config.hotLimit,
873
- paragraph: added.paragraph,
874
- created: added.created,
875
- };
876
- });
877
- }
878
-
879
- async hotRemove(pattern: string, root = this.roots.primary): Promise<{ before: number; after: number; removed: number }> {
880
- return this.locks.run(root, async () => {
881
- const state = await readHot(root, this.config.hotLimit);
882
- const [next, removed] = removeHotMatches(state.content, pattern);
883
- await writeHot(root, next, this.config.hotLimit);
884
- return { before: state.chars, after: next.length, removed };
885
- });
886
- }
887
-
888
- async hotReplace(content: string, root = this.roots.primary): Promise<{ before: number; after: number }> {
889
- return this.locks.run(root, async () => {
890
- const state = await readHot(root, this.config.hotLimit);
891
- await writeHot(root, content, this.config.hotLimit);
892
- return { before: state.chars, after: validateHot(content, this.config.hotLimit).length };
893
- });
894
- }
895
-
896
- /** Check the briefing without deleting facts to meet the budget. */
897
- async hotCompact(root = this.roots.primary): Promise<{ before: number; after: number; removed: string }> {
898
- return this.locks.run(root, async () => {
899
- const state = await readHot(root, this.config.hotLimit);
900
- if (!state.over) {
901
- validateHot(state.content, this.config.hotLimit);
902
- return { before: state.chars, after: state.chars, removed: "" };
903
- }
904
- throw new Error("MEMORY.md is over budget. Read it and use action=replace with a tighter briefing preserving standing facts; compact never drops facts automatically.");
905
- });
906
- }
907
-
908
- /* ---------------------------------------------------------------- */
909
- /* Library maintenance */
910
- /* ---------------------------------------------------------------- */
911
-
912
- async stats(): Promise<{ roots: StoreStats[]; hot: HotFileState; config: MemoriaConfig; recovered?: string[] }> {
913
- const roots = this.activeRoots("all");
914
- const stats: StoreStats[] = [];
915
- const recovered: string[] = [];
916
- for (const root of roots) {
917
- const state = await this.storeFor(root);
918
- if (state.recovered) recovered.push(`${this.displayPath(root)}: ${state.recovered}`);
919
- stats.push(state.index.stats(state.index.persistedBytes));
920
- }
921
- return { roots: stats, hot: await this.hotState(), config: this.config, ...(recovered.length > 0 ? { recovered } : {}) };
922
- }
923
-
924
- async rebuild(scope: Scope = "all"): Promise<Array<{ root: string; docs: number; tookMs: number }>> {
925
- const roots = this.activeRoots(scope);
926
- const out: Array<{ root: string; docs: number; tookMs: number }> = [];
927
- for (const root of roots) {
928
- const index = await this.indexFor(root);
929
- const started = performance.now();
930
- const docs = await index.rebuild();
931
- out.push({ root, docs, tookMs: Math.round((performance.now() - started) * 100) / 100 });
932
- }
933
- this.bumpCacheGeneration();
934
- return out;
935
- }
936
-
937
- async regenerateIndexes(scope: Scope = "all"): Promise<Array<{ root: string; files: number }>> {
938
- const roots = this.activeRoots(scope);
939
- const out: Array<{ root: string; files: number }> = [];
940
- for (const root of roots) {
941
- const index = await this.indexFor(root);
942
- await this.refreshIndex(index, true);
943
- const docs: IndexListingDoc[] = index.liveDocs().map(({ meta }) => ({
944
- relPath: meta.relPath,
945
- title: meta.title,
946
- id: meta.id,
947
- category: meta.category,
948
- tags: meta.tags,
949
- summary: meta.summary,
950
- priority: meta.priority,
951
- updated: meta.updated,
952
- }));
953
- const files = renderLibraryIndexes(docs);
954
- for (const [relPath, content] of files) {
955
- const target = join(root, LIBRARY_DIR, relPath);
956
- const existing = await readFileOrUndefined(target);
957
- if (existing === content) continue;
958
- await atomicWriteFile(target, content);
959
- }
960
- out.push({ root, files: files.size });
961
- }
962
- return out;
963
- }
964
-
965
- /** Per-store synonym table (config inline table + `synonyms.json`). */
966
- async synonymsFor(root: string): Promise<Record<string, string[]>> {
967
- const state = await this.storeFor(root);
968
- const now = Date.now();
969
- if (state.synonyms && now - state.synonyms.loadedAt < 30_000) return state.synonyms.table;
970
- const file = await loadSynonymsFile(root).catch(() => ({}));
971
- const table = expandSynonymTable(mergeSynonymTables(this.config.synonyms, file));
972
- if (state.synonyms && JSON.stringify(state.synonyms.table) !== JSON.stringify(table)) this.bumpCacheGeneration();
973
- state.synonyms = { table, loadedAt: now };
974
- return table;
975
- }
976
-
977
- /** Load the synonym table into every active store (called on init). */
978
- private async primeSynonyms(): Promise<void> {
979
- for (const root of this.activeRoots("all")) {
980
- const state = await this.storeFor(root);
981
- if (state.synonyms) continue;
982
- const file = await loadSynonymsFile(root).catch(() => ({}));
983
- state.synonyms = { table: expandSynonymTable(mergeSynonymTables(this.config.synonyms, file)), loadedAt: Date.now() };
984
- }
985
- }
986
-
987
- /** Weighted similarity metadata for every live note in a root. */
988
- private similarityDocs(index: MemoryIndex): SimilarityDoc[] {
989
- return index.liveDocs().map(({ idx, meta }) => ({
990
- id: meta.id,
991
- title: meta.title,
992
- relPath: meta.relPath,
993
- category: meta.category,
994
- aliases: meta.aliases ?? [],
995
- tokens: index.docTokens(idx),
996
- }));
997
- }
998
-
999
- /** IDF function over an index, for similarity scoring. */
1000
- private idfFor(index: MemoryIndex): (term: string) => number {
1001
- const live = Math.max(2, index.aliveCount);
1002
- return (term: string) => {
1003
- const termId = index.termId(term);
1004
- const df = termId >= 0 ? index.documentFrequency(termId) : 1;
1005
- return Math.log(1 + live / (1 + df)) + 0.5;
1006
- };
1007
- }
1008
-
1009
- /** Note pairs that look like the same subject (suggest `memoria_move`). */
1010
- async mergeSuggestions(limit = 20, scope: Scope = "all"): Promise<MergeSuggestion[]> {
1011
- const out: MergeSuggestion[] = [];
1012
- for (const root of this.activeRoots(scope)) {
1013
- const index = await this.indexFor(root);
1014
- await this.refreshIndex(index);
1015
- out.push(...suggestMerges(this.similarityDocs(index), { threshold: this.config.dedupeThreshold, limit }));
1016
- }
1017
- return out.sort((a, b) => b.score - a.score).slice(0, limit);
1018
- }
1019
-
1020
- /**
1021
- * Notes that look like the same subject as `id`, using only cheap posting
1022
- * lookups (safe to call on every write, unlike `mergeSuggestions`).
1023
- */
1024
- async similarNotes(id: string, limit = 2, scope: Scope = "all"): Promise<Array<{ id: string; title: string; relPath: string; score: number; shared: string[]; linked: boolean }>> {
1025
- const out: Array<{ id: string; title: string; relPath: string; score: number; shared: string[]; linked: boolean }> = [];
1026
- for (const root of this.activeRoots(scope)) {
1027
- const index = await this.indexFor(root);
1028
- const idx = index.idxForId(id);
1029
- if (idx === undefined) continue;
1030
- const byIdx = new Map<number, SimilarityDoc>();
1031
- for (const entry of index.liveDocs()) {
1032
- byIdx.set(entry.idx, {
1033
- id: entry.meta.id,
1034
- title: entry.meta.title,
1035
- relPath: entry.meta.relPath,
1036
- category: entry.meta.category,
1037
- aliases: entry.meta.aliases ?? [],
1038
- tokens: index.docTokens(entry.idx),
1039
- });
1040
- }
1041
- const target = byIdx.get(idx);
1042
- if (!target) continue;
1043
- const idfOf = this.idfFor(index);
1044
- const maxDf = Math.max(2, Math.ceil(Math.max(2, index.aliveCount) * 0.2));
1045
- // Candidate generation is posting-driven: only notes that share a
1046
- // distinctive term can be near-duplicates, so this stays cheap enough
1047
- // to run after every write.
1048
- const candidates = new Set<number>();
1049
- for (const term of Object.keys(target.tokens)) {
1050
- if (!isComparableTerm(term)) continue;
1051
- const termId = index.termId(term);
1052
- if (termId < 0 || index.documentFrequency(termId) > maxDf) continue;
1053
- const posting = index.postings[termId];
1054
- for (let k = 0; k + 1 < posting.length; k += 2) {
1055
- const other = posting[k];
1056
- if (other === idx || !index.alive[other]) continue;
1057
- if (index.docs[other].category !== index.docs[idx].category) continue;
1058
- candidates.add(other);
1059
- }
1060
- }
1061
- for (const other of candidates) {
1062
- const otherDoc = byIdx.get(other);
1063
- if (!otherDoc) continue;
1064
- const similarity = compareDocs(target, otherDoc, idfOf, { threshold: this.config.dedupeThreshold });
1065
- if (!similarity.duplicate) continue;
1066
- const meta = index.docs[other];
1067
- out.push({
1068
- id: meta.id,
1069
- title: meta.title,
1070
- relPath: meta.relPath,
1071
- score: Math.round(similarity.score * 1000) / 1000,
1072
- shared: [...new Set(similarity.shared)].slice(0, 6),
1073
- linked: similarity.linked,
1074
- });
1075
- }
1076
- }
1077
- return out.sort((a, b) => b.score - a.score).slice(0, limit);
1078
- }
1079
-
1080
- /** Fact units that appear to disagree, across the store. */
1081
- async contradictions(limit = 20, scope: Scope = "all"): Promise<Contradiction[]> {
1082
- const out: Contradiction[] = [];
1083
- for (const root of this.activeRoots(scope)) {
1084
- const index = await this.indexFor(root);
1085
- await this.refreshIndex(index);
1086
- const units: Array<{ id: string; relPath: string; label?: string; text: string }> = [];
1087
- for (const { idx, meta } of index.liveDocs()) {
1088
- // Reading every body would make doctor O(store). A contradiction needs
1089
- // a negation, so only notes whose preview has one are read.
1090
- if (!/\b(not|never|no longer|stopped|avoid|avoids|dislike|dislikes|deprecated|cancel|instead)\b/i.test(meta.preview)) continue;
1091
- const body = await index.bodyFor(idx);
1092
- for (const unit of extractFactUnits(body)) units.push({ id: meta.id, relPath: meta.relPath, label: unit.label, text: unit.text });
1093
- if (units.length > 4000) break;
1094
- }
1095
- out.push(...detectContradictions(units, { limit }));
1096
- }
1097
- return out.sort((a, b) => b.shared.length - a.shared.length).slice(0, limit);
1098
- }
1099
-
1100
- /** Notes that have not been retrieved for `staleAfterDays`. */
1101
- async staleNotes(limit = 20, scope: Scope = "all"): Promise<Array<{ id: string; title: string; relPath: string; root: string; idleMs: number; hits: number; writes: number }>> {
1102
- const out: Array<{ id: string; title: string; relPath: string; root: string; idleMs: number; hits: number; writes: number }> = [];
1103
- const threshold = this.config.staleAfterDays * 86_400_000;
1104
- for (const root of this.activeRoots(scope)) {
1105
- const state = await this.storeFor(root);
1106
- if (!state.usage) continue;
1107
- for (const { id, entry, idleMs } of state.usage.stale(threshold, Date.now(), 0)) {
1108
- const idx = state.index.idxForId(id);
1109
- if (idx === undefined) continue;
1110
- const meta = state.index.docs[idx];
1111
- out.push({ id, title: meta.title, relPath: meta.relPath, root, idleMs, hits: entry.hits, writes: entry.writes });
1112
- }
1113
- }
1114
- return out.sort((a, b) => b.idleMs - a.idleMs).slice(0, limit);
1115
- }
1116
-
1117
- /** Notes written often enough to be worth promoting into MEMORY.md. */
1118
- async promotionCandidates(limit = 10, scope: Scope = "primary"): Promise<Array<{ id: string; title: string; relPath: string; root: string; writes: number; hits: number; priority: string }>> {
1119
- const out: Array<{ id: string; title: string; relPath: string; root: string; writes: number; hits: number; priority: string }> = [];
1120
- for (const root of this.activeRoots(scope)) {
1121
- const state = await this.storeFor(root);
1122
- if (!state.usage) continue;
1123
- const hot = await readHot(root, this.config.hotLimit);
1124
- for (const { id, entry } of state.usage.promotionCandidates(this.config.promoteAfterWrites)) {
1125
- const idx = state.index.idxForId(id);
1126
- if (idx === undefined) continue;
1127
- const meta = state.index.docs[idx];
1128
- if (priorityRank(meta.priority) < priorityRank("high")) continue;
1129
- // Already represented in the always-loaded file? Then say nothing.
1130
- const facts = extractFactUnits(state.index.bodyOrPreview(idx)).map((unit) => unit.text);
1131
- if (hotContains(hot.content, meta.title, meta.aliases, facts)) continue;
1132
- out.push({ id, title: meta.title, relPath: meta.relPath, root, writes: entry.writes, hits: entry.hits, priority: meta.priority });
1133
- }
1134
- }
1135
- return out.sort((a, b) => b.writes - a.writes).slice(0, limit);
1136
- }
1137
-
1138
- /** Per-topic review data for `/memoria topics`. */
1139
- async topicsReport(scope: Scope = "all"): Promise<TopicReportEntry[]> {
1140
- const out: TopicReportEntry[] = [];
1141
- for (const root of this.activeRoots(scope)) {
1142
- const state = await this.storeFor(root);
1143
- await this.refreshIndex(state.index);
1144
- const staleThreshold = this.config.staleAfterDays * 86_400_000;
1145
- const now = Date.now();
1146
- for (const { idx, meta } of state.index.liveDocs()) {
1147
- const body = await state.index.bodyFor(idx);
1148
- const usage = state.usage?.get(meta.id);
1149
- const lastUsed = Math.max(meta.lastUsed || 0, usage?.lastUsed ?? 0);
1150
- out.push({
1151
- id: meta.id,
1152
- title: meta.title,
1153
- relPath: meta.relPath,
1154
- root,
1155
- category: meta.category,
1156
- facts: extractFactUnits(body).length,
1157
- bytes: meta.size,
1158
- updated: meta.updated,
1159
- lastUsed,
1160
- hits: usage?.hits ?? 0,
1161
- writes: usage?.writes ?? 0,
1162
- stale: lastUsed > 0 && now - lastUsed >= staleThreshold,
1163
- });
1164
- }
1165
- }
1166
- return out.sort((a, b) => a.category.localeCompare(b.category) || b.updated - a.updated);
1167
- }
1168
-
1169
- /**
1170
- * What changed in the primary store since the previous session started.
1171
- *
1172
- * Advances the session marker as a side effect, so it must be called once
1173
- * per session (from `session_start`). Returns undefined on the first session.
1174
- */
1175
- async sessionDelta(limit = 5): Promise<{ since: number; created: DiffEntry[]; updated: DiffEntry[] } | undefined> {
1176
- const root = this.roots.primary;
1177
- const state = await this.storeFor(root);
1178
- const usage = state.usage;
1179
- const previous = usage?.sessionStartedAt() ?? 0;
1180
- usage?.markSessionStart();
1181
- state.usageSaver();
1182
- if (!previous) return undefined;
1183
- const index = state.index;
1184
- await this.refreshIndex(index);
1185
- const created: DiffEntry[] = [];
1186
- const updated: DiffEntry[] = [];
1187
- for (const { meta } of index.liveDocs()) {
1188
- // `>=` on purpose: a note written in the same millisecond as the marker
1189
- // still belongs to this session's summary.
1190
- if (meta.updated < previous && meta.created < previous) continue;
1191
- const entry: DiffEntry = {
1192
- id: meta.id,
1193
- title: meta.title,
1194
- relPath: meta.relPath,
1195
- category: meta.category,
1196
- created: meta.created,
1197
- updated: meta.updated,
1198
- };
1199
- if (meta.created >= previous) created.push(entry);
1200
- else updated.push(entry);
1201
- }
1202
- const byRecency = (a: DiffEntry, b: DiffEntry): number => b.updated - a.updated;
1203
- return { since: previous, created: created.sort(byRecency).slice(0, limit), updated: updated.sort(byRecency).slice(0, limit) };
1204
- }
1205
-
1206
- /** Notes created and updated within the last `days`. */
1207
- async diffReport(days = 7, scope: Scope = "all"): Promise<{ since: number; created: DiffEntry[]; updated: DiffEntry[] }> {
1208
- const since = Date.now() - Math.max(1, days) * 86_400_000;
1209
- const created: DiffEntry[] = [];
1210
- const updated: DiffEntry[] = [];
1211
- for (const root of this.activeRoots(scope)) {
1212
- const index = await this.indexFor(root);
1213
- await this.refreshIndex(index);
1214
- for (const { meta } of index.liveDocs()) {
1215
- const entry: DiffEntry = {
1216
- id: meta.id,
1217
- title: meta.title,
1218
- relPath: meta.relPath,
1219
- category: meta.category,
1220
- created: meta.created,
1221
- updated: meta.updated,
1222
- };
1223
- if (meta.created >= since) created.push(entry);
1224
- else if (meta.updated >= since) updated.push(entry);
1225
- }
1226
- }
1227
- const byRecency = (a: DiffEntry, b: DiffEntry): number => b.updated - a.updated;
1228
- return { since, created: created.sort(byRecency), updated: updated.sort(byRecency) };
1229
- }
1230
-
1231
- /**
1232
- * One pass over the store producing both structured findings and the
1233
- * human-readable list: `/memoria doctor` renders the structure, the tool and
1234
- * the agent get the strings.
1235
- */
1236
- async health(): Promise<{
1237
- findings: string[];
1238
- merges: MergeSuggestion[];
1239
- contradictions: Contradiction[];
1240
- stale: Awaited<ReturnType<MemoriaRuntime["staleNotes"]>>;
1241
- promotions: Awaited<ReturnType<MemoriaRuntime["promotionCandidates"]>>;
1242
- recovered: string[];
1243
- }> {
1244
- const findings: string[] = [];
1245
- const recovered: string[] = [];
1246
- const merges: MergeSuggestion[] = [];
1247
- const contradictions: Contradiction[] = [];
1248
- for (const root of this.activeRoots("all")) {
1249
- const state = await this.storeFor(root);
1250
- const index = state.index;
1251
- await this.refreshIndex(index, true);
1252
- const label = this.displayPath(root);
1253
- if (state.recovered) {
1254
- recovered.push(`${label}: ${state.recovered}`);
1255
- findings.push(`${root}: ${state.recovered}`);
1256
- }
1257
- if (!existsSync(join(root, "MEMORY.md"))) findings.push(`${root}: MEMORY.md is missing`);
1258
- const hot = await readHot(root, this.config.hotLimit);
1259
- if (hot.over) findings.push(`${root}: MEMORY.md over budget (${hot.chars}/${this.config.hotLimit})`);
1260
- const seenIds = new Map<string, string>();
1261
- const seenAliases = new Map<string, string>();
1262
- const negatedUnits: Array<{ id: string; relPath: string; label?: string; text: string }> = [];
1263
- for (const { idx, meta } of index.liveDocs()) {
1264
- if (seenIds.has(meta.id)) findings.push(`${root}: duplicate id ${meta.id} in ${meta.relPath} and ${seenIds.get(meta.id)}`);
1265
- else seenIds.set(meta.id, meta.relPath);
1266
- if (!meta.title) findings.push(`${root}: ${meta.relPath} has no title`);
1267
- if (!meta.summary) findings.push(`${root}: ${meta.relPath} has no summary`);
1268
- if (!meta.tags.length) findings.push(`${root}: ${meta.relPath} has no tags`);
1269
- if (meta.bodyChars < 8) findings.push(`${root}: ${meta.relPath} has an empty body`);
1270
- // Fact- and relation-shaped file names are a legacy pattern; suggest
1271
- // consolidating them under the subject's real name.
1272
- const fileBase = basename(meta.relPath).replace(/\.md$/i, "").replace(/-\d+$/, "").replace(/[-_]+/g, " ");
1273
- if (looksAtomicTopic(fileBase)) {
1274
- findings.push(
1275
- `${root}: ${meta.relPath} is named after a single fact or relationship; re-file it with memoria_move (e.g. under the person's name) so the name in the file stays true`,
1276
- );
1277
- }
1278
- for (const alias of meta.aliases ?? []) {
1279
- const key = alias.toLowerCase();
1280
- const owner = seenAliases.get(key);
1281
- if (owner && owner !== meta.relPath) findings.push(`${root}: alias "${alias}" is claimed by both ${owner} and ${meta.relPath}`);
1282
- else seenAliases.set(key, meta.relPath);
1283
- }
1284
- // Dangling links make `related`/`supersedes` silently useless.
1285
- for (const [field, links] of [
1286
- ["related", meta.related ?? []],
1287
- ["supersedes", meta.supersedes ?? []],
1288
- ] as const) {
1289
- for (const ref of links) {
1290
- if (index.resolveLink(ref) === undefined) findings.push(`${root}: ${meta.relPath} has a ${field} link to "${ref}" that no longer exists`);
1291
- }
1292
- }
1293
- // Contradiction candidates need a body; only notes whose preview
1294
- // carries a negation are read, which keeps doctor off the O(store) path.
1295
- if (/(not|never|no longer|stopped|avoid|avoids|dislike|dislikes|deprecated|cancel|instead)/i.test(meta.preview)) {
1296
- const body = await index.bodyFor(idx);
1297
- for (const unit of extractFactUnits(body)) negatedUnits.push({ id: meta.id, relPath: meta.relPath, label: unit.label, text: unit.text });
1298
- }
1299
- }
1300
- for (const [category] of categorySummary(index.liveDocs().map((entry) => entry.meta))) {
1301
- const dir = category === "(root)" ? join(root, LIBRARY_DIR) : join(root, LIBRARY_DIR, category);
1302
- if (!existsSync(dir)) findings.push(`${root}: missing category directory ${dir}`);
1303
- }
1304
- const rootMerges = suggestMerges(this.similarityDocs(index), { threshold: this.config.dedupeThreshold, limit: 5 });
1305
- for (const suggestion of rootMerges) {
1306
- merges.push(suggestion);
1307
- findings.push(
1308
- `${root}: ${suggestion.a.relPath} and ${suggestion.b.relPath} look like the same subject (${Math.round(suggestion.score * 100)}% overlap: ${suggestion.shared.join(", ")}); consider memoria_move { from: "${suggestion.b.id}", topic: "${suggestion.a.title}", merge: true }`,
1309
- );
1310
- }
1311
- const rootContradictions = detectContradictions(negatedUnits, { limit: 5 });
1312
- for (const contradiction of rootContradictions) {
1313
- contradictions.push(contradiction);
1314
- findings.push(
1315
- `${root}: ${contradiction.a.relPath} may contradict ${contradiction.b.relPath} (${contradiction.shared.join(", ")}): "${oneLine(contradiction.a.text, 80)}" vs "${oneLine(contradiction.b.text, 80)}". If one replaced the other, link them with memoria_write { id: "...", supersedes: ["..."] } or merge with memoria_move.`,
1316
- );
1317
- }
1318
- }
1319
- const stale = await this.staleNotes(5, "all");
1320
- for (const entry of stale) {
1321
- findings.push(`${this.displayPath(entry.root)}: ${entry.relPath} has not been used in over ${this.config.staleAfterDays} days (${entry.hits} hits)`);
1322
- }
1323
- const promotions = await this.promotionCandidates(5, "primary");
1324
- for (const entry of promotions) {
1325
- findings.push(
1326
- `${this.displayPath(entry.root)}: ${entry.relPath} was written ${entry.writes} times with priority ${entry.priority}; consider promoting it into MEMORY.md with memoria_hot`,
1327
- );
1328
- }
1329
- return { findings, merges, contradictions, stale, promotions, recovered };
1330
- }
1331
-
1332
- /** Check the store for consistency problems (agent-facing string list). */
1333
- async doctor(): Promise<string[]> {
1334
- return (await this.health()).findings;
1335
- }
1336
-
1337
- /** Aggregated overview across every active root (cheap, for the system section). */
1338
- async overview(recentLimit = 4): Promise<{ docs: number; categories: Array<[string, number]>; recent: Array<{ id: string; title: string; relPath: string }> }> {
1339
- const roots = this.activeRoots("all");
1340
- const counts = new Map<string, number>();
1341
- const recent: Array<{ id: string; title: string; relPath: string; created: number }> = [];
1342
- let docs = 0;
1343
- for (const root of roots) {
1344
- const index = await this.indexFor(root);
1345
- await this.refreshIndex(index);
1346
- const overview = index.overview(recentLimit);
1347
- docs += overview.docs;
1348
- for (const [category, count] of overview.categories) counts.set(category, (counts.get(category) ?? 0) + count);
1349
- recent.push(...overview.recent);
1350
- }
1351
- recent.sort((a, b) => b.created - a.created);
1352
- return {
1353
- docs,
1354
- categories: [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])),
1355
- recent: recent.slice(0, recentLimit).map(({ id, title, relPath }) => ({ id, title, relPath })),
1356
- };
1357
- }
1358
-
1359
- async listEntries(scope: Scope = "all"): Promise<Array<{ root: string; meta: ReturnType<MemoryIndex["liveDocs"]>[number]["meta"] }>> {
1360
- const roots = this.activeRoots(scope);
1361
- const out: Array<{ root: string; meta: ReturnType<MemoryIndex["liveDocs"]>[number]["meta"] }> = [];
1362
- const seenIds = new Set<string>();
1363
- for (const root of roots) {
1364
- const index = await this.indexFor(root);
1365
- await this.refreshIndex(index);
1366
- for (const { meta } of index.liveDocs()) {
1367
- // Ids are unique per store, not across stores: list a shared note once.
1368
- if (seenIds.has(meta.id)) continue;
1369
- seenIds.add(meta.id);
1370
- out.push({ root, meta });
1371
- }
1372
- }
1373
- return out;
1374
- }
1375
-
1376
- /** Flush pending index writes and stop watchers. Idempotent. */
1377
- async dispose(): Promise<void> {
1378
- await Promise.allSettled(this.loadingStores.values());
1379
- const stores = [...this.stores.values()];
1380
- this.stores = new Map();
1381
- this.initialized = false;
1382
- this.searchCache.clear();
1383
- for (const state of stores) {
1384
- if (state.settleTimer) clearTimeout(state.settleTimer);
1385
- try {
1386
- state.watcher?.close();
1387
- } catch {
1388
- // ignore
1389
- }
1390
- state.watcher = undefined;
1391
- state.index.watcherActive = false;
1392
- state.usageSaver.cancel();
1393
- await this.flushUsage(state.root, state).catch(() => {});
1394
- await state.usage?.save().catch(() => {});
1395
- await state.saver.flush().catch(() => {});
1396
- }
1397
- }
1398
-
1399
- /** Absolute path helpers used by tools/commands. */
1400
- paths(): { primary: string; project: string; agentDir: string; extras: string[]; library: string; hot: string; indexDir: string; sessions: string[] } {
1401
- return {
1402
- primary: this.roots.primary,
1403
- project: this.roots.project,
1404
- agentDir: this.pathContext.agentDir,
1405
- extras: this.roots.extras,
1406
- library: join(this.roots.primary, LIBRARY_DIR),
1407
- hot: join(this.roots.primary, "MEMORY.md"),
1408
- indexDir: join(this.roots.primary, INDEX_DIR),
1409
- sessions: this.sessionRoots(),
1410
- };
1411
- }
1412
-
1413
- /** Human-readable relative path for UI messages. */
1414
- displayPath(path: string): string {
1415
- const projectRel = relative(this.cwd, path);
1416
- if (!projectRel.startsWith("..")) return normalizePath(projectRel);
1417
- const homeRel = relative(this.pathContext.home, path);
1418
- if (!homeRel.startsWith("..")) return `~/${normalizePath(homeRel)}`;
1419
- return normalizePath(path);
1420
- }
1421
- }
1422
-
1423
- export interface DiffEntry {
1424
- id: string;
1425
- title: string;
1426
- relPath: string;
1427
- category: string;
1428
- created: number;
1429
- updated: number;
1430
- }
1431
-
1432
- /**
1433
- * True when MEMORY.md already carries this note: by title/alias, or because one
1434
- * of the note's facts appears in the prose. The fact check is what makes the
1435
- * promotion hint actually stop after the agent has promoted a note, since
1436
- * MEMORY.md is prose and seldom repeats a note's title.
1437
- */
1438
- export function hotContains(hotContent: string, title: string, aliases: string[] = [], facts: string[] = []): boolean {
1439
- const haystack = normalizeProse(hotContent);
1440
- if (!haystack) return false;
1441
- if (title.length >= 4 && haystack.includes(normalizeProse(title))) return true;
1442
- for (const alias of aliases) {
1443
- if (alias.length >= 4 && haystack.includes(normalizeProse(alias))) return true;
1444
- }
1445
- for (const fact of facts) {
1446
- const needle = normalizeProse(fact);
1447
- if (needle.length >= 12 && haystack.includes(needle)) return true;
1448
- }
1449
- return false;
1450
- }
1451
-
1452
- function normalizeProse(text: string): string {
1453
- return text.toLowerCase().replace(/[`*_\[\]()]/g, " ").replace(/\s+/g, " ").trim();
1454
- }
1455
-
1456
- /** Collapse legacy scope names onto the current model. */
1457
- export function normalizeScope(scope: Scope | undefined): "primary" | "project" | "all" {
1458
- if (scope === "all") return "all";
1459
- if (scope === "project") return "project";
1460
- return "primary";
1461
- }
1462
-
1463
- async function safeLoadConfig(root: string): Promise<MemoriaConfig> {
1464
- try {
1465
- return await loadConfig(root);
1466
- } catch {
1467
- return { ...DEFAULT_CONFIG };
1468
- }
1469
- }