pi-memoria 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +672 -0
- package/index.ts +260 -0
- package/package.json +71 -0
- package/skills/memoria/SKILL.md +65 -0
- package/skills/memoria-sessions/SKILL.md +66 -0
- package/src/commands.ts +472 -0
- package/src/config.ts +196 -0
- package/src/frontmatter.ts +318 -0
- package/src/index-engine.ts +1044 -0
- package/src/learn.ts +299 -0
- package/src/recall.ts +172 -0
- package/src/rerank.ts +85 -0
- package/src/runtime.ts +1633 -0
- package/src/search.ts +499 -0
- package/src/sessions.ts +592 -0
- package/src/similarity.ts +374 -0
- package/src/store.ts +1643 -0
- package/src/synonyms.ts +105 -0
- package/src/timeexpr.ts +145 -0
- package/src/tokenize.ts +295 -0
- package/src/tools.ts +935 -0
- package/src/transfer.ts +180 -0
- package/src/types.ts +533 -0
- package/src/usage.ts +174 -0
- package/src/util.ts +138 -0
package/src/types.ts
ADDED
|
@@ -0,0 +1,533 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Core data types for memoria.
|
|
3
|
+
*
|
|
4
|
+
* Keep this module free of runtime imports so tests and tooling can load it
|
|
5
|
+
* without the pi runtime being present.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export type Priority = "low" | "normal" | "high" | "critical";
|
|
9
|
+
export type Confidence = "low" | "medium" | "high";
|
|
10
|
+
/**
|
|
11
|
+
* Which store(s) an operation targets.
|
|
12
|
+
*
|
|
13
|
+
* - `primary` (default): the user-level store at `<agent dir>/memoria`
|
|
14
|
+
* (`~/.pi/agent/memoria` unless pi's agent dir is customized).
|
|
15
|
+
* - `project`: the per-project store, when `config.projectRoot` is set. It is
|
|
16
|
+
* writable, so a project can keep its own memories without polluting the
|
|
17
|
+
* user-level store.
|
|
18
|
+
* - `all`: primary plus the project store plus every `extraRoots` entry.
|
|
19
|
+
* - `global`: legacy alias for `primary`.
|
|
20
|
+
*
|
|
21
|
+
* `extraRoots` are always read-only additions.
|
|
22
|
+
*/
|
|
23
|
+
export type Scope = "primary" | "all" | "project" | "global";
|
|
24
|
+
|
|
25
|
+
/** Frontmatter fields understood by memoria. Unknown keys are preserved verbatim. */
|
|
26
|
+
export interface MemoryFrontmatter {
|
|
27
|
+
id?: string;
|
|
28
|
+
title?: string;
|
|
29
|
+
/** Broad grouping name; drives the file name and the note title. */
|
|
30
|
+
topic?: string;
|
|
31
|
+
category?: string;
|
|
32
|
+
tags?: string[] | string;
|
|
33
|
+
/**
|
|
34
|
+
* Other names this note answers to: nicknames, full names, former topic
|
|
35
|
+
* names, relations ("Bob's father"). Aliases are indexed, so any of them
|
|
36
|
+
* recalls the note, which is what keeps a rename cosmetic rather than
|
|
37
|
+
* load-bearing.
|
|
38
|
+
*/
|
|
39
|
+
aliases?: string[] | string;
|
|
40
|
+
created?: string;
|
|
41
|
+
updated?: string;
|
|
42
|
+
source?: string;
|
|
43
|
+
confidence?: Confidence | string;
|
|
44
|
+
priority?: Priority | string;
|
|
45
|
+
summary?: string;
|
|
46
|
+
/**
|
|
47
|
+
* Other notes this one is about (ids, relative paths or titles). Rendered in
|
|
48
|
+
* `memoria_read` and used to pull in related notes during recall.
|
|
49
|
+
*/
|
|
50
|
+
related?: string[] | string;
|
|
51
|
+
/**
|
|
52
|
+
* Notes this one replaces. A superseded note stays readable but is demoted in
|
|
53
|
+
* ranking and flagged in recall output, which is how contradictions get
|
|
54
|
+
* resolved without deleting history.
|
|
55
|
+
*/
|
|
56
|
+
supersedes?: string[] | string;
|
|
57
|
+
/** Epoch ms (or ISO string) of the last time this note was retrieved. */
|
|
58
|
+
last_used?: number | string;
|
|
59
|
+
/** Internal: set when the note was generated from a session. */
|
|
60
|
+
session?: string;
|
|
61
|
+
[key: string]: unknown;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** A memory note as parsed from disk. */
|
|
65
|
+
export interface MemoryDoc {
|
|
66
|
+
/** Stable id, `mem_<epochms>_<rand>`. */
|
|
67
|
+
id: string;
|
|
68
|
+
/** Absolute path of the owning store root. */
|
|
69
|
+
root: string;
|
|
70
|
+
/** Absolute file path. */
|
|
71
|
+
path: string;
|
|
72
|
+
/** Path relative to the store root, always forward slashes. e.g. `library/people/alice.md`. */
|
|
73
|
+
relPath: string;
|
|
74
|
+
title: string;
|
|
75
|
+
/** Broad grouping name (frontmatter `topic`, falling back to the title). */
|
|
76
|
+
topic: string;
|
|
77
|
+
/** First folder under `library/`, or `""` when the file lives at the library root. */
|
|
78
|
+
category: string;
|
|
79
|
+
tags: string[];
|
|
80
|
+
/** Alternative names for the same subject (see `MemoryFrontmatter.aliases`). */
|
|
81
|
+
aliases: string[];
|
|
82
|
+
/** Related notes (ids, relative paths or titles). */
|
|
83
|
+
related: string[];
|
|
84
|
+
/** Notes this one replaces (ids, relative paths or titles). */
|
|
85
|
+
supersedes: string[];
|
|
86
|
+
/** Epoch ms of the last retrieval, from frontmatter (`last_used`). */
|
|
87
|
+
lastUsed: number;
|
|
88
|
+
summary: string;
|
|
89
|
+
priority: Priority;
|
|
90
|
+
confidence: Confidence;
|
|
91
|
+
created: number;
|
|
92
|
+
updated: number;
|
|
93
|
+
/** Epoch ms from the filesystem. */
|
|
94
|
+
mtimeMs: number;
|
|
95
|
+
size: number;
|
|
96
|
+
wordCount: number;
|
|
97
|
+
/** Weighted term frequencies used for BM25. Not persisted verbatim. */
|
|
98
|
+
tokens: Record<string, number>;
|
|
99
|
+
/** Sum of all weights in `tokens`. */
|
|
100
|
+
tokenCount: number;
|
|
101
|
+
/** Raw markdown body (frontmatter stripped). Empty when lazily loaded. */
|
|
102
|
+
body: string;
|
|
103
|
+
/** Frontmatter as parsed, preserved for loss-less updates. */
|
|
104
|
+
frontmatter: MemoryFrontmatter;
|
|
105
|
+
/** True when the file lives in `inbox/` and has not been filed yet. */
|
|
106
|
+
unfiled: boolean;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Compact, JSON-persisted projection of a doc (no body, no tf map). */
|
|
110
|
+
export interface IndexedDocMeta {
|
|
111
|
+
id: string;
|
|
112
|
+
root: string;
|
|
113
|
+
path: string;
|
|
114
|
+
relPath: string;
|
|
115
|
+
title: string;
|
|
116
|
+
category: string;
|
|
117
|
+
tags: string[];
|
|
118
|
+
aliases: string[];
|
|
119
|
+
related: string[];
|
|
120
|
+
supersedes: string[];
|
|
121
|
+
summary: string;
|
|
122
|
+
priority: Priority;
|
|
123
|
+
confidence: Confidence;
|
|
124
|
+
created: number;
|
|
125
|
+
updated: number;
|
|
126
|
+
/** Epoch ms of the last retrieval (frontmatter `last_used`). */
|
|
127
|
+
lastUsed: number;
|
|
128
|
+
mtimeMs: number;
|
|
129
|
+
size: number;
|
|
130
|
+
wordCount: number;
|
|
131
|
+
/** Length of the markdown body, so checks can skip reading the file. */
|
|
132
|
+
bodyChars: number;
|
|
133
|
+
tokenCount: number;
|
|
134
|
+
unfiled: boolean;
|
|
135
|
+
/** First ~400 chars of the body, used for listings and index rendering. */
|
|
136
|
+
preview: string;
|
|
137
|
+
/** Cheap content fingerprint to detect edits that keep mtime/size identical. */
|
|
138
|
+
hash: string;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export interface PersistedIndex {
|
|
142
|
+
version: number;
|
|
143
|
+
root: string;
|
|
144
|
+
builtAt: number;
|
|
145
|
+
docs: IndexedDocMeta[];
|
|
146
|
+
/** Parallel to `terms`: flat [docIndex, tf, docIndex, tf, ...] per term. */
|
|
147
|
+
postings: number[][];
|
|
148
|
+
/** Term dictionary, indexed by term id (never reordered). */
|
|
149
|
+
terms: string[];
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** One analyzed query term with its resolved dictionary id and weight. */
|
|
153
|
+
export interface QueryTerm {
|
|
154
|
+
term: string;
|
|
155
|
+
/** Relative importance of this query term. */
|
|
156
|
+
weight: number;
|
|
157
|
+
/** Resolved term id, or -1 when absent from the dictionary. */
|
|
158
|
+
termId: number;
|
|
159
|
+
/** Inverse document frequency (or a high placeholder for missing terms). */
|
|
160
|
+
idf: number;
|
|
161
|
+
/** True for identifiers, paths, versions and error codes. */
|
|
162
|
+
specific: boolean;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export interface SearchOptions {
|
|
166
|
+
limit?: number;
|
|
167
|
+
/** Restrict to a category (folder under library/). */
|
|
168
|
+
category?: string;
|
|
169
|
+
/** All tags must be present. */
|
|
170
|
+
tags?: string[];
|
|
171
|
+
/** Any of these tags must be present. */
|
|
172
|
+
anyTags?: string[];
|
|
173
|
+
/** Minimum priority. */
|
|
174
|
+
minPriority?: Priority;
|
|
175
|
+
/** Only memories created/updated after this epoch ms. */
|
|
176
|
+
since?: number;
|
|
177
|
+
/** Restrict to a given root. */
|
|
178
|
+
root?: string;
|
|
179
|
+
scope?: Scope;
|
|
180
|
+
/** Include unfiled (inbox) memories. Default true. */
|
|
181
|
+
includeUnfiled?: boolean;
|
|
182
|
+
/** Score floor. */
|
|
183
|
+
minScore?: number;
|
|
184
|
+
/** Allow prefix matching. Default true. */
|
|
185
|
+
prefix?: boolean;
|
|
186
|
+
/** Allow fuzzy matching. Default true. */
|
|
187
|
+
fuzzy?: boolean;
|
|
188
|
+
/** Force a filesystem rescan before searching. */
|
|
189
|
+
refresh?: boolean;
|
|
190
|
+
/** Weighted query segments; the first is the primary prompt. */
|
|
191
|
+
parts?: QueryPart[];
|
|
192
|
+
/** Query-side synonym table (term -> alternatives). */
|
|
193
|
+
synonyms?: Record<string, string[]>;
|
|
194
|
+
/** Multiplier applied to synonym-expanded terms. Default 0.6. */
|
|
195
|
+
synonymWeight?: number;
|
|
196
|
+
/** Include a scoring breakdown per hit. */
|
|
197
|
+
explain?: boolean;
|
|
198
|
+
/** Also return notes linked through `related:` frontmatter. Default true. */
|
|
199
|
+
expandRelated?: boolean;
|
|
200
|
+
/** Maximum related notes pulled into one result. Default 3. */
|
|
201
|
+
relatedHits?: number;
|
|
202
|
+
/** Score factor for notes pulled in through `related`. Default 0.35. */
|
|
203
|
+
relatedBoost?: number;
|
|
204
|
+
/** Honour time expressions such as "last week" found in the query. Default true. */
|
|
205
|
+
timeHints?: boolean;
|
|
206
|
+
/** Drop notes that another live note declares in `supersedes`. Default false. */
|
|
207
|
+
dropSuperseded?: boolean;
|
|
208
|
+
/** Called once with the parsed time window, when a query mentions one. */
|
|
209
|
+
onTimeWindow?: (window: TimeWindow) => void;
|
|
210
|
+
/** Maximum characters of body excerpt per hit. */
|
|
211
|
+
snippetChars?: number;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** One weighted piece of a multi-part query (current prompt + previous turns). */
|
|
215
|
+
export interface QueryPart {
|
|
216
|
+
text: string;
|
|
217
|
+
weight: number;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** A time window parsed out of a natural-language query. */
|
|
221
|
+
export interface TimeWindow {
|
|
222
|
+
label: string;
|
|
223
|
+
since?: number;
|
|
224
|
+
until?: number;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
export interface SearchHit {
|
|
228
|
+
doc: IndexedDocMeta;
|
|
229
|
+
score: number;
|
|
230
|
+
/** Which query terms matched, best first. */
|
|
231
|
+
matched: string[];
|
|
232
|
+
/** Body excerpt around the best match. */
|
|
233
|
+
snippet: string;
|
|
234
|
+
/** True when the full query string appears verbatim in the body or title. */
|
|
235
|
+
exact: boolean;
|
|
236
|
+
/** Set when this hit was pulled in through another note's `related:`. */
|
|
237
|
+
relatedTo?: string;
|
|
238
|
+
/** Ids of live notes that declare this note superseded. */
|
|
239
|
+
supersededBy?: string[];
|
|
240
|
+
/** Present only with `explain: true`. */
|
|
241
|
+
breakdown?: ScoreBreakdown;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Per-component scoring detail for one hit (`explain: true`). */
|
|
245
|
+
export interface ScoreBreakdown {
|
|
246
|
+
bm25: number;
|
|
247
|
+
coverage: number;
|
|
248
|
+
priorityRecency: number;
|
|
249
|
+
phrase: number;
|
|
250
|
+
related: number;
|
|
251
|
+
supersededPenalty: number;
|
|
252
|
+
timeBoost: number;
|
|
253
|
+
total: number;
|
|
254
|
+
terms: Array<{ term: string; matches: number; contribution: number }>;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** Retrieval statistics for one note, persisted in `.index/usage.json`. */
|
|
258
|
+
export interface UsageEntry {
|
|
259
|
+
/** Times the note appeared in returned results. */
|
|
260
|
+
hits: number;
|
|
261
|
+
/** Times a fact was written to the note. */
|
|
262
|
+
writes: number;
|
|
263
|
+
/** Epoch ms of the last retrieval. */
|
|
264
|
+
lastUsed: number;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
export interface UsageFile {
|
|
268
|
+
version: number;
|
|
269
|
+
docs: Record<string, UsageEntry>;
|
|
270
|
+
/** Epoch ms of the last session start, for the "what changed" summary. */
|
|
271
|
+
lastSessionAt?: number;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/** Two notes whose contents look like the same subject. */
|
|
275
|
+
export interface MergeSuggestion {
|
|
276
|
+
a: { id: string; title: string; relPath: string; category: string };
|
|
277
|
+
b: { id: string; title: string; relPath: string; category: string };
|
|
278
|
+
/** Cosine similarity of the weighted term vectors, 0..1. */
|
|
279
|
+
score: number;
|
|
280
|
+
/** Distinctive terms the two notes share. */
|
|
281
|
+
shared: string[];
|
|
282
|
+
category: string;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** Two fact units that appear to disagree. */
|
|
286
|
+
export interface Contradiction {
|
|
287
|
+
a: { id: string; relPath: string; text: string };
|
|
288
|
+
b: { id: string; relPath: string; text: string };
|
|
289
|
+
shared: string[];
|
|
290
|
+
reason: string;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
export interface SearchResult {
|
|
294
|
+
hits: SearchHit[];
|
|
295
|
+
total: number;
|
|
296
|
+
/** Wall-clock duration in milliseconds. */
|
|
297
|
+
tookMs: number;
|
|
298
|
+
/** True when result collection was capped by the engine. */
|
|
299
|
+
truncated: boolean;
|
|
300
|
+
/** Terms that produced no match at all. */
|
|
301
|
+
missing: string[];
|
|
302
|
+
/** Time window parsed from the query, when one was recognized. */
|
|
303
|
+
timeWindow?: TimeWindow;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
export interface RecallBlock {
|
|
307
|
+
query: string;
|
|
308
|
+
hits: SearchHit[];
|
|
309
|
+
rendered: string;
|
|
310
|
+
tookMs: number;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
export interface StoreStats {
|
|
314
|
+
root: string;
|
|
315
|
+
docs: number;
|
|
316
|
+
categories: Record<string, number>;
|
|
317
|
+
totalBytes: number;
|
|
318
|
+
indexBytes: number;
|
|
319
|
+
indexBuiltAt: number;
|
|
320
|
+
indexLoadMs: number;
|
|
321
|
+
lastSearchMs: number;
|
|
322
|
+
watcherActive: boolean;
|
|
323
|
+
dirty: boolean;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
export interface HotFileState {
|
|
327
|
+
content: string;
|
|
328
|
+
chars: number;
|
|
329
|
+
limit: number;
|
|
330
|
+
over: boolean;
|
|
331
|
+
path: string;
|
|
332
|
+
mtimeMs: number;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
export interface MemoriaConfig {
|
|
336
|
+
version: number;
|
|
337
|
+
/**
|
|
338
|
+
* Primary store location. Supports `$AGENT_DIR` (pi's user config dir,
|
|
339
|
+
* `~/.pi/agent` by default), a leading `~`, absolute paths, and paths
|
|
340
|
+
* relative to the working directory.
|
|
341
|
+
*/
|
|
342
|
+
rootDir: string;
|
|
343
|
+
/** Additional read-only store roots (absolute, `~` or `$AGENT_DIR` prefixed). */
|
|
344
|
+
extraRoots: string[];
|
|
345
|
+
/** Hard cap for MEMORY.md, in characters. */
|
|
346
|
+
hotLimit: number;
|
|
347
|
+
/** Maximum characters injected per recalled memory snippet. */
|
|
348
|
+
snippetChars: number;
|
|
349
|
+
/** Auto-recall tuning. */
|
|
350
|
+
autoRecall: boolean;
|
|
351
|
+
autoRecallLimit: number;
|
|
352
|
+
autoRecallMinScore: number;
|
|
353
|
+
autoRecallMaxChars: number;
|
|
354
|
+
/** Number of previous user turns folded into the recall query. */
|
|
355
|
+
autoRecallLastTurns: number;
|
|
356
|
+
/** Maximum bytes of body text kept in the in-memory cache. */
|
|
357
|
+
bodyCacheBytes: number;
|
|
358
|
+
/** Milliseconds between filesystem freshness scans when no watcher is active. */
|
|
359
|
+
scanIntervalMs: number;
|
|
360
|
+
/** Extra glob-free path fragments to exclude from indexing. */
|
|
361
|
+
exclude: string[];
|
|
362
|
+
/** Default category folder for new memories without one. */
|
|
363
|
+
defaultCategory: string;
|
|
364
|
+
/** Maximum words in a topic name before falling back to the category. */
|
|
365
|
+
topicMaxWords: number;
|
|
366
|
+
/** A topic note larger than this spills into a numbered sibling file. */
|
|
367
|
+
topicMaxChars: number;
|
|
368
|
+
/**
|
|
369
|
+
* Optional per-project store (writable). Empty disables it. Supports
|
|
370
|
+
* `$AGENT_DIR`, a leading `~`, absolute and cwd-relative paths.
|
|
371
|
+
*/
|
|
372
|
+
projectRoot: string;
|
|
373
|
+
/** Weight of previous turns relative to the current prompt in auto-recall. */
|
|
374
|
+
autoRecallPriorWeight: number;
|
|
375
|
+
/** Track hits/writes/last-used per note in `.index/usage.json`. */
|
|
376
|
+
usageTracking: boolean;
|
|
377
|
+
/** Rewrite a note's `last_used` frontmatter at most this often. */
|
|
378
|
+
lastUsedWriteIntervalMs: number;
|
|
379
|
+
/** A note unused for this many days is reported as stale by doctor. */
|
|
380
|
+
staleAfterDays: number;
|
|
381
|
+
/** Suggest promoting a note to MEMORY.md after this many writes. */
|
|
382
|
+
promoteAfterWrites: number;
|
|
383
|
+
/** Cosine similarity above which two notes are reported as merge candidates. */
|
|
384
|
+
dedupeThreshold: number;
|
|
385
|
+
/** Maximum related notes pulled into a search result. */
|
|
386
|
+
relatedHits: number;
|
|
387
|
+
/** Score factor for notes pulled in through `related:`. */
|
|
388
|
+
relatedBoost: number;
|
|
389
|
+
/** Inline query-side synonyms, merged over `<root>/synonyms.json`. */
|
|
390
|
+
synonyms: Record<string, string[]>;
|
|
391
|
+
/** Multiplier applied to synonym-expanded query terms. */
|
|
392
|
+
synonymWeight: number;
|
|
393
|
+
/** Turn time expressions in queries ("last week") into a recency filter. */
|
|
394
|
+
timeHints: boolean;
|
|
395
|
+
/** LRU cache size for repeated identical searches (0 disables). */
|
|
396
|
+
searchCacheSize: number;
|
|
397
|
+
/** Rerank the top candidates with the session model before returning. */
|
|
398
|
+
rerank: boolean;
|
|
399
|
+
/** Model id for reranking; empty uses the session model. */
|
|
400
|
+
rerankModel: string;
|
|
401
|
+
/** How many candidates the reranker may see. */
|
|
402
|
+
rerankTopK: number;
|
|
403
|
+
/** Reranker budget in milliseconds; falls back to lexical order on timeout. */
|
|
404
|
+
rerankTimeoutMs: number;
|
|
405
|
+
/** Automatic session extraction: off, on-settle (idle) or on-shutdown. */
|
|
406
|
+
autoLearn: string;
|
|
407
|
+
/** Minimum user turns before auto-learn considers a session. */
|
|
408
|
+
autoLearnMinTurns: number;
|
|
409
|
+
/** Minimum transcript characters before auto-learn considers a session. */
|
|
410
|
+
autoLearnMinChars: number;
|
|
411
|
+
/** Minimum time between automatic extractions. */
|
|
412
|
+
autoLearnCooldownMs: number;
|
|
413
|
+
/** Characters per extraction chunk when harvesting a session. */
|
|
414
|
+
learnChunkChars: number;
|
|
415
|
+
/** Idle delay after a filesystem event before refreshing the index. */
|
|
416
|
+
watcherSettleMs: number;
|
|
417
|
+
/** Index persistence format: auto (binary above a threshold), json or binary. */
|
|
418
|
+
indexFormat: string;
|
|
419
|
+
/** Search pi's saved session transcripts (past conversations). */
|
|
420
|
+
sessionSearch: boolean;
|
|
421
|
+
/** Extra session roots, in addition to `<agent dir>/sessions[-archive]`. */
|
|
422
|
+
sessionRoots: string[];
|
|
423
|
+
/** Search sessions automatically when the library returns nothing. */
|
|
424
|
+
sessionFallback: boolean;
|
|
425
|
+
/** Budget for an explicit session search, in milliseconds. */
|
|
426
|
+
sessionScanMs: number;
|
|
427
|
+
/** Characters of context per session hit. */
|
|
428
|
+
sessionExcerptChars: number;
|
|
429
|
+
/** In-memory budget for parsed session messages. */
|
|
430
|
+
sessionCacheBytes: number;
|
|
431
|
+
/** Include tool calls, tool results and compaction summaries by default. */
|
|
432
|
+
sessionIncludeTools: boolean;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/* ------------------------------------------------------------------ */
|
|
436
|
+
/* Session transcripts (past conversations) */
|
|
437
|
+
/* ------------------------------------------------------------------ */
|
|
438
|
+
|
|
439
|
+
/** One searchable message extracted from a saved session transcript. */
|
|
440
|
+
export interface SessionMessage {
|
|
441
|
+
/** 1-based line in the transcript file. */
|
|
442
|
+
line: number;
|
|
443
|
+
/** Who produced the text. `summary` is a compaction summary, `tool` derived output. */
|
|
444
|
+
role: "user" | "assistant" | "tool" | "summary";
|
|
445
|
+
/** Epoch ms of the entry. */
|
|
446
|
+
timestamp: number;
|
|
447
|
+
/** Extracted text, capped per message. */
|
|
448
|
+
text: string;
|
|
449
|
+
/** Normalized text used for phrase matching. */
|
|
450
|
+
normalized: string;
|
|
451
|
+
/** Tokenized terms, built lazily only for candidate messages. */
|
|
452
|
+
terms?: Set<string>;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/** A transcript file on disk. */
|
|
456
|
+
export interface SessionFile {
|
|
457
|
+
path: string;
|
|
458
|
+
root: string;
|
|
459
|
+
relPath: string;
|
|
460
|
+
/** Project the session ran in: the recorded cwd, or a decode of the folder name. */
|
|
461
|
+
project: string;
|
|
462
|
+
/** Short display name for the project (basename of the cwd). */
|
|
463
|
+
projectName: string;
|
|
464
|
+
/** Session start, from the file name or the header. */
|
|
465
|
+
startedAt: number;
|
|
466
|
+
mtimeMs: number;
|
|
467
|
+
size: number;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
export interface SessionSearchOptions {
|
|
471
|
+
limit?: number;
|
|
472
|
+
/** Only sessions started within the last N days. */
|
|
473
|
+
sinceDays?: number;
|
|
474
|
+
/** Substring match on the project path or folder name. */
|
|
475
|
+
project?: string;
|
|
476
|
+
/** Include tool calls, tool results, custom payloads and compaction summaries. */
|
|
477
|
+
includeTools?: boolean;
|
|
478
|
+
/** Only user messages. */
|
|
479
|
+
userOnly?: boolean;
|
|
480
|
+
/** Wall-clock budget; stops scanning (newest first) and reports `partial`. */
|
|
481
|
+
budgetMs?: number;
|
|
482
|
+
/** Excerpt size override. */
|
|
483
|
+
excerptChars?: number;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
export interface SessionHit {
|
|
487
|
+
path: string;
|
|
488
|
+
relPath: string;
|
|
489
|
+
project: string;
|
|
490
|
+
projectName: string;
|
|
491
|
+
line: number;
|
|
492
|
+
role: SessionMessage["role"];
|
|
493
|
+
timestamp: number;
|
|
494
|
+
score: number;
|
|
495
|
+
matched: string[];
|
|
496
|
+
exact: boolean;
|
|
497
|
+
excerpt: string;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
export interface SessionScanStats {
|
|
501
|
+
roots: string[];
|
|
502
|
+
files: number;
|
|
503
|
+
messages: number;
|
|
504
|
+
bytes: number;
|
|
505
|
+
/** Files served from the parsed-message cache. */
|
|
506
|
+
cachedFiles: number;
|
|
507
|
+
/** Unreadable files or lines that could not be parsed. */
|
|
508
|
+
skipped: number;
|
|
509
|
+
/** True when the budget stopped the scan before every file was read. */
|
|
510
|
+
partial: boolean;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
export interface SessionSearchResult {
|
|
514
|
+
hits: SessionHit[];
|
|
515
|
+
stats: SessionScanStats;
|
|
516
|
+
tookMs: number;
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
export interface SessionWindowMessage {
|
|
520
|
+
line: number;
|
|
521
|
+
role: string;
|
|
522
|
+
timestamp: number;
|
|
523
|
+
text: string;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
export interface SessionReadResult {
|
|
527
|
+
path: string;
|
|
528
|
+
relPath: string;
|
|
529
|
+
projectName: string;
|
|
530
|
+
startLine: number;
|
|
531
|
+
endLine: number;
|
|
532
|
+
messages: SessionWindowMessage[];
|
|
533
|
+
}
|
package/src/usage.ts
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retrieval and write statistics, kept in `.index/usage.json`.
|
|
3
|
+
*
|
|
4
|
+
* This is derived data: deleting the file loses promotion and staleness hints
|
|
5
|
+
* and nothing else. The search path must never do disk I/O, so hits are counted
|
|
6
|
+
* in memory and flushed on a debounce; the runtime also refreshes a note's
|
|
7
|
+
* `last_used` frontmatter at most once per `lastUsedWriteIntervalMs`.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
import { INDEX_DIR, USAGE_FILE } from "./config.ts";
|
|
12
|
+
import { atomicWriteFile, readFileOrUndefined } from "./util.ts";
|
|
13
|
+
import type { UsageEntry, UsageFile } from "./types.ts";
|
|
14
|
+
|
|
15
|
+
export const USAGE_VERSION = 1;
|
|
16
|
+
|
|
17
|
+
/** Upper bound on tracked notes; the least recently used are dropped first. */
|
|
18
|
+
export const MAX_USAGE_ENTRIES = 50_000;
|
|
19
|
+
|
|
20
|
+
function emptyEntry(): UsageEntry {
|
|
21
|
+
return { hits: 0, writes: 0, lastUsed: 0 };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function coerceEntry(raw: unknown): UsageEntry | undefined {
|
|
25
|
+
if (!raw || typeof raw !== "object") return undefined;
|
|
26
|
+
const input = raw as Record<string, unknown>;
|
|
27
|
+
const number = (value: unknown): number => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0);
|
|
28
|
+
const entry = { hits: number(input.hits), writes: number(input.writes), lastUsed: number(input.lastUsed) };
|
|
29
|
+
if (entry.hits === 0 && entry.writes === 0 && entry.lastUsed === 0) return undefined;
|
|
30
|
+
return entry;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** In-memory usage table with an explicit, debounced flush. */
|
|
34
|
+
export class UsageStore {
|
|
35
|
+
private docs = new Map<string, UsageEntry>();
|
|
36
|
+
private changed = false;
|
|
37
|
+
private lastSessionAt = 0;
|
|
38
|
+
|
|
39
|
+
constructor(root: string) {
|
|
40
|
+
this.root = root;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
private readonly root: string;
|
|
44
|
+
|
|
45
|
+
/** Load `<root>/.index/usage.json`; a missing or corrupt file starts empty. */
|
|
46
|
+
static async load(root: string): Promise<UsageStore> {
|
|
47
|
+
const store = new UsageStore(root);
|
|
48
|
+
const raw = await readFileOrUndefined(join(root, INDEX_DIR, USAGE_FILE));
|
|
49
|
+
if (!raw) return store;
|
|
50
|
+
try {
|
|
51
|
+
const parsed = JSON.parse(raw) as UsageFile;
|
|
52
|
+
if (!parsed || typeof parsed !== "object" || !parsed.docs || typeof parsed.docs !== "object") return store;
|
|
53
|
+
if (typeof parsed.lastSessionAt === "number" && Number.isFinite(parsed.lastSessionAt)) store.lastSessionAt = parsed.lastSessionAt;
|
|
54
|
+
for (const [id, value] of Object.entries(parsed.docs)) {
|
|
55
|
+
const entry = coerceEntry(value);
|
|
56
|
+
if (entry) store.docs.set(id, entry);
|
|
57
|
+
}
|
|
58
|
+
} catch {
|
|
59
|
+
// Derived data: a corrupt file is simply ignored.
|
|
60
|
+
}
|
|
61
|
+
return store;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
get(id: string): UsageEntry | undefined {
|
|
65
|
+
return this.docs.get(id);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Count one retrieval (`hit`) or one fact write (`write`). */
|
|
69
|
+
record(id: string, kind: "hit" | "write", at = Date.now()): void {
|
|
70
|
+
if (!id) return;
|
|
71
|
+
let entry = this.docs.get(id);
|
|
72
|
+
if (!entry) {
|
|
73
|
+
entry = emptyEntry();
|
|
74
|
+
this.docs.set(id, entry);
|
|
75
|
+
}
|
|
76
|
+
if (kind === "hit") entry.hits += 1;
|
|
77
|
+
else entry.writes += 1;
|
|
78
|
+
if (at > entry.lastUsed) entry.lastUsed = at;
|
|
79
|
+
this.changed = true;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Note that a note was touched without counting a read or a write. */
|
|
83
|
+
markUsed(id: string, at: number): void {
|
|
84
|
+
const entry = this.docs.get(id);
|
|
85
|
+
if (entry) {
|
|
86
|
+
if (at > entry.lastUsed) {
|
|
87
|
+
entry.lastUsed = at;
|
|
88
|
+
this.changed = true;
|
|
89
|
+
}
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
this.docs.set(id, { hits: 0, writes: 0, lastUsed: at });
|
|
93
|
+
this.changed = true;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
entries(): Array<[string, UsageEntry]> {
|
|
97
|
+
return [...this.docs.entries()];
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
get size(): number {
|
|
101
|
+
return this.docs.size;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
get dirty(): boolean {
|
|
105
|
+
return this.changed;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Drop entries whose note no longer exists, then cap the table size. */
|
|
109
|
+
prune(validIds: Set<string>): number {
|
|
110
|
+
let removed = 0;
|
|
111
|
+
for (const id of [...this.docs.keys()]) {
|
|
112
|
+
if (!validIds.has(id)) {
|
|
113
|
+
this.docs.delete(id);
|
|
114
|
+
removed += 1;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
if (this.docs.size > MAX_USAGE_ENTRIES) {
|
|
118
|
+
const byAge = [...this.docs.entries()].sort((a, b) => a[1].lastUsed - b[1].lastUsed);
|
|
119
|
+
for (const [id] of byAge.slice(0, this.docs.size - MAX_USAGE_ENTRIES)) {
|
|
120
|
+
this.docs.delete(id);
|
|
121
|
+
removed += 1;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (removed > 0) this.changed = true;
|
|
125
|
+
return removed;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Epoch ms of the previous session start (0 when unknown). */
|
|
129
|
+
sessionStartedAt(): number {
|
|
130
|
+
return this.lastSessionAt;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Record this session's start so the next one can summarise what changed. */
|
|
134
|
+
markSessionStart(at = Date.now()): void {
|
|
135
|
+
this.lastSessionAt = at;
|
|
136
|
+
this.changed = true;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async save(): Promise<void> {
|
|
140
|
+
if (!this.changed) return;
|
|
141
|
+
this.changed = false;
|
|
142
|
+
const payload: UsageFile = { version: USAGE_VERSION, docs: Object.fromEntries(this.docs), lastSessionAt: this.lastSessionAt };
|
|
143
|
+
await atomicWriteFile(join(this.root, INDEX_DIR, USAGE_FILE), JSON.stringify(payload));
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** Notes with no recorded use in `staleMs`, oldest first. */
|
|
147
|
+
stale(staleMs: number, now = Date.now(), minWrites = 1): Array<{ id: string; entry: UsageEntry; idleMs: number }> {
|
|
148
|
+
const out: Array<{ id: string; entry: UsageEntry; idleMs: number }> = [];
|
|
149
|
+
for (const [id, entry] of this.docs) {
|
|
150
|
+
if (entry.writes < minWrites && entry.hits === 0) continue;
|
|
151
|
+
const idleMs = now - (entry.lastUsed || 0);
|
|
152
|
+
if (idleMs >= staleMs) out.push({ id, entry, idleMs });
|
|
153
|
+
}
|
|
154
|
+
return out.sort((a, b) => b.idleMs - a.idleMs);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Notes written often enough to be worth promoting into MEMORY.md. */
|
|
158
|
+
promotionCandidates(minWrites: number): Array<{ id: string; entry: UsageEntry }> {
|
|
159
|
+
const out: Array<{ id: string; entry: UsageEntry }> = [];
|
|
160
|
+
for (const [id, entry] of this.docs) if (entry.writes >= minWrites) out.push({ id, entry });
|
|
161
|
+
return out.sort((a, b) => b.entry.writes - a.entry.writes);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Human-readable idle duration, e.g. "3 months" or "12 days". */
|
|
166
|
+
export function describeAge(ms: number): string {
|
|
167
|
+
const days = Math.floor(ms / 86_400_000);
|
|
168
|
+
if (days < 1) return "today";
|
|
169
|
+
if (days < 45) return `${days} day${days === 1 ? "" : "s"}`;
|
|
170
|
+
const months = Math.round(days / 30);
|
|
171
|
+
if (months < 18) return `${months} months`;
|
|
172
|
+
const years = Math.max(1, Math.round(days / 365));
|
|
173
|
+
return `${years} year${years === 1 ? "" : "s"}`;
|
|
174
|
+
}
|