pi-memoria 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +53 -0
- package/README.md +126 -166
- package/index.ts +299 -204
- package/package.json +28 -69
- package/src/aliases.ts +104 -0
- package/src/config.ts +22 -184
- package/src/database.ts +301 -0
- package/src/errors.ts +55 -0
- package/src/files.ts +29 -0
- package/src/hot-memory.ts +231 -0
- package/src/results.ts +264 -0
- package/src/sessions.ts +246 -843
- package/src/text.ts +48 -0
- package/skills/memoria/SKILL.md +0 -77
- package/skills/memoria-sessions/SKILL.md +0 -69
- package/src/commands.ts +0 -398
- package/src/frontmatter.ts +0 -318
- package/src/index-engine.ts +0 -1071
- package/src/recall.ts +0 -211
- package/src/rerank.ts +0 -85
- package/src/runtime.ts +0 -1469
- package/src/search.ts +0 -499
- package/src/similarity.ts +0 -374
- package/src/store.ts +0 -1724
- package/src/synonyms.ts +0 -105
- package/src/timeexpr.ts +0 -145
- package/src/tokenize.ts +0 -295
- package/src/tools.ts +0 -813
- package/src/types.ts +0 -541
- package/src/usage.ts +0 -188
- package/src/util.ts +0 -141
package/src/database.ts
ADDED
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { chmodSync, mkdirSync } from "node:fs";
|
|
3
|
+
import { dirname, resolve } from "node:path";
|
|
4
|
+
import { DatabaseSync, type StatementSync } from "node:sqlite";
|
|
5
|
+
import { expandQuery, loadAliases, type AliasConfiguration } from "./aliases.ts";
|
|
6
|
+
import { ExpectedError, isSqliteError, storeUnavailable } from "./errors.ts";
|
|
7
|
+
import { chunk, excerpt, integer, MEMORY_LIMIT, nonempty, tags as validateTags } from "./text.ts";
|
|
8
|
+
|
|
9
|
+
export interface Memory {
|
|
10
|
+
id: string;
|
|
11
|
+
content: string;
|
|
12
|
+
tags: string[];
|
|
13
|
+
source: string;
|
|
14
|
+
created_at: string;
|
|
15
|
+
updated_at: string;
|
|
16
|
+
revision: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface MemoryInput { content: string; tags?: string[]; source?: string }
|
|
20
|
+
export type SearchMode = "any" | "all" | "phrase" | "literal";
|
|
21
|
+
export interface MemorySearchOptions {
|
|
22
|
+
/** Expand configured terminology alternatives for any/all keyword modes. Defaults to true. */
|
|
23
|
+
expandAliases?: boolean;
|
|
24
|
+
/** Require exact, case-sensitive membership for every supplied tag. */
|
|
25
|
+
tags?: string[];
|
|
26
|
+
}
|
|
27
|
+
interface Row { [key: string]: unknown }
|
|
28
|
+
|
|
29
|
+
function validatedInput(input: MemoryInput, preserveWhitespace = false): Required<MemoryInput> {
|
|
30
|
+
const content = preserveWhitespace ? input.content : nonempty(input.content, "content");
|
|
31
|
+
if (!content.length || content.length > MEMORY_LIMIT || content.includes("\0")) {
|
|
32
|
+
throw new ExpectedError("invalid_input", `content must contain 1–${MEMORY_LIMIT} characters and no NUL bytes.`, { field: "content" });
|
|
33
|
+
}
|
|
34
|
+
return {
|
|
35
|
+
content, tags: validateTags(input.tags),
|
|
36
|
+
source: input.source ? nonempty(input.source, "source", 2_000) : "",
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function memory(row: Row): Memory {
|
|
41
|
+
return {
|
|
42
|
+
id: String(row.id), content: String(row.content), tags: JSON.parse(String(row.tags)) as string[],
|
|
43
|
+
source: String(row.source), created_at: String(row.created_at), updated_at: String(row.updated_at),
|
|
44
|
+
revision: Number(row.revision),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export class MemoryDatabase {
|
|
49
|
+
private db: DatabaseSync;
|
|
50
|
+
private statements = new Map<string, StatementSync>();
|
|
51
|
+
private aliases?: AliasConfiguration;
|
|
52
|
+
private closed = false;
|
|
53
|
+
|
|
54
|
+
constructor(readonly path: string, private readonly aliasesPath?: string) {
|
|
55
|
+
try { mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); }
|
|
56
|
+
catch (error) { throw storeUnavailable("open", path, error); }
|
|
57
|
+
try { this.db = new DatabaseSync(path); }
|
|
58
|
+
catch (error) { throw storeUnavailable("open", path, error); }
|
|
59
|
+
try {
|
|
60
|
+
chmodSync(path, 0o600);
|
|
61
|
+
this.db.exec("PRAGMA busy_timeout=5000; PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;");
|
|
62
|
+
this.db.function("memoria_fold", { deterministic: true }, (value) => String(value).toLowerCase());
|
|
63
|
+
this.transaction(() => {
|
|
64
|
+
const version = Number(this.db.prepare("PRAGMA user_version").get()?.user_version);
|
|
65
|
+
if (version > 2) throw new Error(`Database schema ${version} is newer than this pi-memoria version.`);
|
|
66
|
+
if (version === 0) {
|
|
67
|
+
this.db.exec(`
|
|
68
|
+
CREATE TABLE memories (
|
|
69
|
+
rowid INTEGER PRIMARY KEY,
|
|
70
|
+
id TEXT NOT NULL UNIQUE,
|
|
71
|
+
content TEXT NOT NULL,
|
|
72
|
+
tags TEXT NOT NULL DEFAULT '[]',
|
|
73
|
+
tag_text TEXT NOT NULL DEFAULT '',
|
|
74
|
+
source TEXT NOT NULL DEFAULT '',
|
|
75
|
+
created_at TEXT NOT NULL,
|
|
76
|
+
updated_at TEXT NOT NULL,
|
|
77
|
+
revision INTEGER NOT NULL DEFAULT 1
|
|
78
|
+
) STRICT;
|
|
79
|
+
CREATE VIRTUAL TABLE memories_fts USING fts5(
|
|
80
|
+
content, tag_text, source, content='memories', content_rowid='rowid',
|
|
81
|
+
tokenize='porter unicode61 remove_diacritics 2', prefix='2 3'
|
|
82
|
+
);
|
|
83
|
+
CREATE TRIGGER memories_insert AFTER INSERT ON memories BEGIN
|
|
84
|
+
INSERT INTO memories_fts(rowid, content, tag_text, source) VALUES(new.rowid, new.content, new.tag_text, new.source);
|
|
85
|
+
END;
|
|
86
|
+
CREATE TRIGGER memories_delete AFTER DELETE ON memories BEGIN
|
|
87
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, tag_text, source)
|
|
88
|
+
VALUES('delete', old.rowid, old.content, old.tag_text, old.source);
|
|
89
|
+
END;
|
|
90
|
+
CREATE TRIGGER memories_update AFTER UPDATE ON memories BEGIN
|
|
91
|
+
INSERT INTO memories_fts(memories_fts, rowid, content, tag_text, source)
|
|
92
|
+
VALUES('delete', old.rowid, old.content, old.tag_text, old.source);
|
|
93
|
+
INSERT INTO memories_fts(rowid, content, tag_text, source) VALUES(new.rowid, new.content, new.tag_text, new.source);
|
|
94
|
+
END;
|
|
95
|
+
CREATE TABLE session_roots (path TEXT PRIMARY KEY) STRICT;
|
|
96
|
+
PRAGMA user_version=1;
|
|
97
|
+
`);
|
|
98
|
+
}
|
|
99
|
+
if (version < 2) {
|
|
100
|
+
// Keep existing duplicates and archive evidence intact; serialize new adds below.
|
|
101
|
+
this.db.exec("CREATE INDEX memories_content ON memories(content); PRAGMA user_version=2;");
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
} catch (error) {
|
|
105
|
+
this.closed = true;
|
|
106
|
+
try { this.db.close(); } catch { /* keep the original failure */ }
|
|
107
|
+
throw storeUnavailable("open", path, error);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
private attempt<T>(operation: string, work: () => T): T {
|
|
112
|
+
try { return work(); }
|
|
113
|
+
catch (error) {
|
|
114
|
+
if (error instanceof ExpectedError) throw error;
|
|
115
|
+
if (isSqliteError(error)) throw storeUnavailable(operation, this.path, error);
|
|
116
|
+
throw error;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
private statement(sql: string): StatementSync {
|
|
121
|
+
if (this.closed) throw storeUnavailable("access", this.path, new Error("database is closed"));
|
|
122
|
+
let result = this.statements.get(sql);
|
|
123
|
+
if (!result) { result = this.db.prepare(sql); this.statements.set(sql, result); }
|
|
124
|
+
return result;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
private transaction<T>(operation: () => T): T {
|
|
128
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
129
|
+
try {
|
|
130
|
+
const result = operation();
|
|
131
|
+
this.db.exec("COMMIT");
|
|
132
|
+
return result;
|
|
133
|
+
} catch (error) { this.db.exec("ROLLBACK"); throw error; }
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
add(input: MemoryInput): { memory: Memory; created: boolean } {
|
|
137
|
+
const value = validatedInput(input);
|
|
138
|
+
// BEGIN IMMEDIATE covers both lookup and insertion, including competing Pi processes.
|
|
139
|
+
return this.attempt("add", () => this.transaction(() => {
|
|
140
|
+
const existing = this.statement("SELECT * FROM memories WHERE content=? ORDER BY rowid LIMIT 1").get(value.content);
|
|
141
|
+
if (existing) return { memory: memory(existing), created: false };
|
|
142
|
+
const id = `m_${randomUUID()}`;
|
|
143
|
+
this.insert(id, value);
|
|
144
|
+
return { memory: this.get(id), created: true };
|
|
145
|
+
}));
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
private insert(id: string, { content, tags, source }: Required<MemoryInput>, ignore = false): void {
|
|
149
|
+
const timestamp = new Date().toISOString();
|
|
150
|
+
this.statement(`INSERT ${ignore ? "OR IGNORE" : ""} INTO memories(id,content,tags,tag_text,source,created_at,updated_at) VALUES(?,?,?,?,?,?,?)`)
|
|
151
|
+
.run(id, content, JSON.stringify(tags), tags.join("\n"), source, timestamp, timestamp);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Save first, then let the caller replace MEMORY.md. Retrying is idempotent. */
|
|
155
|
+
archive(entries: { id: string; content: string }[], sourcePath: string): string[] {
|
|
156
|
+
return this.attempt("archive", () => this.transaction(() => entries.map((entry) => {
|
|
157
|
+
const id = `m_archive_${createHash("sha256").update(entry.id + "\0" + entry.content).digest("hex").slice(0, 24)}`;
|
|
158
|
+
// Oversized hand-written bullets are preserved in complete, bounded pieces.
|
|
159
|
+
const parts: string[] = [];
|
|
160
|
+
for (let offset = 0; offset < entry.content.length;) {
|
|
161
|
+
const part = chunk(entry.content, offset, 19_000);
|
|
162
|
+
parts.push(part.text);
|
|
163
|
+
offset = part.next_text_offset ?? entry.content.length;
|
|
164
|
+
}
|
|
165
|
+
parts.forEach((content, index) => this.insert(parts.length === 1 ? id : `${id}_${index + 1}`, validatedInput({
|
|
166
|
+
content, tags: ["hot-archive"], source: `${sourcePath}#${entry.id}${parts.length === 1 ? "" : ` (part ${index + 1}/${parts.length})`}`,
|
|
167
|
+
}, true), true));
|
|
168
|
+
return id;
|
|
169
|
+
})));
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
get(id: string): Memory {
|
|
173
|
+
return this.attempt("read", () => {
|
|
174
|
+
const row = this.statement("SELECT * FROM memories WHERE id=?").get(id);
|
|
175
|
+
if (!row) throw new ExpectedError("not_found", `Memory ${id} does not exist. Search before editing or deleting.`, { id });
|
|
176
|
+
return memory(row);
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
edit(id: string, changes: Partial<MemoryInput>, expectedRevision?: number): Memory {
|
|
181
|
+
return this.attempt("edit", () => this.transaction(() => {
|
|
182
|
+
const previous = this.get(id);
|
|
183
|
+
this.checkRevision(previous, expectedRevision);
|
|
184
|
+
const content = changes.content === undefined ? previous.content : nonempty(changes.content, "content");
|
|
185
|
+
const tags = changes.tags === undefined ? previous.tags : validateTags(changes.tags);
|
|
186
|
+
const source = changes.source === undefined ? previous.source : changes.source ? nonempty(changes.source, "source", 2_000) : "";
|
|
187
|
+
this.statement("UPDATE memories SET content=?,tags=?,tag_text=?,source=?,updated_at=?,revision=revision+1 WHERE id=?")
|
|
188
|
+
.run(content, JSON.stringify(tags), tags.join("\n"), source, new Date().toISOString(), id);
|
|
189
|
+
return this.get(id);
|
|
190
|
+
}));
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
delete(id: string, expectedRevision?: number): Memory {
|
|
194
|
+
return this.attempt("delete", () => this.transaction(() => {
|
|
195
|
+
const previous = this.get(id);
|
|
196
|
+
this.checkRevision(previous, expectedRevision);
|
|
197
|
+
this.statement("DELETE FROM memories WHERE id=?").run(id);
|
|
198
|
+
return previous;
|
|
199
|
+
}));
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
private checkRevision(value: Memory, expected?: number): void {
|
|
203
|
+
if (expected !== undefined && value.revision !== expected) {
|
|
204
|
+
throw new ExpectedError("conflict", `Memory ${value.id} changed (revision ${value.revision}, expected ${expected}). Search again before changing it.`, {
|
|
205
|
+
id: value.id, current_revision: value.revision, expected_revision: expected,
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
search(query = "", mode: SearchMode = "any", limit = 20, offset = 0, options: MemorySearchOptions = {}) {
|
|
211
|
+
integer(limit, "limit", 1, 50);
|
|
212
|
+
integer(offset, "offset", 0, Number.MAX_SAFE_INTEGER);
|
|
213
|
+
if (query.length > 2_000 || query.includes("\0")) {
|
|
214
|
+
throw new ExpectedError("invalid_input", "Query must be at most 2,000 characters without NUL bytes.", { field: "query" });
|
|
215
|
+
}
|
|
216
|
+
if (!["any", "all", "phrase", "literal"].includes(mode)) {
|
|
217
|
+
throw new ExpectedError("invalid_input", "Unknown memory search mode.", { field: "mode" });
|
|
218
|
+
}
|
|
219
|
+
const requestedTags = validateTags(options.tags);
|
|
220
|
+
const start = performance.now();
|
|
221
|
+
const trimmed = query.trim();
|
|
222
|
+
const terms = [...new Set(trimmed.match(/[\p{L}\p{N}\p{M}_]+/gu) ?? [])];
|
|
223
|
+
const keyword = mode === "any" || mode === "all";
|
|
224
|
+
const aliasExpansionEnabled = keyword && terms.length > 0 && (options.expandAliases ?? true);
|
|
225
|
+
const alias = aliasExpansionEnabled ? this.aliasConfiguration() : undefined;
|
|
226
|
+
const expansion = alias ? expandQuery(terms, alias.aliases) : { expansions: [] as Array<{ term: string; aliases: string[] }> };
|
|
227
|
+
const warnings = alias ? [alias.warning, expansion.warning].filter((value): value is string => value !== undefined) : [];
|
|
228
|
+
const tagClauses = requestedTags.map(() => "EXISTS (SELECT 1 FROM json_each(m.tags) AS requested_tag WHERE requested_tag.value = ? COLLATE BINARY)");
|
|
229
|
+
const tagWhere = tagClauses.join(" AND ");
|
|
230
|
+
|
|
231
|
+
let usedMode: SearchMode | "browse" = mode;
|
|
232
|
+
let rows: Row[];
|
|
233
|
+
const literal = () => this.statement(`SELECT m.* FROM memories m
|
|
234
|
+
WHERE instr(memoria_fold(m.content || char(10) || m.tag_text || char(10) || m.source), memoria_fold(?)) > 0${tagWhere ? ` AND ${tagWhere}` : ""}
|
|
235
|
+
ORDER BY m.created_at, m.id LIMIT ? OFFSET ?`).all(trimmed, ...requestedTags, limit + 1, offset);
|
|
236
|
+
if (!trimmed) {
|
|
237
|
+
usedMode = "browse";
|
|
238
|
+
rows = this.statement(`SELECT m.* FROM memories m${tagWhere ? ` WHERE ${tagWhere}` : ""} ORDER BY m.created_at,m.id LIMIT ? OFFSET ?`)
|
|
239
|
+
.all(...requestedTags, limit + 1, offset);
|
|
240
|
+
} else if (mode === "literal" || !terms.length) {
|
|
241
|
+
usedMode = "literal";
|
|
242
|
+
rows = literal();
|
|
243
|
+
} else {
|
|
244
|
+
const fts = mode === "phrase"
|
|
245
|
+
? `"${terms.join(" ")}"`
|
|
246
|
+
: terms.map((term) => {
|
|
247
|
+
const alternatives = expansion.expansions.find((value) => value.term === term)?.aliases ?? [];
|
|
248
|
+
const tokens = [term, ...alternatives].map((token) => `"${token}"*`);
|
|
249
|
+
return tokens.length > 1 ? `(${tokens.join(" OR ")})` : tokens[0]!;
|
|
250
|
+
}).join(mode === "all" ? " AND " : " OR ");
|
|
251
|
+
rows = this.statement(`SELECT m.* FROM memories_fts JOIN memories m ON m.rowid=memories_fts.rowid
|
|
252
|
+
WHERE memories_fts MATCH ?${tagWhere ? ` AND ${tagWhere}` : ""} ORDER BY bm25(memories_fts),m.id LIMIT ? OFFSET ?`)
|
|
253
|
+
.all(fts, ...requestedTags, limit + 1, offset);
|
|
254
|
+
// Decide fallback independently of page offset so subsequent pages retain the same semantics.
|
|
255
|
+
// The existence probe uses the same tag predicate so an all-filtered corpus can still fall back.
|
|
256
|
+
if (!this.statement(`SELECT m.rowid FROM memories_fts JOIN memories m ON m.rowid=memories_fts.rowid
|
|
257
|
+
WHERE memories_fts MATCH ?${tagWhere ? ` AND ${tagWhere}` : ""} LIMIT 1`).get(fts, ...requestedTags)) {
|
|
258
|
+
usedMode = "literal";
|
|
259
|
+
rows = literal();
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
const hasMore = rows.length > limit;
|
|
263
|
+
return {
|
|
264
|
+
mode: usedMode,
|
|
265
|
+
query: trimmed,
|
|
266
|
+
tags: requestedTags,
|
|
267
|
+
alias_expansion_enabled: aliasExpansionEnabled,
|
|
268
|
+
expansions: expansion.expansions,
|
|
269
|
+
warnings,
|
|
270
|
+
results: rows.slice(0, limit).map((row) => {
|
|
271
|
+
const item = memory(row);
|
|
272
|
+
return { ...item, content: excerpt(item.content, terms[0] ?? trimmed), content_characters: item.content.length, content_clipped: item.content.length > 600 };
|
|
273
|
+
}),
|
|
274
|
+
next_offset: hasMore ? offset + limit : null,
|
|
275
|
+
elapsed_ms: Number((performance.now() - start).toFixed(3)),
|
|
276
|
+
hint: "Use id to read a complete memory. Try mode=literal for exact substrings, shorter/alternate terms, or query='' to browse everything. For historical claims or uncertain/empty recall, use memoria_sessions; an empty keyword result does not prove absence.",
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
private aliasConfiguration(): AliasConfiguration {
|
|
281
|
+
this.aliases ??= loadAliases(this.aliasesPath);
|
|
282
|
+
return this.aliases;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
rememberRoots(paths: string[]): void {
|
|
286
|
+
this.attempt("remember_roots", () => this.transaction(() => {
|
|
287
|
+
for (const path of paths) this.statement("INSERT OR IGNORE INTO session_roots(path) VALUES(?)").run(resolve(path));
|
|
288
|
+
}));
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
roots(): string[] {
|
|
292
|
+
return this.attempt("list_roots", () => this.statement("SELECT path FROM session_roots ORDER BY path").all().map((row) => String(row.path)));
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
close(): void {
|
|
296
|
+
if (this.closed) return;
|
|
297
|
+
this.closed = true;
|
|
298
|
+
this.statements.clear();
|
|
299
|
+
this.db.close();
|
|
300
|
+
}
|
|
301
|
+
}
|
package/src/errors.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/** Stable codes for expected, recoverable failures. Unexpected errors pass through untouched. */
|
|
2
|
+
export type ErrorCode =
|
|
3
|
+
| "invalid_input"
|
|
4
|
+
| "not_found"
|
|
5
|
+
| "conflict"
|
|
6
|
+
| "duplicate_hot_content"
|
|
7
|
+
| "hot_capacity"
|
|
8
|
+
| "store_unavailable"
|
|
9
|
+
| "hot_locked";
|
|
10
|
+
|
|
11
|
+
export type ErrorDetails = Record<string, unknown>;
|
|
12
|
+
|
|
13
|
+
export class ExpectedError extends Error {
|
|
14
|
+
readonly code: ErrorCode;
|
|
15
|
+
readonly details: ErrorDetails;
|
|
16
|
+
|
|
17
|
+
constructor(code: ErrorCode, message: string, details: ErrorDetails = {}) {
|
|
18
|
+
super(message);
|
|
19
|
+
this.name = "ExpectedError";
|
|
20
|
+
this.code = code;
|
|
21
|
+
this.details = details;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function record(error: unknown): { code?: unknown; errcode?: unknown; errstr?: unknown; message?: unknown } {
|
|
26
|
+
return error !== null && typeof error === "object" ? error as Record<string, unknown> : {};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Extract the SQLite error code/string without losing the original message. */
|
|
30
|
+
export function sqliteInfo(error: unknown): { code?: string; errcode?: number; errstr?: string; message: string } {
|
|
31
|
+
const value = record(error);
|
|
32
|
+
return {
|
|
33
|
+
...(typeof value.code === "string" ? { code: value.code } : {}),
|
|
34
|
+
...(typeof value.errcode === "number" ? { errcode: value.errcode } : {}),
|
|
35
|
+
...(typeof value.errstr === "string" ? { errstr: value.errstr } : {}),
|
|
36
|
+
message: error instanceof Error ? error.message : String(error),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** True when an error came from the SQLite driver, including a closed connection. */
|
|
41
|
+
export function isSqliteError(error: unknown): boolean {
|
|
42
|
+
const code = record(error).code;
|
|
43
|
+
return typeof code === "string" && (code.startsWith("ERR_SQLITE") || code === "ERR_INVALID_STATE");
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Classify a known database access failure. Passes through already classified errors. */
|
|
47
|
+
export function storeUnavailable(operation: string, path: string, error: unknown): ExpectedError {
|
|
48
|
+
if (error instanceof ExpectedError) return error;
|
|
49
|
+
const info = sqliteInfo(error);
|
|
50
|
+
return new ExpectedError("store_unavailable", `SQLite is unavailable for ${operation} (${path}): ${info.message}`, {
|
|
51
|
+
operation,
|
|
52
|
+
path,
|
|
53
|
+
sqlite: info,
|
|
54
|
+
});
|
|
55
|
+
}
|
package/src/files.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { open, rename, rm } from "node:fs/promises";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
|
|
5
|
+
/** Same-directory replacement: readers see the previous or the complete new file. */
|
|
6
|
+
export async function atomicWrite(path: string, text: string): Promise<void> {
|
|
7
|
+
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
8
|
+
try {
|
|
9
|
+
const file = await open(temporary, "wx", 0o600);
|
|
10
|
+
try {
|
|
11
|
+
await file.writeFile(text, "utf8");
|
|
12
|
+
await file.sync();
|
|
13
|
+
} finally {
|
|
14
|
+
await file.close();
|
|
15
|
+
}
|
|
16
|
+
await rename(temporary, path);
|
|
17
|
+
// Directory fsync is not supported on Windows.
|
|
18
|
+
if (process.platform !== "win32") {
|
|
19
|
+
const directory = await open(dirname(path), "r");
|
|
20
|
+
try { await directory.sync(); } finally { await directory.close(); }
|
|
21
|
+
}
|
|
22
|
+
} finally {
|
|
23
|
+
await rm(temporary, { force: true });
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function isMissing(error: unknown): boolean {
|
|
28
|
+
return (error as NodeJS.ErrnoException)?.code === "ENOENT";
|
|
29
|
+
}
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import lockfile from "proper-lockfile";
|
|
5
|
+
import { atomicWrite, isMissing } from "./files.ts";
|
|
6
|
+
import { ExpectedError } from "./errors.ts";
|
|
7
|
+
import { HOT_LIMIT, integer, singleLine } from "./text.ts";
|
|
8
|
+
|
|
9
|
+
export interface HotMemory { id: string; priority: number; content: string }
|
|
10
|
+
export interface HotSnapshot { text: string; entries: HotMemory[]; characters: number; remaining: number; archived: HotMemory[]; backup?: string }
|
|
11
|
+
export interface HotAddResult extends HotSnapshot { memory: HotMemory; created: boolean; note?: string }
|
|
12
|
+
export interface HotEditResult extends HotSnapshot { memory: HotMemory }
|
|
13
|
+
export interface HotDeleteResult extends HotSnapshot { deleted: HotMemory }
|
|
14
|
+
type Queue = <T>(path: string, operation: () => Promise<T>) => Promise<T>;
|
|
15
|
+
|
|
16
|
+
function serialize(entries: HotMemory[]): string {
|
|
17
|
+
return entries.map((entry) => `- [${entry.id}] [p=${entry.priority}] ${entry.content}\n`).join("");
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function parse(text: string): HotMemory[] {
|
|
21
|
+
const entries: HotMemory[] = [];
|
|
22
|
+
for (const line of text.split(/\r?\n/u)) {
|
|
23
|
+
if (!line.trim() || /^#\s+MEMORY(?:\.md)?\s*$/iu.test(line)) continue;
|
|
24
|
+
if (/^\s+\S/u.test(line) && entries.length) {
|
|
25
|
+
entries[entries.length - 1]!.content += ` ${line.trim()}`;
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
const body = line.replace(/^\s*[-*+]\s+/u, "").trim();
|
|
29
|
+
const match = /^\[(h_[a-zA-Z0-9_-]+)\]\s+\[p=(\d+)\]\s+(.+)$/u.exec(body);
|
|
30
|
+
const content = match ? match[3]! : body;
|
|
31
|
+
const id = match ? match[1]! : `h_${createHash("sha256").update(body).digest("hex").slice(0, 16)}`;
|
|
32
|
+
if (entries.some((entry) => entry.id === id)) throw new Error(`Duplicate hot memory ID ${id}; repair MEMORY.md before editing.`);
|
|
33
|
+
entries.push({ id, priority: match ? integer(Number(match[2]), "priority", 0, 100) : 50, content });
|
|
34
|
+
}
|
|
35
|
+
return entries;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface CapacityEntry { id: string; priority: number; characters: number; content: string }
|
|
39
|
+
|
|
40
|
+
function described(entry: HotMemory): CapacityEntry {
|
|
41
|
+
return { id: entry.id, priority: entry.priority, characters: serialize([entry]).length, content: entry.content };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Exact serialized diagnostics for a refused hot operation. `current` is the
|
|
46
|
+
* pre-change list, `proposed` includes the requested add/edit, and `entry` is
|
|
47
|
+
* the proposed or duplicated bullet.
|
|
48
|
+
*/
|
|
49
|
+
function capacityDetails(reason: string, operation: string, current: HotMemory[], proposed: HotMemory[], entry: HotMemory) {
|
|
50
|
+
const currentCharacters = serialize(current).length;
|
|
51
|
+
const proposedCharacters = serialize(proposed).length;
|
|
52
|
+
const eligible = current
|
|
53
|
+
.filter((item) => item.id !== entry.id && item.priority < entry.priority)
|
|
54
|
+
.sort((a, b) => a.priority - b.priority)
|
|
55
|
+
.map(described);
|
|
56
|
+
const eligibleCharacters = eligible.reduce((sum, item) => sum + item.characters, 0);
|
|
57
|
+
return {
|
|
58
|
+
reason,
|
|
59
|
+
operation,
|
|
60
|
+
limit: HOT_LIMIT,
|
|
61
|
+
current_characters: currentCharacters,
|
|
62
|
+
proposed_characters: proposedCharacters,
|
|
63
|
+
over_by: Math.max(0, proposedCharacters - HOT_LIMIT),
|
|
64
|
+
remaining: Math.max(0, HOT_LIMIT - currentCharacters),
|
|
65
|
+
proposed_entry: described(entry),
|
|
66
|
+
entries: current.map(described),
|
|
67
|
+
eligible,
|
|
68
|
+
eligible_characters: eligibleCharacters,
|
|
69
|
+
shortfall_after_eligible_eviction: Math.max(0, proposedCharacters - eligibleCharacters - HOT_LIMIT),
|
|
70
|
+
live_file_unchanged: true,
|
|
71
|
+
next_actions: [
|
|
72
|
+
"Shorten the bullet or explicitly delete lower-value bullets with memoria_delete.",
|
|
73
|
+
"Save the fact in SQLite long-term memory instead.",
|
|
74
|
+
"Use memoria_edit to raise the priority of an important entry only when justified.",
|
|
75
|
+
],
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export class HotMemoryStore {
|
|
80
|
+
constructor(
|
|
81
|
+
readonly path: string,
|
|
82
|
+
private archive: (entries: HotMemory[]) => void,
|
|
83
|
+
private queue: Queue = async (_path, operation) => operation(),
|
|
84
|
+
) {}
|
|
85
|
+
|
|
86
|
+
private async locked<T>(operation: () => Promise<T>, signal?: AbortSignal): Promise<T> {
|
|
87
|
+
return this.queue(this.path, async () => {
|
|
88
|
+
signal?.throwIfAborted();
|
|
89
|
+
await mkdir(dirname(this.path), { recursive: true, mode: 0o700 });
|
|
90
|
+
let release: () => Promise<void>;
|
|
91
|
+
try {
|
|
92
|
+
release = await lockfile.lock(this.path, {
|
|
93
|
+
realpath: false, stale: 10_000, update: 2_000,
|
|
94
|
+
retries: { retries: 80, minTimeout: 10, maxTimeout: 100, randomize: true },
|
|
95
|
+
});
|
|
96
|
+
} catch (error) {
|
|
97
|
+
const code = (error as NodeJS.ErrnoException | undefined)?.code;
|
|
98
|
+
if (code === "ELOCKED") {
|
|
99
|
+
throw new ExpectedError("hot_locked", `Could not lock MEMORY.md within the retry window (ELOCKED). Another process is mutating it; retry shortly.`, { path: this.path, lock_code: code });
|
|
100
|
+
}
|
|
101
|
+
throw error;
|
|
102
|
+
}
|
|
103
|
+
try { signal?.throwIfAborted(); return await operation(); } finally { await release(); }
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private async read(): Promise<string | undefined> {
|
|
108
|
+
try { return await readFile(this.path, "utf8"); }
|
|
109
|
+
catch (error) { if (isMissing(error)) return undefined; throw error; }
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
private snapshot(entries: HotMemory[], archived: HotMemory[], backup?: string): HotSnapshot {
|
|
113
|
+
const text = serialize(entries);
|
|
114
|
+
return { text, entries, characters: text.length, remaining: HOT_LIMIT - text.length, archived, ...(backup ? { backup } : {}) };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
private capacityError(reason: string, operation: string, current: HotMemory[], proposed: HotMemory[], entry: HotMemory): ExpectedError {
|
|
118
|
+
const details = capacityDetails(reason, operation, current, proposed, entry);
|
|
119
|
+
const prefix = reason === "bullet_too_large"
|
|
120
|
+
? `A single hot bullet serializes to ${details.proposed_entry.characters} characters including metadata, over the ${HOT_LIMIT}-character total limit.`
|
|
121
|
+
: reason === "existing_over_limit"
|
|
122
|
+
? `MEMORY.md is already over the ${HOT_LIMIT}-character limit and must be repaired before duplicate content can be returned safely.`
|
|
123
|
+
: `MEMORY.md is ${details.over_by} characters over budget even after evicting every eligible lower-priority entry (shortfall ${details.shortfall_after_eligible_eviction}).`;
|
|
124
|
+
return new ExpectedError("hot_capacity", `${prefix} The live file was not changed. Shorten the content, delete a bullet, use SQLite, or adjust priority.`, details);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Normalize manual edits; preserve a backup before any automatic capacity repair. */
|
|
128
|
+
async load(): Promise<HotSnapshot> {
|
|
129
|
+
return this.locked(async () => {
|
|
130
|
+
const original = await this.read();
|
|
131
|
+
const previous = original ?? "";
|
|
132
|
+
const entries = parse(previous);
|
|
133
|
+
const archived: HotMemory[] = [];
|
|
134
|
+
while (serialize(entries).length > HOT_LIMIT) {
|
|
135
|
+
const victim = entries.reduce((a, b) => b.priority < a.priority ? b : a);
|
|
136
|
+
archived.push(victim);
|
|
137
|
+
entries.splice(entries.indexOf(victim), 1);
|
|
138
|
+
}
|
|
139
|
+
let backup: string | undefined;
|
|
140
|
+
if (archived.length) {
|
|
141
|
+
const backups = join(dirname(this.path), "backups");
|
|
142
|
+
await mkdir(backups, { recursive: true, mode: 0o700 });
|
|
143
|
+
backup = join(backups, `MEMORY-${Date.now()}-${randomUUID()}.md`);
|
|
144
|
+
await writeFile(backup, previous, { flag: "wx", mode: 0o600, flush: true });
|
|
145
|
+
this.archive(archived);
|
|
146
|
+
}
|
|
147
|
+
const result = this.snapshot(entries, archived, backup);
|
|
148
|
+
// Ensure a real file exists even when the store is empty.
|
|
149
|
+
if (result.text !== previous || original === undefined) await atomicWrite(this.path, result.text);
|
|
150
|
+
return result;
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
async add(content: string, priority = 50, signal?: AbortSignal): Promise<HotAddResult> {
|
|
155
|
+
const normalized = singleLine(content);
|
|
156
|
+
const requested = integer(priority, "priority", 0, 100);
|
|
157
|
+
return this.locked(async () => {
|
|
158
|
+
const entries = parse((await this.read()) ?? "");
|
|
159
|
+
const existing = entries.find((entry) => entry.content === normalized);
|
|
160
|
+
if (existing) {
|
|
161
|
+
// An oversized live state must not be reported as a successful snapshot.
|
|
162
|
+
if (serialize(entries).length > HOT_LIMIT) throw this.capacityError("existing_over_limit", "add", entries, entries, existing);
|
|
163
|
+
return {
|
|
164
|
+
...this.snapshot(entries, []), memory: existing, created: false,
|
|
165
|
+
note: `Reused existing hot memory ${existing.id}; its content, priority ${existing.priority}, and file position were preserved. Use memoria_edit to change its content or priority.`,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
const entry: HotMemory = { id: `h_${randomUUID().replaceAll("-", "").slice(0, 16)}`, priority: requested, content: normalized };
|
|
169
|
+
return { ...await this.commit(entries, entry, undefined, "add", signal), created: true };
|
|
170
|
+
}, signal);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async edit(id: string, content?: string, priority?: number, expectedContent?: string, signal?: AbortSignal): Promise<HotEditResult> {
|
|
174
|
+
const normalized = content === undefined ? undefined : singleLine(content);
|
|
175
|
+
const requested = priority === undefined ? undefined : integer(priority, "priority", 0, 100);
|
|
176
|
+
return this.locked(async () => {
|
|
177
|
+
const entries = parse((await this.read()) ?? "");
|
|
178
|
+
const index = entries.findIndex((entry) => entry.id === id);
|
|
179
|
+
if (index === -1) throw new ExpectedError("not_found", `Hot memory ${id} does not exist. Read MEMORY.md again.`, { id });
|
|
180
|
+
const previous = entries[index]!;
|
|
181
|
+
if (expectedContent !== undefined && previous.content !== expectedContent) {
|
|
182
|
+
throw new ExpectedError("conflict", `Hot memory ${id} changed (stored content differs from expected_content). Read MEMORY.md again before editing.`, { id, expected_content: expectedContent, current_content: previous.content });
|
|
183
|
+
}
|
|
184
|
+
const entry: HotMemory = { id, content: normalized ?? previous.content, priority: requested ?? previous.priority };
|
|
185
|
+
if (normalized !== undefined && normalized !== previous.content) {
|
|
186
|
+
const duplicate = entries.find((item) => item.id !== id && item.content === normalized);
|
|
187
|
+
if (duplicate) {
|
|
188
|
+
throw new ExpectedError("duplicate_hot_content", `Edit rejected: setting ${id} to that content would duplicate hot memory ${duplicate.id}. Neither entry was changed; keep one entry or choose distinct content.`, { id, existing_id: duplicate.id });
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return this.commit(entries, entry, index, "edit", signal);
|
|
192
|
+
}, signal);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
private async commit(entriesBefore: HotMemory[], entry: HotMemory, index: number | undefined, operation: string, signal?: AbortSignal): Promise<HotEditResult> {
|
|
196
|
+
signal?.throwIfAborted();
|
|
197
|
+
const proposed = index === undefined ? [...entriesBefore, entry] : entriesBefore.map((item, current) => current === index ? entry : item);
|
|
198
|
+
if (serialize([entry]).length > HOT_LIMIT) throw this.capacityError("bullet_too_large", operation, entriesBefore, proposed, entry);
|
|
199
|
+
const entries = proposed.slice();
|
|
200
|
+
const archived: HotMemory[] = [];
|
|
201
|
+
if (serialize(entries).length > HOT_LIMIT) {
|
|
202
|
+
const candidates = entriesBefore.filter((item) => item.id !== entry.id && item.priority < entry.priority).sort((a, b) => a.priority - b.priority);
|
|
203
|
+
while (serialize(entries).length > HOT_LIMIT) {
|
|
204
|
+
const victim = candidates.shift();
|
|
205
|
+
if (!victim) throw this.capacityError("cannot_free_space", operation, entriesBefore, proposed, entry);
|
|
206
|
+
archived.push(victim);
|
|
207
|
+
entries.splice(entries.indexOf(victim), 1);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
if (archived.length) this.archive(archived);
|
|
211
|
+
const result = this.snapshot(entries, archived);
|
|
212
|
+
await atomicWrite(this.path, result.text);
|
|
213
|
+
return { ...result, memory: entry };
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
async delete(id: string, expectedContent?: string, signal?: AbortSignal): Promise<HotDeleteResult> {
|
|
217
|
+
return this.locked(async () => {
|
|
218
|
+
const entries = parse((await this.read()) ?? "");
|
|
219
|
+
const index = entries.findIndex((entry) => entry.id === id);
|
|
220
|
+
if (index === -1) throw new ExpectedError("not_found", `Hot memory ${id} does not exist.`, { id });
|
|
221
|
+
const deleted = entries[index]!;
|
|
222
|
+
if (expectedContent !== undefined && deleted.content !== expectedContent) {
|
|
223
|
+
throw new ExpectedError("conflict", `Hot memory ${id} changed (stored content differs from expected_content). Read MEMORY.md again before deleting.`, { id, expected_content: expectedContent, current_content: deleted.content });
|
|
224
|
+
}
|
|
225
|
+
entries.splice(index, 1);
|
|
226
|
+
const result = this.snapshot(entries, []);
|
|
227
|
+
await atomicWrite(this.path, result.text);
|
|
228
|
+
return { ...result, deleted };
|
|
229
|
+
}, signal);
|
|
230
|
+
}
|
|
231
|
+
}
|