renor-engine 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +73 -0
- package/README.md +105 -2
- package/dist/BUILD.json +6 -0
- package/dist/api/apiAuth.js +43 -0
- package/dist/api/server.js +172 -0
- package/dist/buildIdentity.js +132 -0
- package/dist/cli-extractors.js +46 -0
- package/dist/cli.js +491 -0
- package/dist/config.js +65 -0
- package/dist/context/composer.js +320 -0
- package/dist/context/readHonesty.js +478 -0
- package/dist/context/rowStore.js +238 -0
- package/dist/conversation/formatter.js +81 -0
- package/dist/conversation/parser.js +189 -0
- package/dist/conversation/types.js +2 -0
- package/dist/conversation/writer.js +42 -0
- package/dist/db/client.js +147 -0
- package/dist/db/indexSalvage.js +81 -0
- package/dist/db/migrations.js +642 -0
- package/dist/db/readCache.js +109 -0
- package/dist/db/schema.js +5 -0
- package/dist/db/statements.js +89 -0
- package/dist/digests/digestCache.js +142 -0
- package/dist/digests/digests.js +597 -0
- package/dist/digests/worker.js +348 -0
- package/dist/embeddings/caching.js +337 -0
- package/dist/embeddings/coalescing.js +177 -0
- package/dist/embeddings/interactive.js +103 -0
- package/dist/embeddings/local.js +68 -0
- package/dist/embeddings/openai.js +105 -0
- package/dist/embeddings/rebuild.js +166 -0
- package/dist/embeddings/simdScan.js +258 -0
- package/dist/embeddings/space.js +21 -0
- package/dist/embeddings/types.js +14 -0
- package/dist/embeddings/vectorIndex.js +675 -0
- package/dist/embeddings/vertex.js +224 -0
- package/dist/eval/artifacts.js +42 -0
- package/dist/eval/cacheReport.js +139 -0
- package/dist/eval/comparison.js +182 -0
- package/dist/eval/context700-candidate.js +145 -0
- package/dist/eval/evalCache.js +525 -0
- package/dist/eval/hashEmbedding.js +35 -0
- package/dist/eval/longmemeval.js +56 -0
- package/dist/eval/metrics.js +181 -0
- package/dist/eval/renorbenchBaseline.js +486 -0
- package/dist/eval/renorbenchDocuments.js +134 -0
- package/dist/eval/renorbenchProduct.js +182 -0
- package/dist/eval/runLedger.js +279 -0
- package/dist/eval/runManifest.js +479 -0
- package/dist/eval/scoring.js +197 -0
- package/dist/evidence/attach.js +267 -0
- package/dist/evidence/background.js +147 -0
- package/dist/evidence/evidenceCache.js +182 -0
- package/dist/evidence/gate.js +37 -0
- package/dist/evidence/keyPool.js +85 -0
- package/dist/evidence/ollamaChat.js +84 -0
- package/dist/evidence/optin.js +165 -0
- package/dist/evidence/options.js +130 -0
- package/dist/evidence/secondPass.js +456 -0
- package/dist/extraction/classifyCapture.js +34 -0
- package/dist/extraction/decision-model.js +174 -0
- package/dist/extraction/eventExtractor.js +113 -0
- package/dist/extraction/extractionClient.js +122 -0
- package/dist/extraction/factExtractor.js +46 -0
- package/dist/extraction/factsWritePath.js +179 -0
- package/dist/extraction/preferenceExtractor.js +153 -0
- package/dist/extraction/secretGuard.js +176 -0
- package/dist/extraction/secretScanner.js +665 -0
- package/dist/extraction/typedFacts.js +224 -0
- package/dist/extraction/writePathAdjudication.js +110 -0
- package/dist/importing/apply.js +644 -0
- package/dist/importing/bundle-write.js +80 -0
- package/dist/importing/cli.js +407 -0
- package/dist/importing/existing.js +49 -0
- package/dist/importing/facts.js +123 -0
- package/dist/importing/forgetImported.js +79 -0
- package/dist/importing/hosted.js +467 -0
- package/dist/importing/index.js +29 -0
- package/dist/importing/journalLog.js +96 -0
- package/dist/importing/markdown.js +98 -0
- package/dist/importing/plan.js +206 -0
- package/dist/importing/secretScreen.js +113 -0
- package/dist/importing/serialize.js +188 -0
- package/dist/importing/source.js +463 -0
- package/dist/importing/types.js +6 -0
- package/dist/indexer/chunker.js +319 -0
- package/dist/indexer/dailyEntries.js +63 -0
- package/dist/indexer/hasher.js +10 -0
- package/dist/indexer/laneRuns.js +48 -0
- package/dist/indexer/o200k.js +481 -0
- package/dist/indexer/o200k.json +8 -0
- package/dist/indexer/outsideRoot.js +77 -0
- package/dist/indexer/parser.js +167 -0
- package/dist/indexer/pipeline.js +1168 -0
- package/dist/indexer/staleness.js +201 -0
- package/dist/indexer/tokenizer.js +77 -0
- package/dist/indexer/walker.js +184 -0
- package/dist/mcp/hive.js +88 -0
- package/dist/mcp/requestErrors.js +47 -0
- package/dist/mcp/server.js +396 -0
- package/dist/memory/agentLease.js +43 -0
- package/dist/memory/confinedWrite.js +315 -0
- package/dist/memory/controls.js +73 -0
- package/dist/memory/dailyNote.js +265 -0
- package/dist/memory/durableAppend.js +57 -0
- package/dist/memory/factConflicts.js +203 -0
- package/dist/memory/factTerms.js +321 -0
- package/dist/memory/hiveNameHeader.cjs +10 -0
- package/dist/memory/inventory.js +55 -0
- package/dist/memory/invisibleText.js +20 -0
- package/dist/memory/lifecycle.js +1756 -0
- package/dist/memory/limits.js +13 -0
- package/dist/memory/requests.js +183 -0
- package/dist/memory/sourceName.js +200 -0
- package/dist/memory/whatsNew.js +158 -0
- package/dist/memory/writer.js +175 -0
- package/dist/memory/writerConflicts.js +85 -0
- package/dist/memorymap/memoryMap.js +1568 -0
- package/dist/memorymap/replacedJournal.js +58 -0
- package/dist/memorymap/replacements.js +405 -0
- package/dist/mvp/activity.js +86 -0
- package/dist/mvp/agentMemory.js +1744 -0
- package/dist/mvp/auth.js +531 -0
- package/dist/mvp/cli.js +73 -0
- package/dist/mvp/client-store.js +462 -0
- package/dist/mvp/compileCache.js +249 -0
- package/dist/mvp/compileSeed.js +27 -0
- package/dist/mvp/config.js +288 -0
- package/dist/mvp/connectionStatus.js +80 -0
- package/dist/mvp/contextRelevance.js +413 -0
- package/dist/mvp/engine.js +1065 -0
- package/dist/mvp/exportAll.js +359 -0
- package/dist/mvp/factLanePack.js +77 -0
- package/dist/mvp/factMarkers.js +49 -0
- package/dist/mvp/factsLane.js +203 -0
- package/dist/mvp/hiddenSpans.js +585 -0
- package/dist/mvp/history.js +413 -0
- package/dist/mvp/hitDates.js +31 -0
- package/dist/mvp/hitFloor.js +36 -0
- package/dist/mvp/identity.js +18 -0
- package/dist/mvp/idle.js +113 -0
- package/dist/mvp/memoryDir.js +98 -0
- package/dist/mvp/posthogMcp.js +210 -0
- package/dist/mvp/readScope.js +104 -0
- package/dist/mvp/readUsefulness.js +222 -0
- package/dist/mvp/sanitize.js +48 -0
- package/dist/mvp/schemaRejections.js +91 -0
- package/dist/mvp/scope.js +127 -0
- package/dist/mvp/server.js +2089 -0
- package/dist/mvp/serverCard.js +29 -0
- package/dist/mvp/storedSize.js +139 -0
- package/dist/mvp/telemetry.js +441 -0
- package/dist/retrieval/adviceLines.js +195 -0
- package/dist/retrieval/adviceQuestion.js +44 -0
- package/dist/retrieval/conversationShape.js +44 -0
- package/dist/retrieval/dailyEntryTime.js +55 -0
- package/dist/retrieval/entityLookup.js +113 -0
- package/dist/retrieval/factsFirst.js +188 -0
- package/dist/retrieval/keyword.js +84 -0
- package/dist/retrieval/ledgerEventTime.js +51 -0
- package/dist/retrieval/ledgerLiveness.js +83 -0
- package/dist/retrieval/ledgerPresentation.js +161 -0
- package/dist/retrieval/noteIdentity.js +57 -0
- package/dist/retrieval/orchestrator.js +1086 -0
- package/dist/retrieval/queryHelpers.js +142 -0
- package/dist/retrieval/questionSpan.js +475 -0
- package/dist/retrieval/readerEvents.js +690 -0
- package/dist/retrieval/readerLayout.js +503 -0
- package/dist/retrieval/relativeTimeHints.js +113 -0
- package/dist/retrieval/semantic.js +67 -0
- package/dist/router/routerCache.js +106 -0
- package/dist/router/sessionRouter.js +726 -0
- package/dist/toolRefusal.js +120 -0
- package/dist/watcher/watcher.js +75 -0
- package/package.json +120 -4
|
@@ -0,0 +1,1568 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.NOFOLLOW = exports.DEFAULT_MAP_LIMIT = exports.RETIRED_DIR = void 0;
|
|
7
|
+
exports.resolveVaultFile = resolveVaultFile;
|
|
8
|
+
exports.parseEntries = parseEntries;
|
|
9
|
+
exports.appendedEntryId = appendedEntryId;
|
|
10
|
+
exports.parseMemoryId = parseMemoryId;
|
|
11
|
+
exports.openConfined = openConfined;
|
|
12
|
+
exports.readPrefix = readPrefix;
|
|
13
|
+
exports.readWhole = readWhole;
|
|
14
|
+
exports.ensureRealDir = ensureRealDir;
|
|
15
|
+
exports.readJournal = readJournal;
|
|
16
|
+
exports.activeRetired = activeRetired;
|
|
17
|
+
exports.buildMemoryMap = buildMemoryMap;
|
|
18
|
+
exports.findMemories = findMemories;
|
|
19
|
+
exports.readMemory = readMemory;
|
|
20
|
+
exports.retireMemory = retireMemory;
|
|
21
|
+
exports.restoreMemory = restoreMemory;
|
|
22
|
+
exports.purgeRetired = purgeRetired;
|
|
23
|
+
exports.dailyEntriesForExport = dailyEntriesForExport;
|
|
24
|
+
exports.retiredForExport = retiredForExport;
|
|
25
|
+
const promises_1 = __importDefault(require("fs/promises"));
|
|
26
|
+
const fs_1 = require("fs");
|
|
27
|
+
const path_1 = __importDefault(require("path"));
|
|
28
|
+
const crypto_1 = __importDefault(require("crypto"));
|
|
29
|
+
const hasher_1 = require("../indexer/hasher");
|
|
30
|
+
const parser_1 = require("../indexer/parser");
|
|
31
|
+
const walker_1 = require("../indexer/walker");
|
|
32
|
+
const pipeline_1 = require("../indexer/pipeline");
|
|
33
|
+
const lifecycle_1 = require("../memory/lifecycle");
|
|
34
|
+
const writerConflicts_1 = require("../memory/writerConflicts");
|
|
35
|
+
const dailyNote_1 = require("../memory/dailyNote");
|
|
36
|
+
const writer_1 = require("../memory/writer");
|
|
37
|
+
const scope_1 = require("../mvp/scope");
|
|
38
|
+
const secretGuard_1 = require("../extraction/secretGuard");
|
|
39
|
+
const toolRefusal_1 = require("../toolRefusal");
|
|
40
|
+
const evidenceCache_1 = require("../evidence/evidenceCache");
|
|
41
|
+
const forgetImported_1 = require("../importing/forgetImported");
|
|
42
|
+
const replacedJournal_1 = require("./replacedJournal");
|
|
43
|
+
const orchestrator_1 = require("../retrieval/orchestrator");
|
|
44
|
+
// ---------------------------------------------------------------------------
|
|
45
|
+
// Memory map: a read model of everything one workspace holds, shaped for a
|
|
46
|
+
// human to browse (the dashboard's graph + reader), plus reversible retirement.
|
|
47
|
+
//
|
|
48
|
+
// It is derived only from what the engine already stores — note files on disk,
|
|
49
|
+
// the notes/facts/entities tables and write_events. Nothing is inferred by a
|
|
50
|
+
// model: a link exists because a wikilink, a tag or a literal name mention says
|
|
51
|
+
// so, and every link carries which of those it is.
|
|
52
|
+
//
|
|
53
|
+
// Units a person thinks of as "a memory":
|
|
54
|
+
// - note : one Markdown file (imported notes, remember_note files, …);
|
|
55
|
+
// - entry : one entry of a daily log (Memory/**/Daily/YYYY-MM-DD.md), which is
|
|
56
|
+
// where the unstructured `remember` path appends. An entry is its
|
|
57
|
+
// timestamped line plus any continuation lines a multi-line
|
|
58
|
+
// statement wrote below it.
|
|
59
|
+
// Lifecycle facts (the canonical ledger) are listed separately with their full
|
|
60
|
+
// history. A fact is retired through the ledger (memory/lifecycle.ts, retireFact):
|
|
61
|
+
// retire_memory and restore_memory take its id, `fact:` and its event id, and
|
|
62
|
+
// hand it there.
|
|
63
|
+
//
|
|
64
|
+
// Retirement is a move, never a delete. Every retirement is journalled BEFORE
|
|
65
|
+
// the move, so an interrupted one is either still in place (and the journal
|
|
66
|
+
// record is void) or moved (and the record restores it):
|
|
67
|
+
// - a note file moves under `.trash/renor-retired/files/`;
|
|
68
|
+
// - an entry is cut out of its daily file by an atomic rewrite, under the
|
|
69
|
+
// same per-file lock every daily-file writer takes, and its exact bytes live
|
|
70
|
+
// in the journal.
|
|
71
|
+
// `.trash/` is ignored by the indexer, so retired text leaves retrieval, and
|
|
72
|
+
// restore puts the exact bytes back and re-indexes them.
|
|
73
|
+
//
|
|
74
|
+
// Why move bytes rather than mark them: retrieval does not only read the index —
|
|
75
|
+
// the evidence pass hydrates whole note files from disk — so a "retired" line
|
|
76
|
+
// left inside a daily log could still reach an agent's context. Moving it out of
|
|
77
|
+
// the file is the only guarantee that holds for every reader.
|
|
78
|
+
//
|
|
79
|
+
// Identical entries in one daily log (the same statement saved twice) are one
|
|
80
|
+
// memory: they share an id, and retiring it removes every copy.
|
|
81
|
+
//
|
|
82
|
+
// Confinement: ids address vault-relative Markdown paths only. No component may
|
|
83
|
+
// be hidden ('.'-prefixed: .trash, .obsidian, the index), no component may be a
|
|
84
|
+
// symlink, the real path must stay inside the vault's real path, and the fact
|
|
85
|
+
// ledger is never addressable as a memory. Reads open with O_NOFOLLOW and check
|
|
86
|
+
// the opened inode against the path. Threat model: the vault is written only by
|
|
87
|
+
// this engine; these checks are defence in depth against a mis-provisioned vault
|
|
88
|
+
// (a stray symlink), not a sandbox against a concurrent local attacker, who would
|
|
89
|
+
// already own the files.
|
|
90
|
+
// ---------------------------------------------------------------------------
|
|
91
|
+
const LIFECYCLE_NOTE_PATH = '_mcp_';
|
|
92
|
+
/** Frontmatter keys the indexer turns into note facts; they are note metadata, not knowledge. */
|
|
93
|
+
const METADATA_PREDICATES = new Set(['owner', 'status', 'updated', 'tags']);
|
|
94
|
+
/** The front-matter keys read_memory returns as `metadata` (MemoryDetail). Any other key is not copied. */
|
|
95
|
+
const METADATA_KEYS = new Set(['title', 'status', 'owner', 'updated', 'tags']);
|
|
96
|
+
exports.RETIRED_DIR = path_1.default.join('.trash', 'renor-retired');
|
|
97
|
+
const RETIRED_FILES_DIR = path_1.default.join(exports.RETIRED_DIR, 'files');
|
|
98
|
+
const RETIRED_INDEX = path_1.default.join(exports.RETIRED_DIR, 'retired.jsonl');
|
|
99
|
+
const MAX_MEMORIES = 2000;
|
|
100
|
+
/** Notes considered per map (newest first); bounds work before the memory cap applies. */
|
|
101
|
+
const MAX_NOTES_SCANNED = 5000;
|
|
102
|
+
/** Newest entries kept per daily log. */
|
|
103
|
+
const MAX_ENTRIES_PER_DAY = 500;
|
|
104
|
+
const MAX_THINGS = 2000;
|
|
105
|
+
const MAX_FACTS = 5000;
|
|
106
|
+
const MAX_LINKS = 20000;
|
|
107
|
+
const MAX_EVENTS = 1000;
|
|
108
|
+
const MAX_RETIRED_LISTED = 500;
|
|
109
|
+
/**
|
|
110
|
+
* Items per list when a caller names no limit (memory_map's first call): small enough for one tool result on a large
|
|
111
|
+
* workspace, and the reply says how many more exist.
|
|
112
|
+
*/
|
|
113
|
+
exports.DEFAULT_MAP_LIMIT = 20;
|
|
114
|
+
/** Bytes read per note for the map (excerpt + mention matching). */
|
|
115
|
+
const MAP_READ_BYTES = 64 * 1024;
|
|
116
|
+
/** Bytes read for one memory in full. */
|
|
117
|
+
const READ_BYTES = 512 * 1024;
|
|
118
|
+
/** A daily file larger than this is not rewritten (retire/restore refuse instead of truncating). */
|
|
119
|
+
const MAX_REWRITE_BYTES = 8 * 1024 * 1024;
|
|
120
|
+
const EXCERPT_CHARS = 280;
|
|
121
|
+
const MAX_READ_CHARS = 40_000;
|
|
122
|
+
const MIN_MENTION_CHARS = 3;
|
|
123
|
+
// ------------------------------- paths --------------------------------------
|
|
124
|
+
function toRel(memoryDir, stored) {
|
|
125
|
+
const root = path_1.default.resolve(memoryDir);
|
|
126
|
+
const resolved = path_1.default.resolve(root, stored);
|
|
127
|
+
const rel = path_1.default.relative(root, resolved);
|
|
128
|
+
if (!rel || rel === '..' || rel.startsWith('..' + path_1.default.sep) || path_1.default.isAbsolute(rel))
|
|
129
|
+
return null;
|
|
130
|
+
return rel.split(path_1.default.sep).join('/');
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* When a daily entry was saved, as retrieval dates it: the instant its header records, capped at its note's write time
|
|
134
|
+
* (createNoteTimeBound, the bound get_context and search_memory use), so a self-chosen file date cannot list an entry
|
|
135
|
+
* above what was written after it. Without a readable bound, the write time itself is the answer.
|
|
136
|
+
*/
|
|
137
|
+
function entryAt(bound, notePath, day, header, writtenAt) {
|
|
138
|
+
return bound(notePath, (0, dailyNote_1.entryInstant)(day, header)) ?? writtenAt;
|
|
139
|
+
}
|
|
140
|
+
function within(root, candidate) {
|
|
141
|
+
const rel = path_1.default.relative(root, candidate);
|
|
142
|
+
return rel !== '' && rel !== '..' && !rel.startsWith('..' + path_1.default.sep) && !path_1.default.isAbsolute(rel);
|
|
143
|
+
}
|
|
144
|
+
/** Lexical checks shared by every addressed path. */
|
|
145
|
+
function lexicalRel(rel) {
|
|
146
|
+
if (typeof rel !== 'string' || rel.trim() === '' || path_1.default.isAbsolute(rel) || rel.includes('\0') || rel.includes('\\')) {
|
|
147
|
+
throw (0, toolRefusal_1.refusal)('unsafe_path', 'memory path must be a vault-relative path');
|
|
148
|
+
}
|
|
149
|
+
const parts = rel.split('/');
|
|
150
|
+
if (parts.some((p) => p === '..'))
|
|
151
|
+
throw (0, toolRefusal_1.refusal)('unsafe_path', 'memory path escapes the memory directory');
|
|
152
|
+
if (parts.some((p) => p === '' || p === '.'))
|
|
153
|
+
throw (0, toolRefusal_1.refusal)('unsafe_path', 'memory path must be a vault-relative path');
|
|
154
|
+
if (parts.some((p) => p.startsWith('.')))
|
|
155
|
+
throw (0, toolRefusal_1.refusal)('unsafe_path', 'memory path points into a hidden or system directory');
|
|
156
|
+
if (!rel.endsWith('.md'))
|
|
157
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'memory path must be a Markdown file');
|
|
158
|
+
return parts.join('/');
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Resolve a vault-relative path and prove it is confined: every existing component below the vault
|
|
162
|
+
* root is a real directory (no symlink), the file itself is not a symlink, and the real path is
|
|
163
|
+
* inside the vault's real path. `mustExist: false` allows a missing file (restore) but still checks
|
|
164
|
+
* every directory that exists.
|
|
165
|
+
*/
|
|
166
|
+
async function resolveVaultFile(ctx, rel, mustExist) {
|
|
167
|
+
const clean = lexicalRel(rel);
|
|
168
|
+
const rootReal = await promises_1.default.realpath(ctx.memoryDir);
|
|
169
|
+
const parts = clean.split('/');
|
|
170
|
+
let cursor = rootReal;
|
|
171
|
+
for (let i = 0; i < parts.length; i++) {
|
|
172
|
+
cursor = path_1.default.join(cursor, parts[i]);
|
|
173
|
+
let st;
|
|
174
|
+
try {
|
|
175
|
+
st = await promises_1.default.lstat(cursor);
|
|
176
|
+
}
|
|
177
|
+
catch (err) {
|
|
178
|
+
if (err.code === 'ENOENT') {
|
|
179
|
+
if (mustExist)
|
|
180
|
+
throw (0, toolRefusal_1.refusal)('memory_not_found', 'memory not found (it may have been retired)');
|
|
181
|
+
break; // nothing below exists; nothing below can be a symlink yet
|
|
182
|
+
}
|
|
183
|
+
throw err;
|
|
184
|
+
}
|
|
185
|
+
if (st.isSymbolicLink())
|
|
186
|
+
throw (0, toolRefusal_1.refusal)('unsafe_path', 'memory path goes through a symbolic link');
|
|
187
|
+
if (i < parts.length - 1 && !st.isDirectory())
|
|
188
|
+
throw (0, toolRefusal_1.refusal)('memory_not_found', 'memory not found');
|
|
189
|
+
if (i === parts.length - 1 && !st.isFile())
|
|
190
|
+
throw (0, toolRefusal_1.refusal)('unsafe_path', 'memory path is not a file');
|
|
191
|
+
}
|
|
192
|
+
const real = path_1.default.join(rootReal, ...parts);
|
|
193
|
+
if (!within(rootReal, real))
|
|
194
|
+
throw (0, toolRefusal_1.refusal)('unsafe_path', 'memory path escapes the memory directory');
|
|
195
|
+
const file = { rel: clean, indexed: path_1.default.resolve(ctx.memoryDir, ...parts), real };
|
|
196
|
+
if (await isLedgerFile(ctx, file))
|
|
197
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'the fact ledger is not a memory; read facts through memory_map');
|
|
198
|
+
return file;
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* A file the owner left out of memory (.semanticignore, node_modules, ...) is not found by search, so it is not read or moved
|
|
202
|
+
* by its id either. Only read and retire ask: restore and reconcile must still reach a note that was retired before the
|
|
203
|
+
* owner added the rule.
|
|
204
|
+
*/
|
|
205
|
+
async function assertNotExcluded(ctx, file) {
|
|
206
|
+
if (await (0, walker_1.isIgnoredByVault)(ctx.memoryDir, file.rel))
|
|
207
|
+
throw (0, toolRefusal_1.refusal)('memory_not_found', 'memory not found');
|
|
208
|
+
}
|
|
209
|
+
async function isLedgerFile(ctx, file) {
|
|
210
|
+
let ledger;
|
|
211
|
+
try {
|
|
212
|
+
ledger = (0, lifecycle_1.resolveLedgerPath)(ctx.memoryDir, ctx.ledgerRelPath);
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
return false;
|
|
216
|
+
}
|
|
217
|
+
if (path_1.default.resolve(ledger) === path_1.default.resolve(file.indexed))
|
|
218
|
+
return true;
|
|
219
|
+
try {
|
|
220
|
+
return (await promises_1.default.realpath(ledger)) === file.real;
|
|
221
|
+
}
|
|
222
|
+
catch {
|
|
223
|
+
return false;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
function isLedgerIndexed(ctx, indexed) {
|
|
227
|
+
try {
|
|
228
|
+
return path_1.default.resolve(indexed) === (0, lifecycle_1.resolveLedgerPath)(ctx.memoryDir, ctx.ledgerRelPath);
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
return false;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
function entryId(rel, raw) {
|
|
235
|
+
const digest = crypto_1.default.createHash('sha256').update(raw).digest('hex').slice(0, 16);
|
|
236
|
+
return `entry:${rel}#${digest}`;
|
|
237
|
+
}
|
|
238
|
+
/** A blank line is layout: spaces, tabs and carriage returns. Any other character on a line is the user's. */
|
|
239
|
+
const BLANK_LINE_RE = /^[ \t\r]*$/;
|
|
240
|
+
/**
|
|
241
|
+
* Split a daily file into entries: a timestamped line plus the continuation lines below it (a
|
|
242
|
+
* multi-line statement). Identical entries (same time, source and text) are one memory with every
|
|
243
|
+
* copy's line range, so an id never moves from one copy to another.
|
|
244
|
+
*
|
|
245
|
+
* `raw` is the entry as stored (it is the id, and the block retire and restore move); `text` is the
|
|
246
|
+
* statement as it was said: in an entry the writer marked, its escape on a continuation line is undone here
|
|
247
|
+
* (dailyNote.ts), and an entry without the mark reads back exactly as stored. A carriage return is content,
|
|
248
|
+
* not a line ending to drop.
|
|
249
|
+
*/
|
|
250
|
+
function parseEntries(rel, content) {
|
|
251
|
+
const lines = content.split('\n');
|
|
252
|
+
const starts = [];
|
|
253
|
+
// An entry starts at column 0. A look-alike behind a backslash (the writer's escape for the user's words)
|
|
254
|
+
// or behind an indent is a continuation.
|
|
255
|
+
lines.forEach((l, i) => {
|
|
256
|
+
if (dailyNote_1.ENTRY_LINE_RE.test(l))
|
|
257
|
+
starts.push(i);
|
|
258
|
+
});
|
|
259
|
+
const byRaw = new Map();
|
|
260
|
+
starts.forEach((start, k) => {
|
|
261
|
+
const end = k + 1 < starts.length ? starts[k + 1] : lines.length;
|
|
262
|
+
// Trailing blank lines are layout, not part of the entry.
|
|
263
|
+
let last = end;
|
|
264
|
+
while (last > start + 1 && BLANK_LINE_RE.test(lines[last - 1]))
|
|
265
|
+
last--;
|
|
266
|
+
const raw = lines.slice(start, last).join('\n');
|
|
267
|
+
const existing = byRaw.get(raw);
|
|
268
|
+
if (existing) {
|
|
269
|
+
existing.ranges.push([start, last]);
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
const header = (0, dailyNote_1.parseEntryHeader)(lines[start]);
|
|
273
|
+
const rest = lines.slice(start + 1, last);
|
|
274
|
+
const continuation = (header.escaped ? rest.map(dailyNote_1.decodeEntryLine) : rest).join('\n');
|
|
275
|
+
byRaw.set(raw, {
|
|
276
|
+
ranges: [[start, last]], raw, time: header.time, ...(header.offset ? { offset: header.offset } : {}), source: header.source,
|
|
277
|
+
text: continuation ? `${header.text}\n${continuation}` : header.text,
|
|
278
|
+
id: entryId(rel, raw),
|
|
279
|
+
});
|
|
280
|
+
});
|
|
281
|
+
return { lines, blocks: [...byRaw.values()] };
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* The id the map gives an entry a writer has just appended (`entry`, as appendDailyEntry wrote it) to the daily log at
|
|
285
|
+
* vault-relative `rel`. The entry is read the way the map reads the file, so the two cannot differ: it runs to the next
|
|
286
|
+
* entry (the next append) less its trailing blank lines. Null when `rel` is no daily log or `entry` no entry.
|
|
287
|
+
*/
|
|
288
|
+
function appendedEntryId(rel, entry) {
|
|
289
|
+
if (!dailyNote_1.DAILY_FILE_RE.test(rel))
|
|
290
|
+
return null;
|
|
291
|
+
return parseEntries(rel, entry).blocks[0]?.id ?? null;
|
|
292
|
+
}
|
|
293
|
+
function withoutRanges(lines, ranges) {
|
|
294
|
+
const drop = new Set();
|
|
295
|
+
for (const [a, b] of ranges)
|
|
296
|
+
for (let i = a; i < b; i++)
|
|
297
|
+
drop.add(i);
|
|
298
|
+
return lines.filter((_, i) => !drop.has(i));
|
|
299
|
+
}
|
|
300
|
+
/** The entries just before and just after a block, which parseEntries returns in file order. */
|
|
301
|
+
function entryNeighbours(blocks, block) {
|
|
302
|
+
const i = blocks.indexOf(block);
|
|
303
|
+
return {
|
|
304
|
+
...(i > 0 ? { prevId: blocks[i - 1].id } : {}),
|
|
305
|
+
...(i >= 0 && i + 1 < blocks.length ? { nextId: blocks[i + 1].id } : {}),
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* The line a restored entry goes back to, or null for the end of the file. It goes beside the neighbours it had when it
|
|
310
|
+
* was retired (the entry after it, else the one before it) while the log still agrees with time there: no entry in front
|
|
311
|
+
* of the place is later than it, and none at the place is earlier. Otherwise, before the first entry later than it.
|
|
312
|
+
*/
|
|
313
|
+
function restoreLine(blocks, record, time) {
|
|
314
|
+
const startOf = (b) => Math.min(...b.ranges.map(([a]) => a));
|
|
315
|
+
const endOf = (b) => Math.max(...b.ranges.map(([, b2]) => b2));
|
|
316
|
+
const next = blocks.find((b) => b.id === record.nextId);
|
|
317
|
+
const prev = blocks.find((b) => b.id === record.prevId);
|
|
318
|
+
const beside = next ? startOf(next) : prev ? endOf(prev) : null;
|
|
319
|
+
if (beside !== null) {
|
|
320
|
+
const before = blocks.filter((b) => startOf(b) < beside).pop();
|
|
321
|
+
const after = blocks.find((b) => startOf(b) >= beside);
|
|
322
|
+
if ((!before || before.time <= time) && (!after || after.time >= time))
|
|
323
|
+
return beside;
|
|
324
|
+
}
|
|
325
|
+
const later = blocks.find((b) => b.time > time);
|
|
326
|
+
return later ? startOf(later) : null;
|
|
327
|
+
}
|
|
328
|
+
function parseMemoryId(id) {
|
|
329
|
+
if (typeof id !== 'string')
|
|
330
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'memory id must be a string');
|
|
331
|
+
if (id.startsWith('note:'))
|
|
332
|
+
return { kind: 'note', rel: id.slice(5) };
|
|
333
|
+
if (id.startsWith('entry:')) {
|
|
334
|
+
const hashAt = id.lastIndexOf('#');
|
|
335
|
+
if (hashAt <= 6 || !/^[0-9a-f]{16}$/.test(id.slice(hashAt + 1)))
|
|
336
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'malformed entry id');
|
|
337
|
+
return { kind: 'entry', rel: id.slice(6, hashAt) };
|
|
338
|
+
}
|
|
339
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'unknown memory id kind');
|
|
340
|
+
}
|
|
341
|
+
// ------------------------------- helpers ------------------------------------
|
|
342
|
+
function collapse(text, max) {
|
|
343
|
+
const t = text.replace(/\s+/g, ' ').trim();
|
|
344
|
+
return t.length <= max ? t : `${t.slice(0, max - 1)}…`;
|
|
345
|
+
}
|
|
346
|
+
function tagsOf(frontmatter) {
|
|
347
|
+
const raw = frontmatter.tags;
|
|
348
|
+
const list = Array.isArray(raw) ? raw.map(String) : typeof raw === 'string' ? raw.split(',') : [];
|
|
349
|
+
return [...new Set(list.map((t) => t.trim().replace(/^#/, '').toLowerCase()).filter(Boolean))];
|
|
350
|
+
}
|
|
351
|
+
function escapeRe(s) {
|
|
352
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
353
|
+
}
|
|
354
|
+
/** Literal, case-insensitive, whole-word mention test. */
|
|
355
|
+
function mentionMatcher(name) {
|
|
356
|
+
const n = name.trim();
|
|
357
|
+
if (n.length < MIN_MENTION_CHARS)
|
|
358
|
+
return null;
|
|
359
|
+
return new RegExp(`(^|[^\\p{L}\\p{N}_])${escapeRe(n)}($|[^\\p{L}\\p{N}_])`, 'iu');
|
|
360
|
+
}
|
|
361
|
+
exports.NOFOLLOW = promises_1.default.constants?.O_NOFOLLOW ?? 0;
|
|
362
|
+
/** Open a confined file for reading: never through a final symlink, and only the inode at `real`. */
|
|
363
|
+
async function openConfined(real) {
|
|
364
|
+
// O_NONBLOCK: opening a FIFO for reading waits for a writer; with it the open returns and the isFile check below refuses.
|
|
365
|
+
const handle = await promises_1.default.open(real, promises_1.default.constants.O_RDONLY | promises_1.default.constants.O_NONBLOCK | exports.NOFOLLOW);
|
|
366
|
+
try {
|
|
367
|
+
const [opened, there] = await Promise.all([handle.stat(), promises_1.default.lstat(real)]);
|
|
368
|
+
if (!opened.isFile() || opened.ino !== there.ino || opened.dev !== there.dev) {
|
|
369
|
+
throw new Error('memory path changed while it was being read');
|
|
370
|
+
}
|
|
371
|
+
return handle;
|
|
372
|
+
}
|
|
373
|
+
catch (err) {
|
|
374
|
+
await handle.close();
|
|
375
|
+
throw err;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
async function readPrefix(real, bytes) {
|
|
379
|
+
let handle;
|
|
380
|
+
try {
|
|
381
|
+
handle = await openConfined(real);
|
|
382
|
+
}
|
|
383
|
+
catch {
|
|
384
|
+
return null;
|
|
385
|
+
}
|
|
386
|
+
try {
|
|
387
|
+
const size = (await handle.stat()).size;
|
|
388
|
+
const buf = Buffer.alloc(Math.min(size, bytes));
|
|
389
|
+
const { bytesRead } = await handle.read(buf, 0, buf.length, 0);
|
|
390
|
+
return { text: buf.subarray(0, bytesRead).toString('utf8'), truncated: size > bytes };
|
|
391
|
+
}
|
|
392
|
+
finally {
|
|
393
|
+
await handle.close();
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
/** Whole-file read for a rewrite; refuses rather than silently truncating. */
|
|
397
|
+
async function readWhole(real) {
|
|
398
|
+
const handle = await openConfined(real);
|
|
399
|
+
try {
|
|
400
|
+
if ((await handle.stat()).size > MAX_REWRITE_BYTES) {
|
|
401
|
+
throw new Error('this daily log is too large to edit safely; nothing was changed');
|
|
402
|
+
}
|
|
403
|
+
return await handle.readFile('utf8');
|
|
404
|
+
}
|
|
405
|
+
finally {
|
|
406
|
+
await handle.close();
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
/**
|
|
410
|
+
* A note file read in full so the secret guard (RENOR_SECRET_GUARD) can screen it before it is moved. A file too
|
|
411
|
+
* large to hold is refused, as the guard does whenever it cannot check.
|
|
412
|
+
*/
|
|
413
|
+
async function readToScreen(real, action) {
|
|
414
|
+
const handle = await openConfined(real);
|
|
415
|
+
try {
|
|
416
|
+
if ((await handle.stat()).size > MAX_REWRITE_BYTES)
|
|
417
|
+
(0, secretGuard_1.refuseUnchecked)('memory', action);
|
|
418
|
+
return await handle.readFile('utf8');
|
|
419
|
+
}
|
|
420
|
+
finally {
|
|
421
|
+
await handle.close();
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
/**
|
|
425
|
+
* Make sure a vault-relative directory exists as real directories all the way down (creating what
|
|
426
|
+
* is missing) and return its real path. Refuses a symlink at any level, so neither the journal nor
|
|
427
|
+
* a retired file can be written outside the vault through one.
|
|
428
|
+
*/
|
|
429
|
+
async function ensureRealDir(rootReal, relDir) {
|
|
430
|
+
let cursor = rootReal;
|
|
431
|
+
for (const part of relDir.split(/[\\/]/).filter((p) => p && p !== '.')) {
|
|
432
|
+
cursor = path_1.default.join(cursor, part);
|
|
433
|
+
try {
|
|
434
|
+
await promises_1.default.mkdir(cursor, { mode: 0o700 });
|
|
435
|
+
}
|
|
436
|
+
catch (err) {
|
|
437
|
+
if (err.code !== 'EEXIST')
|
|
438
|
+
throw err;
|
|
439
|
+
}
|
|
440
|
+
const st = await promises_1.default.lstat(cursor);
|
|
441
|
+
if (st.isSymbolicLink() || !st.isDirectory())
|
|
442
|
+
throw new Error(`refusing to use ${relDir}: it is not a plain directory inside the vault`);
|
|
443
|
+
}
|
|
444
|
+
return cursor;
|
|
445
|
+
}
|
|
446
|
+
/** Replace a file's content atomically: a crash leaves either the old or the new bytes, never a mix. */
|
|
447
|
+
async function atomicWrite(abs, content) {
|
|
448
|
+
const tmp = `${abs}.${process.pid}.${crypto_1.default.randomBytes(4).toString('hex')}.tmp`;
|
|
449
|
+
let mode;
|
|
450
|
+
try {
|
|
451
|
+
mode = (await promises_1.default.stat(abs)).mode & 0o777;
|
|
452
|
+
}
|
|
453
|
+
catch {
|
|
454
|
+
mode = undefined;
|
|
455
|
+
}
|
|
456
|
+
const handle = await promises_1.default.open(tmp, 'wx', mode ?? 0o600);
|
|
457
|
+
try {
|
|
458
|
+
await handle.writeFile(content, 'utf8');
|
|
459
|
+
await handle.sync();
|
|
460
|
+
}
|
|
461
|
+
finally {
|
|
462
|
+
await handle.close();
|
|
463
|
+
}
|
|
464
|
+
try {
|
|
465
|
+
await promises_1.default.rename(tmp, abs);
|
|
466
|
+
}
|
|
467
|
+
catch (err) {
|
|
468
|
+
await promises_1.default.rm(tmp, { force: true });
|
|
469
|
+
throw err;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
async function readJournal(ctx) {
|
|
473
|
+
const rootReal = await promises_1.default.realpath(ctx.memoryDir);
|
|
474
|
+
let raw;
|
|
475
|
+
try {
|
|
476
|
+
// Confined like every other read: never through a symlinked .trash or journal.
|
|
477
|
+
for (const dir of [path_1.default.join(rootReal, '.trash'), path_1.default.join(rootReal, exports.RETIRED_DIR)]) {
|
|
478
|
+
const st = await promises_1.default.lstat(dir);
|
|
479
|
+
if (st.isSymbolicLink() || !st.isDirectory())
|
|
480
|
+
throw new Error('the retired-memory journal is not inside the vault');
|
|
481
|
+
}
|
|
482
|
+
const handle = await openConfined(path_1.default.join(rootReal, RETIRED_INDEX));
|
|
483
|
+
try {
|
|
484
|
+
raw = await handle.readFile('utf8');
|
|
485
|
+
}
|
|
486
|
+
finally {
|
|
487
|
+
await handle.close();
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
catch (err) {
|
|
491
|
+
if (err.code === 'ENOENT')
|
|
492
|
+
return [];
|
|
493
|
+
throw err;
|
|
494
|
+
}
|
|
495
|
+
const out = [];
|
|
496
|
+
for (const line of raw.split('\n')) {
|
|
497
|
+
if (!line.trim())
|
|
498
|
+
continue;
|
|
499
|
+
try {
|
|
500
|
+
out.push(JSON.parse(line));
|
|
501
|
+
}
|
|
502
|
+
catch {
|
|
503
|
+
// A torn final line (crash mid-append) is skipped, never fatal.
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
return out;
|
|
507
|
+
}
|
|
508
|
+
/** Retirements still in effect (the latest record per retirement wins; a restore or a void ends it). */
|
|
509
|
+
function activeRetired(records) {
|
|
510
|
+
const latest = new Map();
|
|
511
|
+
for (const r of records)
|
|
512
|
+
if (typeof r.rid === 'string')
|
|
513
|
+
latest.set(r.rid, r);
|
|
514
|
+
return [...latest.values()].filter((r) => !r.restoredAt && !r.voided && !r.purgedAt);
|
|
515
|
+
}
|
|
516
|
+
async function appendJournal(ctx, record) {
|
|
517
|
+
const dir = await ensureRealDir(await promises_1.default.realpath(ctx.memoryDir), exports.RETIRED_DIR);
|
|
518
|
+
const file = path_1.default.join(dir, path_1.default.basename(RETIRED_INDEX));
|
|
519
|
+
// O_NONBLOCK: a FIFO planted under the journal's name is an error here, never a wait.
|
|
520
|
+
const handle = await promises_1.default.open(file, promises_1.default.constants.O_RDWR | promises_1.default.constants.O_APPEND | promises_1.default.constants.O_CREAT | promises_1.default.constants.O_NONBLOCK | exports.NOFOLLOW, 0o600);
|
|
521
|
+
try {
|
|
522
|
+
const stat = await handle.stat();
|
|
523
|
+
if (!stat.isFile() || stat.nlink !== 1)
|
|
524
|
+
throw new Error(`refusing to write ${JSON.stringify(RETIRED_INDEX)}: it is not a regular file with a single name`);
|
|
525
|
+
// A crash can leave a torn last line; start on a fresh line so this record is never glued to it.
|
|
526
|
+
const size = stat.size;
|
|
527
|
+
let prefix = '';
|
|
528
|
+
if (size > 0) {
|
|
529
|
+
const last = Buffer.alloc(1);
|
|
530
|
+
await handle.read(last, 0, 1, size - 1);
|
|
531
|
+
if (last[0] !== 0x0a)
|
|
532
|
+
prefix = '\n';
|
|
533
|
+
}
|
|
534
|
+
await handle.appendFile(prefix + JSON.stringify(record) + '\n', 'utf8');
|
|
535
|
+
await handle.sync();
|
|
536
|
+
}
|
|
537
|
+
finally {
|
|
538
|
+
await handle.close();
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
/** Retire/restore events carry the memory id and its title as JSON in `summary`. */
|
|
542
|
+
function recordEvent(db, rel, action, id, title) {
|
|
543
|
+
db.prepare('INSERT INTO write_events (note_path, action, summary, created_at) VALUES (?, ?, ?, ?)')
|
|
544
|
+
.run(rel, action, JSON.stringify({ id, title }), new Date().toISOString());
|
|
545
|
+
}
|
|
546
|
+
function parseEventRef(summary) {
|
|
547
|
+
try {
|
|
548
|
+
const v = JSON.parse(summary ?? '');
|
|
549
|
+
return { id: typeof v.id === 'string' ? v.id : null, title: typeof v.title === 'string' ? v.title : null };
|
|
550
|
+
}
|
|
551
|
+
catch {
|
|
552
|
+
return { id: null, title: summary };
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
async function exists(abs) {
|
|
556
|
+
try {
|
|
557
|
+
await promises_1.default.lstat(abs);
|
|
558
|
+
return true;
|
|
559
|
+
}
|
|
560
|
+
catch {
|
|
561
|
+
return false;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
/**
|
|
565
|
+
* Whether a daily log is too large to be read whole, so a read of it would be cut and the map marked truncated. A note
|
|
566
|
+
* a lookup does not read is still counted by this, through the same confinement a read applies.
|
|
567
|
+
*/
|
|
568
|
+
async function readsWhole(ctx, stored) {
|
|
569
|
+
// Most daily logs are far below the size, so the confinement check runs only for the few that could be cut.
|
|
570
|
+
try {
|
|
571
|
+
if ((0, fs_1.statSync)(path_1.default.resolve(ctx.memoryDir, stored)).size <= MAX_REWRITE_BYTES)
|
|
572
|
+
return false;
|
|
573
|
+
}
|
|
574
|
+
catch {
|
|
575
|
+
return false;
|
|
576
|
+
}
|
|
577
|
+
try {
|
|
578
|
+
const file = await resolveVaultFile(ctx, stored, true);
|
|
579
|
+
return (await promises_1.default.stat(file.real)).size > MAX_REWRITE_BYTES;
|
|
580
|
+
}
|
|
581
|
+
catch {
|
|
582
|
+
return false;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
/**
|
|
586
|
+
* Whether a note's file has changed since it was indexed: its text may then hold words the index does not. A synchronous
|
|
587
|
+
* stat: one per note, and a queued async call per note cost about 2 s on a loaded host where the stats themselves take 50 ms.
|
|
588
|
+
*/
|
|
589
|
+
function changedSinceIndexed(memoryDir, n) {
|
|
590
|
+
try {
|
|
591
|
+
return (0, fs_1.statSync)(path_1.default.resolve(memoryDir, n.path)).mtime.toISOString() !== n.modified_at;
|
|
592
|
+
}
|
|
593
|
+
catch {
|
|
594
|
+
return true;
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
/**
|
|
598
|
+
* The notes with a chunk the predicate accepts, read from the index's chunk text. A note's chunks are its body cut at line
|
|
599
|
+
* breaks (a word never spans one), so a word the note's body holds is in one chunk, at the same place in its line. It
|
|
600
|
+
* can therefore only be a candidate here, never a verdict: keep still decides on the file's own text.
|
|
601
|
+
*/
|
|
602
|
+
function notesWithIndexedText(db, hasWord) {
|
|
603
|
+
const out = new Set();
|
|
604
|
+
const rows = db.prepare('SELECT note_path, text FROM chunks').iterate();
|
|
605
|
+
for (const row of rows) {
|
|
606
|
+
if (!out.has(row.note_path) && hasWord(row.text))
|
|
607
|
+
out.add(row.note_path);
|
|
608
|
+
}
|
|
609
|
+
return out;
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* The daily logs that hold a retired entry. Their entries are read whole, because a retirement counts only once its entry
|
|
613
|
+
* is gone from its log.
|
|
614
|
+
*/
|
|
615
|
+
async function retiredEntryFiles(ctx) {
|
|
616
|
+
return new Set(activeRetired(await readJournal(ctx)).filter((r) => r.kind === 'entry').map((r) => r.path));
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* Every memory a person can open, newest first: each note file and each entry of a daily log. With `keep`, only the
|
|
620
|
+
* memories it keeps are collected, the cap counts those alone, and a daily log's every entry is considered rather than
|
|
621
|
+
* its newest ones, so a lookup reaches any memory the scan reads.
|
|
622
|
+
*/
|
|
623
|
+
async function collectMemories(ctx, keep, mayMatch) {
|
|
624
|
+
const { db, memoryDir } = ctx;
|
|
625
|
+
const notes = db.prepare(`SELECT path, title, modified_at, frontmatter_json FROM notes ORDER BY modified_at DESC LIMIT ?`).all(MAX_NOTES_SCANNED + 1);
|
|
626
|
+
const memories = [];
|
|
627
|
+
const text = new Map();
|
|
628
|
+
const wikilinks = new Map();
|
|
629
|
+
const seen = new Set();
|
|
630
|
+
const bound = (0, orchestrator_1.createNoteTimeBound)(db);
|
|
631
|
+
let truncated = notes.length > MAX_NOTES_SCANNED;
|
|
632
|
+
const add = (memory, said) => {
|
|
633
|
+
seen.add(memory.id);
|
|
634
|
+
if (keep && !keep(memory, said))
|
|
635
|
+
return false;
|
|
636
|
+
memories.push(memory);
|
|
637
|
+
text.set(memory.id, said);
|
|
638
|
+
return true;
|
|
639
|
+
};
|
|
640
|
+
for (const n of notes.slice(0, MAX_NOTES_SCANNED)) {
|
|
641
|
+
if (memories.length >= MAX_MEMORIES) {
|
|
642
|
+
truncated = true;
|
|
643
|
+
break;
|
|
644
|
+
}
|
|
645
|
+
if (n.path === LIFECYCLE_NOTE_PATH || isLedgerIndexed(ctx, n.path))
|
|
646
|
+
continue;
|
|
647
|
+
const stored = toRel(memoryDir, n.path);
|
|
648
|
+
if (!stored)
|
|
649
|
+
continue;
|
|
650
|
+
// A note that cannot match is not read at all: keep would refuse every memory in it.
|
|
651
|
+
if (mayMatch && !(await mayMatch(n))) {
|
|
652
|
+
if (dailyNote_1.DAILY_FILE_RE.test(stored) && (await readsWhole(ctx, stored)))
|
|
653
|
+
truncated = true;
|
|
654
|
+
continue;
|
|
655
|
+
}
|
|
656
|
+
// The same confinement as read/retire: a note reached through a symlink, a hidden directory or
|
|
657
|
+
// an alias of the ledger is not listed.
|
|
658
|
+
let file;
|
|
659
|
+
try {
|
|
660
|
+
file = await resolveVaultFile(ctx, stored, true);
|
|
661
|
+
}
|
|
662
|
+
catch {
|
|
663
|
+
continue;
|
|
664
|
+
}
|
|
665
|
+
const rel = file.rel;
|
|
666
|
+
const daily = dailyNote_1.DAILY_FILE_RE.exec(rel);
|
|
667
|
+
const read = await readPrefix(file.real, daily ? MAX_REWRITE_BYTES : MAP_READ_BYTES);
|
|
668
|
+
if (read === null)
|
|
669
|
+
continue; // indexed but gone from disk: not a memory a person can open
|
|
670
|
+
let fm = {};
|
|
671
|
+
try {
|
|
672
|
+
fm = JSON.parse(n.frontmatter_json);
|
|
673
|
+
}
|
|
674
|
+
catch { /* keep {} */ }
|
|
675
|
+
if (daily) {
|
|
676
|
+
const { blocks } = parseEntries(rel, read.text);
|
|
677
|
+
const dated = blocks.map((b) => ({ b, at: entryAt(bound, n.path, daily[2], b, n.modified_at) }));
|
|
678
|
+
const newest = keep ? dated : [...dated].sort((x, y) => y.at.localeCompare(x.at)).slice(0, MAX_ENTRIES_PER_DAY);
|
|
679
|
+
if (newest.length < blocks.length)
|
|
680
|
+
truncated = true;
|
|
681
|
+
for (const { b, at } of newest) {
|
|
682
|
+
add({
|
|
683
|
+
id: b.id, kind: 'entry', path: rel,
|
|
684
|
+
title: collapse(b.text, 90),
|
|
685
|
+
at,
|
|
686
|
+
source: b.source || null,
|
|
687
|
+
tags: [], status: null,
|
|
688
|
+
excerpt: collapse(b.text, EXCERPT_CHARS),
|
|
689
|
+
}, b.text);
|
|
690
|
+
}
|
|
691
|
+
if (read.truncated)
|
|
692
|
+
truncated = true;
|
|
693
|
+
continue;
|
|
694
|
+
}
|
|
695
|
+
const parsed = (0, parser_1.parseNote)(n.path, read.text);
|
|
696
|
+
const id = `note:${rel}`;
|
|
697
|
+
const kept = add({
|
|
698
|
+
id, kind: 'note', path: rel,
|
|
699
|
+
title: n.title,
|
|
700
|
+
at: n.modified_at,
|
|
701
|
+
source: typeof fm.source === 'string' ? fm.source : null,
|
|
702
|
+
tags: tagsOf(fm),
|
|
703
|
+
status: typeof fm.status === 'string' ? fm.status : null,
|
|
704
|
+
excerpt: collapse(parsed.body.replace(/^#\s+.*$/m, ''), EXCERPT_CHARS),
|
|
705
|
+
}, `${n.title}\n${parsed.body}`);
|
|
706
|
+
if (kept)
|
|
707
|
+
wikilinks.set(id, parsed.wikilinks);
|
|
708
|
+
}
|
|
709
|
+
memories.sort((a, b) => b.at.localeCompare(a.at));
|
|
710
|
+
return { memories, truncated, text, wikilinks, seen };
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* Retirements that completed, newest first (a note's file is in the retired area; an entry is no longer among `live`).
|
|
714
|
+
* An interrupted one is still a live memory and is not listed twice.
|
|
715
|
+
*/
|
|
716
|
+
async function completedRetirements(ctx, live) {
|
|
717
|
+
const rootReal = await promises_1.default.realpath(ctx.memoryDir);
|
|
718
|
+
const out = [];
|
|
719
|
+
for (const r of activeRetired(await readJournal(ctx)).reverse()) {
|
|
720
|
+
const completed = r.kind === 'entry'
|
|
721
|
+
? !live.has(r.id)
|
|
722
|
+
: Boolean(r.storedAt) && (await exists(path_1.default.join(rootReal, ...r.storedAt.split('/'))));
|
|
723
|
+
if (!completed)
|
|
724
|
+
continue;
|
|
725
|
+
const day = dailyNote_1.DAILY_FILE_RE.exec(r.path)?.[2];
|
|
726
|
+
const header = r.block ? (0, dailyNote_1.parseEntryHeader)(r.block.split('\n')[0]) : null;
|
|
727
|
+
out.push({
|
|
728
|
+
memory: {
|
|
729
|
+
id: r.rid, memoryId: r.id, kind: r.kind, path: r.path, title: r.title, retiredAt: r.retiredAt,
|
|
730
|
+
// A retired entry's file is gone, so no note row bounds it; it was saved before it was retired, so that time caps it.
|
|
731
|
+
...(r.kind === 'entry' && day && header ? { savedAt: (0, orchestrator_1.clampSessionDate)((0, dailyNote_1.entryInstant)(day, header), r.retiredAt) ?? r.retiredAt } : {}),
|
|
732
|
+
},
|
|
733
|
+
text: r.block ?? '',
|
|
734
|
+
...(r.kind === 'note' && r.storedAt ? { storedAt: r.storedAt } : {}),
|
|
735
|
+
});
|
|
736
|
+
}
|
|
737
|
+
const facts = (0, lifecycle_1.listRetiredFacts)(ctx.db).map((f) => ({
|
|
738
|
+
memory: { id: f.id, memoryId: f.id, kind: 'fact', path: ctx.ledgerRelPath, title: f.title, retiredAt: f.retiredAt },
|
|
739
|
+
text: f.title,
|
|
740
|
+
scope: f.scope,
|
|
741
|
+
}));
|
|
742
|
+
// Both lists are newest first; the sort is stable.
|
|
743
|
+
return [...out, ...facts].sort((a, b) => b.memory.retiredAt.localeCompare(a.memory.retiredAt));
|
|
744
|
+
}
|
|
745
|
+
/**
|
|
746
|
+
* The body of a retired note's stored copy, for a lookup to match. Only a copy inside the retired area, reached through
|
|
747
|
+
* real directories, is read (the checks restore makes before it moves the copy back), and as much of it as the map
|
|
748
|
+
* reads of a live note.
|
|
749
|
+
*/
|
|
750
|
+
async function retiredNoteBody(rootReal, storedAt) {
|
|
751
|
+
const stored = path_1.default.join(rootReal, ...storedAt.split('/'));
|
|
752
|
+
if (!within(path_1.default.join(rootReal, RETIRED_FILES_DIR), stored))
|
|
753
|
+
return '';
|
|
754
|
+
try {
|
|
755
|
+
if ((await promises_1.default.realpath(path_1.default.dirname(stored))) !== path_1.default.dirname(stored))
|
|
756
|
+
return '';
|
|
757
|
+
}
|
|
758
|
+
catch {
|
|
759
|
+
return '';
|
|
760
|
+
}
|
|
761
|
+
const read = await readPrefix(stored, MAP_READ_BYTES);
|
|
762
|
+
return read === null ? '' : (0, parser_1.parseNote)(stored, read.text).body;
|
|
763
|
+
}
|
|
764
|
+
async function buildMemoryMap(ctx, options = {}) {
|
|
765
|
+
const { db } = ctx;
|
|
766
|
+
const cap = options.limit ?? exports.DEFAULT_MAP_LIMIT;
|
|
767
|
+
const collected = await collectMemories(ctx);
|
|
768
|
+
const memories = collected.memories;
|
|
769
|
+
const mentionText = collected.text;
|
|
770
|
+
const wikilinksOf = collected.wikilinks;
|
|
771
|
+
const shown = memories.slice(0, cap);
|
|
772
|
+
const truncatedMemories = collected.truncated || memories.length > cap;
|
|
773
|
+
// Wikilink targets: an exact vault path wins; a bare name or title links only when unambiguous across the whole vault.
|
|
774
|
+
const byPath = new Map();
|
|
775
|
+
const byName = new Map();
|
|
776
|
+
for (const m of memories) {
|
|
777
|
+
if (m.kind !== 'note')
|
|
778
|
+
continue;
|
|
779
|
+
byPath.set(m.path.replace(/\.md$/i, '').toLowerCase(), m.id);
|
|
780
|
+
for (const key of new Set([path_1.default.posix.basename(m.path, '.md').toLowerCase(), m.title.toLowerCase()])) {
|
|
781
|
+
byName.set(key, [...(byName.get(key) ?? []), m.id]);
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
const resolveWikilink = (target) => {
|
|
785
|
+
const t = target.trim().replace(/\.md$/i, '').toLowerCase();
|
|
786
|
+
if (byPath.has(t))
|
|
787
|
+
return byPath.get(t);
|
|
788
|
+
if (t.includes('/'))
|
|
789
|
+
return null; // a path that names no note is not guessed from its basename
|
|
790
|
+
const candidates = byName.get(t) ?? [];
|
|
791
|
+
return candidates.length === 1 ? candidates[0] : null;
|
|
792
|
+
};
|
|
793
|
+
// Things: the subjects of lifecycle (ledger) facts, by their newest fact in the same order the facts list uses. So the
|
|
794
|
+
// newest `limit` facts always name things the reply lists: a thing with a newer fact ranks above it.
|
|
795
|
+
const thingRows = db.prepare(`SELECT e.id, e.canonical_name AS name,
|
|
796
|
+
SUM(CASE WHEN f.valid_to IS NULL THEN 1 ELSE 0 END) AS current,
|
|
797
|
+
(SELECT g.valid_from FROM facts g WHERE g.subject_entity_id = e.id AND g.source_path = ?
|
|
798
|
+
ORDER BY g.valid_from DESC, g.id DESC LIMIT 1) AS newest_from,
|
|
799
|
+
(SELECT g.id FROM facts g WHERE g.subject_entity_id = e.id AND g.source_path = ?
|
|
800
|
+
ORDER BY g.valid_from DESC, g.id DESC LIMIT 1) AS newest_id
|
|
801
|
+
FROM entities e JOIN facts f ON f.subject_entity_id = e.id AND f.source_path = ?
|
|
802
|
+
WHERE e.source_note = ?
|
|
803
|
+
GROUP BY e.id ORDER BY newest_from DESC, newest_id DESC, e.canonical_name LIMIT ?`).all(LIFECYCLE_NOTE_PATH, LIFECYCLE_NOTE_PATH, LIFECYCLE_NOTE_PATH, LIFECYCLE_NOTE_PATH, MAX_THINGS + 1);
|
|
804
|
+
const truncatedThings = thingRows.length > MAX_THINGS;
|
|
805
|
+
const allThings = thingRows.slice(0, MAX_THINGS).map((t) => ({ id: `thing:${t.id}`, name: t.name, currentFacts: t.current }));
|
|
806
|
+
const things = allThings.slice(0, cap);
|
|
807
|
+
// Facts: every lifecycle fact of a ledger thing, newest first. Not cut to the listed things: each fact names its subject.
|
|
808
|
+
const factRows = db.prepare(`SELECT f.id, f.subject_entity_id AS sid, e.canonical_name AS subject, f.predicate, f.object_text AS object,
|
|
809
|
+
f.valid_from, f.valid_to, f.scope, f.event_id
|
|
810
|
+
FROM facts f JOIN entities e ON e.id = f.subject_entity_id
|
|
811
|
+
WHERE f.source_path = ? AND e.source_note = ?
|
|
812
|
+
ORDER BY f.valid_from DESC, f.id DESC LIMIT ?`).all(LIFECYCLE_NOTE_PATH, LIFECYCLE_NOTE_PATH, MAX_FACTS + 1);
|
|
813
|
+
const allFacts = factRows.slice(0, MAX_FACTS).map((f) => ({
|
|
814
|
+
id: f.id, thingId: `thing:${f.sid}`, subject: f.subject, predicate: f.predicate,
|
|
815
|
+
object: f.object ?? '', validFrom: f.valid_from, validTo: f.valid_to, scope: f.scope, eventId: f.event_id,
|
|
816
|
+
}));
|
|
817
|
+
// A fact's object is shown as an excerpt (the link search below reads the whole object): a map is for browsing, and
|
|
818
|
+
// get_context returns the fact in full. A current fact that replaced another writer's value carries its conflict line.
|
|
819
|
+
const disputes = (0, writerConflicts_1.writerConflictLines)(db, allFacts.slice(0, cap).map((f) => (f.validTo === null ? f.eventId : null)));
|
|
820
|
+
const facts = allFacts.slice(0, cap).map((f) => {
|
|
821
|
+
const line = f.validTo === null && f.eventId !== null ? disputes.get(f.eventId) : undefined;
|
|
822
|
+
return { ...f, object: collapse(f.object, EXCERPT_CHARS), ...(line === undefined ? {} : { writerConflict: line }) };
|
|
823
|
+
});
|
|
824
|
+
const truncatedFacts = factRows.length > MAX_FACTS || allFacts.length > cap;
|
|
825
|
+
// Links over the whole map, then only those between what the reply lists (no link points at a missing node). Each kind
|
|
826
|
+
// has its own bucket and its own build cap, so a vault with many tags cannot starve the mention and fact links; the
|
|
827
|
+
// total counts every link the map holds, so `truncated` says more exist even when the listed nodes have few.
|
|
828
|
+
// `seq` is the order the links were found in: the reply lists the links it keeps in that order.
|
|
829
|
+
const linksByKind = { tag: [], wikilink: [], mention: [], fact: [] };
|
|
830
|
+
let linkSeq = 0;
|
|
831
|
+
let truncatedLinks = false;
|
|
832
|
+
const push = (l) => {
|
|
833
|
+
const bucket = linksByKind[l.kind];
|
|
834
|
+
if (bucket.length >= MAX_LINKS) {
|
|
835
|
+
truncatedLinks = true;
|
|
836
|
+
return false;
|
|
837
|
+
}
|
|
838
|
+
bucket.push({ seq: linkSeq++, link: l });
|
|
839
|
+
return true;
|
|
840
|
+
};
|
|
841
|
+
const topicCount = new Map();
|
|
842
|
+
for (const m of shown) {
|
|
843
|
+
for (const t of m.tags)
|
|
844
|
+
topicCount.set(t, (topicCount.get(t) ?? 0) + 1);
|
|
845
|
+
}
|
|
846
|
+
for (const m of memories) {
|
|
847
|
+
for (const t of m.tags)
|
|
848
|
+
push({ source: m.id, target: `topic:${t}`, kind: 'tag' });
|
|
849
|
+
for (const w of wikilinksOf.get(m.id) ?? []) {
|
|
850
|
+
const target = resolveWikilink(w);
|
|
851
|
+
if (target && target !== m.id)
|
|
852
|
+
push({ source: m.id, target, kind: 'wikilink' });
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
const matchers = allThings
|
|
856
|
+
.map((t) => ({
|
|
857
|
+
id: t.id, re: mentionMatcher(t.name),
|
|
858
|
+
// For ASCII names, absence of the folded literal proves absence of a /iu match.
|
|
859
|
+
// Non-ASCII names still use the regex alone (Unicode folding is not lowercasing).
|
|
860
|
+
literal: /^[\x00-\x7f]+$/.test(t.name.trim()) ? t.name.trim().toLowerCase() : null,
|
|
861
|
+
}))
|
|
862
|
+
.filter((t) => t.re !== null);
|
|
863
|
+
const hasLiteral = matchers.some((t) => t.literal !== null);
|
|
864
|
+
// Long s and Kelvin sign are the only non-ASCII letters /iu folds to ASCII. Lowercasing
|
|
865
|
+
// handles Kelvin; long s needs an explicit fold. The regex still decides boundaries.
|
|
866
|
+
const fold = (text) => hasLiteral ? text.toLowerCase().replace(/\u017f/g, 's') : '';
|
|
867
|
+
outer: for (const m of memories) {
|
|
868
|
+
const text = mentionText.get(m.id) ?? '';
|
|
869
|
+
const folded = fold(text);
|
|
870
|
+
for (const t of matchers) {
|
|
871
|
+
if (t.literal !== null && !folded.includes(t.literal))
|
|
872
|
+
continue;
|
|
873
|
+
if (t.re.test(text) && !push({ source: m.id, target: t.id, kind: 'mention' }))
|
|
874
|
+
break outer;
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
const seenFactLink = new Set();
|
|
878
|
+
outerFacts: for (const f of allFacts) {
|
|
879
|
+
if (f.validTo !== null)
|
|
880
|
+
continue;
|
|
881
|
+
const folded = fold(f.object);
|
|
882
|
+
for (const t of matchers) {
|
|
883
|
+
if (t.id === f.thingId || (t.literal !== null && !folded.includes(t.literal)) || !t.re.test(f.object))
|
|
884
|
+
continue;
|
|
885
|
+
const key = `${f.thingId}|${t.id}|${f.predicate}`;
|
|
886
|
+
if (seenFactLink.has(key))
|
|
887
|
+
continue;
|
|
888
|
+
seenFactLink.add(key);
|
|
889
|
+
if (!push({ source: f.thingId, target: t.id, kind: 'fact', label: f.predicate }))
|
|
890
|
+
break outerFacts;
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
const topics = [...topicCount.entries()]
|
|
894
|
+
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
895
|
+
.map(([name, count]) => ({ id: `topic:${name}`, name, memories: count }));
|
|
896
|
+
// Events: what the engine itself recorded, newest first. The save of a fact that is retired now is left out, as the
|
|
897
|
+
// fact is: its summary names the value, at the fact's own time.
|
|
898
|
+
const eventRows = db.prepare(`SELECT note_path, action, summary, created_at, author FROM write_events w
|
|
899
|
+
WHERE NOT (w.note_path = '_memory_ledger_' AND w.action IN ('store', 'supersede') AND EXISTS (
|
|
900
|
+
SELECT 1 FROM retired_facts r WHERE r.valid_from = w.created_at AND instr(w.summary, ' ' || r.predicate || ' → ' || r.object_text) > 0))
|
|
901
|
+
ORDER BY created_at DESC, id DESC LIMIT ?`).all(MAX_EVENTS + 1);
|
|
902
|
+
const allEvents = eventRows.slice(0, MAX_EVENTS).map((e) => {
|
|
903
|
+
if (e.action === 'retire' || e.action === 'restore') {
|
|
904
|
+
const ref = parseEventRef(e.summary);
|
|
905
|
+
return { at: e.created_at, action: e.action, memoryId: ref.id, summary: ref.title, ...(e.author ? { source: e.author } : {}) };
|
|
906
|
+
}
|
|
907
|
+
return { at: e.created_at, action: e.action, memoryId: null, summary: e.summary, ...(e.author ? { source: e.author } : {}) };
|
|
908
|
+
});
|
|
909
|
+
const events = allEvents.slice(0, cap).map((e) => ({ ...e, summary: e.summary === null ? null : collapse(e.summary, EXCERPT_CHARS) }));
|
|
910
|
+
// Retired: only retirements that completed, newest first.
|
|
911
|
+
const retiredAll = (await completedRetirements(ctx, collected.seen)).map((r) => r.memory);
|
|
912
|
+
const retired = retiredAll.slice(0, Math.min(MAX_RETIRED_LISTED, cap));
|
|
913
|
+
// A link joins two nodes the reply lists: a memory or a topic of a listed memory, or a listed thing.
|
|
914
|
+
const listed = new Set([...shown.map((m) => m.id), ...things.map((t) => t.id), ...topics.map((t) => t.id)]);
|
|
915
|
+
// The reply's links are shared out across the kinds, one each in turn, so every kind that exists keeps its share of
|
|
916
|
+
// `limit` (a kind with fewer links gives its turns to the others). The reply never holds more than `limit` links.
|
|
917
|
+
const kindOrder = ['tag', 'wikilink', 'mention', 'fact'];
|
|
918
|
+
const shownByKind = kindOrder.map((k) => linksByKind[k].filter(({ link: l }) => listed.has(l.source) && listed.has(l.target)));
|
|
919
|
+
const kept = [];
|
|
920
|
+
for (let turn = 0; kept.length < cap; turn++) {
|
|
921
|
+
const picks = shownByKind.flatMap((bucket) => (turn < bucket.length ? [bucket[turn]] : []));
|
|
922
|
+
if (picks.length === 0)
|
|
923
|
+
break;
|
|
924
|
+
kept.push(...picks.slice(0, cap - kept.length));
|
|
925
|
+
}
|
|
926
|
+
const links = kept.sort((a, b) => a.seq - b.seq).map((k) => k.link);
|
|
927
|
+
const totalLinks = kindOrder.reduce((sum, k) => sum + linksByKind[k].length, 0);
|
|
928
|
+
return {
|
|
929
|
+
ok: true,
|
|
930
|
+
generatedAt: new Date().toISOString(),
|
|
931
|
+
limit: cap,
|
|
932
|
+
memories: shown,
|
|
933
|
+
things,
|
|
934
|
+
topics,
|
|
935
|
+
facts,
|
|
936
|
+
links,
|
|
937
|
+
events,
|
|
938
|
+
retired,
|
|
939
|
+
total: {
|
|
940
|
+
memories: memories.length,
|
|
941
|
+
things: allThings.length,
|
|
942
|
+
facts: allFacts.length,
|
|
943
|
+
links: totalLinks,
|
|
944
|
+
events: allEvents.length,
|
|
945
|
+
retired: retiredAll.length,
|
|
946
|
+
},
|
|
947
|
+
truncated: {
|
|
948
|
+
memories: truncatedMemories,
|
|
949
|
+
things: truncatedThings || allThings.length > cap,
|
|
950
|
+
facts: truncatedFacts,
|
|
951
|
+
links: truncatedLinks || totalLinks > links.length,
|
|
952
|
+
events: eventRows.length > MAX_EVENTS || allEvents.length > cap,
|
|
953
|
+
retired: retiredAll.length > retired.length,
|
|
954
|
+
},
|
|
955
|
+
};
|
|
956
|
+
}
|
|
957
|
+
/**
|
|
958
|
+
* The memories a few words find, so one can be read, retired or restored without loading the whole map: every memory
|
|
959
|
+
* whose path, title or text has a word starting with one of `terms` (lowercase words, as `atoms` gives them), and every
|
|
960
|
+
* retired memory whose path, title or stored text has one (an entry's text from the journal, a note's from its copy in
|
|
961
|
+
* the retired area). A term matches at the start of a word, so "rent" finds
|
|
962
|
+
* "rented" but not "current". A term counts for more the fewer memories have it, so the memory that has the rare word
|
|
963
|
+
* of "the note about my landlord" comes before the many that only say "note"; equal ones stay newest first. At most
|
|
964
|
+
* `limit` of each. With `scope`, only the memories in that scope (by path, the rule every note read follows) are looked
|
|
965
|
+
* at, before the cap: the lookup used to look everywhere, and finding one memory to forget cost a lookup of the whole
|
|
966
|
+
* workspace (live dogfood, 2026-10-03, D12).
|
|
967
|
+
*/
|
|
968
|
+
async function findMemories(ctx, args) {
|
|
969
|
+
const inScope = (rel) => args.scope === undefined || (0, scope_1.notePathMatchesScope)(rel, args.scope);
|
|
970
|
+
const patterns = [...new Set(args.terms)].map((t) => new RegExp(`(?<![\\p{L}\\p{N}\\p{M}])${escapeRe(t)}`, 'u'));
|
|
971
|
+
const termsIn = (...parts) => {
|
|
972
|
+
const text = parts.join('\n').toLowerCase();
|
|
973
|
+
return patterns.flatMap((re, i) => (re.test(text) ? [i] : []));
|
|
974
|
+
};
|
|
975
|
+
/** Each found item first by the summed rarity of the terms it has; the sort is stable, so the given order breaks ties. */
|
|
976
|
+
const best = (found) => {
|
|
977
|
+
const holders = patterns.map((_, i) => found.filter((f) => f.terms.includes(i)).length);
|
|
978
|
+
const weight = (terms) => terms.reduce((sum, i) => sum + 1 / holders[i], 0);
|
|
979
|
+
return found.map((f) => ({ item: f.item, w: weight(f.terms) })).sort((a, b) => b.w - a.w).map((f) => f.item);
|
|
980
|
+
};
|
|
981
|
+
const matched = new Map();
|
|
982
|
+
// Only the notes that can hold one of the words are read: the index names them, and a note whose file changed since
|
|
983
|
+
// it was indexed is read anyway. A retired entry's daily log is always read, as its liveness is decided by reading it.
|
|
984
|
+
const retiredDays = await retiredEntryFiles(ctx);
|
|
985
|
+
const indexed = notesWithIndexedText(ctx.db, (text) => termsIn(text).length > 0);
|
|
986
|
+
const collected = await collectMemories(ctx, (m, text) => {
|
|
987
|
+
if (!inScope(m.path))
|
|
988
|
+
return false;
|
|
989
|
+
const terms = termsIn(m.path, m.title, text);
|
|
990
|
+
if (terms.length > 0)
|
|
991
|
+
matched.set(m.id, terms);
|
|
992
|
+
return terms.length > 0;
|
|
993
|
+
}, async (n) => {
|
|
994
|
+
// The index keeps each note under its absolute path; the journal and the scope use the vault-relative one.
|
|
995
|
+
const rel = toRel(ctx.memoryDir, n.path) ?? '';
|
|
996
|
+
if (retiredDays.has(rel))
|
|
997
|
+
return true;
|
|
998
|
+
if (!inScope(rel))
|
|
999
|
+
return false;
|
|
1000
|
+
return termsIn(rel, n.title).length > 0 || indexed.has(n.path) || changedSinceIndexed(ctx.memoryDir, n);
|
|
1001
|
+
});
|
|
1002
|
+
const memories = best(collected.memories.map((m) => ({ item: m, terms: matched.get(m.id) })));
|
|
1003
|
+
const rootReal = await promises_1.default.realpath(ctx.memoryDir);
|
|
1004
|
+
const foundRetired = [];
|
|
1005
|
+
for (const r of await completedRetirements(ctx, collected.seen)) {
|
|
1006
|
+
if (r.scope !== undefined ? args.scope !== undefined && !(0, scope_1.savedScopeMatches)(r.scope, args.scope) : !inScope(r.memory.path))
|
|
1007
|
+
continue;
|
|
1008
|
+
const terms = termsIn(r.memory.path, r.memory.title, r.storedAt ? await retiredNoteBody(rootReal, r.storedAt) : r.text);
|
|
1009
|
+
if (terms.length > 0)
|
|
1010
|
+
foundRetired.push({ item: r.memory, terms });
|
|
1011
|
+
}
|
|
1012
|
+
const retired = best(foundRetired);
|
|
1013
|
+
return {
|
|
1014
|
+
ok: true,
|
|
1015
|
+
generatedAt: new Date().toISOString(),
|
|
1016
|
+
query: args.query,
|
|
1017
|
+
...(args.scope === undefined ? {} : { scope: args.scope }),
|
|
1018
|
+
memories: memories.slice(0, args.limit),
|
|
1019
|
+
retired: retired.slice(0, args.limit),
|
|
1020
|
+
truncated: { memories: collected.truncated || memories.length > args.limit, retired: retired.length > args.limit },
|
|
1021
|
+
};
|
|
1022
|
+
}
|
|
1023
|
+
async function readMemory(ctx, id) {
|
|
1024
|
+
const ref = parseMemoryId(id);
|
|
1025
|
+
const file = await resolveVaultFile(ctx, ref.rel, true);
|
|
1026
|
+
await assertNotExcluded(ctx, file);
|
|
1027
|
+
if (ref.kind === 'entry') {
|
|
1028
|
+
const daily = dailyNote_1.DAILY_FILE_RE.exec(file.rel);
|
|
1029
|
+
if (!daily)
|
|
1030
|
+
throw (0, toolRefusal_1.refusal)('memory_not_found', 'memory not found');
|
|
1031
|
+
const block = parseEntries(file.rel, await readWhole(file.real)).blocks.find((b) => b.id === id);
|
|
1032
|
+
if (!block)
|
|
1033
|
+
throw (0, toolRefusal_1.refusal)('memory_not_found', 'memory not found (it may have been retired)');
|
|
1034
|
+
// The same bound the list shows the entry with: its note's indexed write time (main's createNoteTimeBound).
|
|
1035
|
+
const written = ctx.db.prepare('SELECT modified_at FROM notes WHERE path = ?').get(file.indexed)?.modified_at
|
|
1036
|
+
?? (await promises_1.default.stat(file.real)).mtime.toISOString();
|
|
1037
|
+
const dated = (0, orchestrator_1.createNoteTimeBound)(ctx.db)(file.indexed, (0, dailyNote_1.entryInstant)(daily[2], block));
|
|
1038
|
+
return {
|
|
1039
|
+
ok: true,
|
|
1040
|
+
memory: {
|
|
1041
|
+
id, kind: 'entry', path: file.rel, title: collapse(block.text, 90), at: dated ?? written,
|
|
1042
|
+
...(dated ? { session_date: dated } : {}),
|
|
1043
|
+
source: block.source || null, tags: [], status: null, excerpt: collapse(block.text, EXCERPT_CHARS),
|
|
1044
|
+
},
|
|
1045
|
+
text: block.text.slice(0, MAX_READ_CHARS),
|
|
1046
|
+
textTruncated: block.text.length > MAX_READ_CHARS,
|
|
1047
|
+
metadata: {},
|
|
1048
|
+
facts: [],
|
|
1049
|
+
};
|
|
1050
|
+
}
|
|
1051
|
+
const read = await readPrefix(file.real, READ_BYTES);
|
|
1052
|
+
if (read === null)
|
|
1053
|
+
throw (0, toolRefusal_1.refusal)('memory_not_found', 'memory not found (it may have been retired)');
|
|
1054
|
+
const parsed = (0, parser_1.parseNote)(file.indexed, read.text);
|
|
1055
|
+
const row = ctx.db.prepare('SELECT title, modified_at, frontmatter_json FROM notes WHERE path = ?').get(file.indexed);
|
|
1056
|
+
const dated = row ? (0, orchestrator_1.resolveSessionDate)(row.frontmatter_json, row.modified_at) : undefined;
|
|
1057
|
+
// Only the documented keys are copied. A front-matter key the type does not name stays in the note and the export.
|
|
1058
|
+
const metadata = {};
|
|
1059
|
+
for (const [k, v] of Object.entries(parsed.frontmatter)) {
|
|
1060
|
+
if (!METADATA_KEYS.has(k) || v === null || v === undefined)
|
|
1061
|
+
continue;
|
|
1062
|
+
metadata[k] = Array.isArray(v) ? v.map(String).join(', ') : v instanceof Date ? v.toISOString().slice(0, 10) : String(v);
|
|
1063
|
+
}
|
|
1064
|
+
const factRows = ctx.db.prepare(`SELECT predicate, object_text AS object, valid_from, valid_to FROM facts WHERE source_path = ? ORDER BY id`).all(file.indexed);
|
|
1065
|
+
const body = parsed.body;
|
|
1066
|
+
return {
|
|
1067
|
+
ok: true,
|
|
1068
|
+
memory: {
|
|
1069
|
+
id, kind: 'note', path: file.rel, title: row?.title ?? parsed.title, at: row?.modified_at ?? new Date(0).toISOString(),
|
|
1070
|
+
...(dated ? { session_date: dated } : {}),
|
|
1071
|
+
source: typeof parsed.frontmatter.source === 'string' ? parsed.frontmatter.source : null,
|
|
1072
|
+
tags: tagsOf(parsed.frontmatter),
|
|
1073
|
+
status: typeof parsed.frontmatter.status === 'string' ? parsed.frontmatter.status : null,
|
|
1074
|
+
excerpt: collapse(body, EXCERPT_CHARS),
|
|
1075
|
+
},
|
|
1076
|
+
text: body.slice(0, MAX_READ_CHARS),
|
|
1077
|
+
textTruncated: body.length > MAX_READ_CHARS || read.truncated,
|
|
1078
|
+
metadata,
|
|
1079
|
+
facts: factRows
|
|
1080
|
+
.filter((f) => !METADATA_PREDICATES.has(f.predicate))
|
|
1081
|
+
.map((f) => ({ predicate: f.predicate, object: f.object ?? '', validFrom: f.valid_from, validTo: f.valid_to })),
|
|
1082
|
+
};
|
|
1083
|
+
}
|
|
1084
|
+
// ------------------------------- retire / restore ---------------------------
|
|
1085
|
+
let chain = Promise.resolve();
|
|
1086
|
+
function serialize(fn) {
|
|
1087
|
+
const result = chain.then(fn, fn);
|
|
1088
|
+
chain = result.then(() => undefined, () => undefined);
|
|
1089
|
+
return result;
|
|
1090
|
+
}
|
|
1091
|
+
/** Remove a note's derived rows (chunks, FTS, entities, structured rows cascade) and its vectors. */
|
|
1092
|
+
function dropNoteFromIndex(ctx, indexed) {
|
|
1093
|
+
const chunkIds = ctx.db.prepare('SELECT id FROM chunks WHERE note_path = ?').all(indexed).map((r) => r.id);
|
|
1094
|
+
ctx.db.transaction(() => {
|
|
1095
|
+
ctx.db.prepare('DELETE FROM facts WHERE source_path = ?').run(indexed);
|
|
1096
|
+
ctx.db.prepare('DELETE FROM relations WHERE source_note = ?').run(indexed);
|
|
1097
|
+
ctx.db.prepare('DELETE FROM notes WHERE path = ?').run(indexed);
|
|
1098
|
+
})();
|
|
1099
|
+
for (const id of chunkIds)
|
|
1100
|
+
ctx.indexOptions.vectorIndex?.delete(id);
|
|
1101
|
+
}
|
|
1102
|
+
/**
|
|
1103
|
+
* Settle retirements and restores an earlier crash interrupted, so the journal, the files and the
|
|
1104
|
+
* index agree again:
|
|
1105
|
+
* - note, file never moved (no stored copy, original present): void the record and re-index the
|
|
1106
|
+
* note (its index rows may already have been dropped);
|
|
1107
|
+
* - note, moved but index rows survived: drop them;
|
|
1108
|
+
* - note, restored on disk but the journal not yet told (original present, stored copy gone):
|
|
1109
|
+
* covered by the first case;
|
|
1110
|
+
* - entry still in its file: void the record (the retirement never happened, or it was restored);
|
|
1111
|
+
* - entry gone from its file: re-index the file, so a crash between the rewrite and the index
|
|
1112
|
+
* update never leaves retired text searchable.
|
|
1113
|
+
*/
|
|
1114
|
+
async function reconcile(ctx) {
|
|
1115
|
+
const rootReal = await promises_1.default.realpath(ctx.memoryDir);
|
|
1116
|
+
const journal = await readJournal(ctx);
|
|
1117
|
+
await removePurgedCopies(rootReal, journal);
|
|
1118
|
+
for (const r of activeRetired(journal)) {
|
|
1119
|
+
let file;
|
|
1120
|
+
try {
|
|
1121
|
+
file = await resolveVaultFile(ctx, r.path, false);
|
|
1122
|
+
}
|
|
1123
|
+
catch {
|
|
1124
|
+
continue; // a record that no longer resolves safely is left alone, never acted on
|
|
1125
|
+
}
|
|
1126
|
+
if (r.kind === 'note') {
|
|
1127
|
+
const stored = r.storedAt ? path_1.default.join(rootReal, ...r.storedAt.split('/')) : null;
|
|
1128
|
+
const storedThere = stored ? await exists(stored) : false;
|
|
1129
|
+
const originalThere = await exists(file.real);
|
|
1130
|
+
if (!storedThere && originalThere) {
|
|
1131
|
+
// Index first: if it fails, the record stays open and the next attempt repairs it again.
|
|
1132
|
+
await (0, pipeline_1.indexFile)(ctx.db, file.indexed, ctx.indexOptions);
|
|
1133
|
+
await appendJournal(ctx, voidRecord(r));
|
|
1134
|
+
}
|
|
1135
|
+
else if (storedThere && !originalThere) {
|
|
1136
|
+
if (ctx.db.prepare('SELECT 1 FROM notes WHERE path = ?').get(file.indexed))
|
|
1137
|
+
dropNoteFromIndex(ctx, file.indexed);
|
|
1138
|
+
}
|
|
1139
|
+
continue;
|
|
1140
|
+
}
|
|
1141
|
+
if (!(await exists(file.real)))
|
|
1142
|
+
continue;
|
|
1143
|
+
const present = await (0, writer_1.withDailyFileLock)(file.indexed, async () => parseEntries(file.rel, await readWhole(file.real)).blocks.some((b) => b.id === r.id));
|
|
1144
|
+
await (0, pipeline_1.indexFile)(ctx.db, file.indexed, ctx.indexOptions);
|
|
1145
|
+
if (present)
|
|
1146
|
+
await appendJournal(ctx, voidRecord(r));
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
/**
|
|
1150
|
+
* The record that ends an interrupted retirement. It is never read for its text, so while the secret guard is on
|
|
1151
|
+
* it does not carry a second copy of the block, which may be what the guard would have refused to retire.
|
|
1152
|
+
*/
|
|
1153
|
+
function voidRecord(r) {
|
|
1154
|
+
const voided = { ...r, voided: true };
|
|
1155
|
+
if ((0, secretGuard_1.secretGuardEnabled)())
|
|
1156
|
+
delete voided.block;
|
|
1157
|
+
return voided;
|
|
1158
|
+
}
|
|
1159
|
+
function newRid() {
|
|
1160
|
+
return `retired:${crypto_1.default.randomBytes(8).toString('hex')}`;
|
|
1161
|
+
}
|
|
1162
|
+
function retireMemory(ctx, id, expectedHash) {
|
|
1163
|
+
if ((0, lifecycle_1.factEventId)(id) !== null)
|
|
1164
|
+
return (0, lifecycle_1.retireFact)(ctx.db, ctx.memoryDir, ctx.ledgerRelPath, id, ctx.source);
|
|
1165
|
+
return serialize(async () => {
|
|
1166
|
+
await reconcile(ctx);
|
|
1167
|
+
const ref = parseMemoryId(id);
|
|
1168
|
+
const file = await resolveVaultFile(ctx, ref.rel, true);
|
|
1169
|
+
await assertNotExcluded(ctx, file);
|
|
1170
|
+
const rootReal = await promises_1.default.realpath(ctx.memoryDir);
|
|
1171
|
+
const at = new Date().toISOString();
|
|
1172
|
+
const rid = newRid();
|
|
1173
|
+
if (ref.kind === 'entry') {
|
|
1174
|
+
if (!dailyNote_1.DAILY_FILE_RE.test(file.rel))
|
|
1175
|
+
throw (0, toolRefusal_1.refusal)('memory_not_found', 'memory not found');
|
|
1176
|
+
const { title, emptied } = await (0, writer_1.withDailyFileLock)(file.indexed, async () => {
|
|
1177
|
+
const { lines, blocks } = parseEntries(file.rel, await readWhole(file.real));
|
|
1178
|
+
const block = blocks.find((b) => b.id === id);
|
|
1179
|
+
if (!block)
|
|
1180
|
+
throw (0, toolRefusal_1.refusal)('memory_not_found', 'memory not found (it may already be retired)');
|
|
1181
|
+
const entryTitle = collapse(block.text, 90);
|
|
1182
|
+
// RENOR_SECRET_GUARD (default ON): retiring copies the block into the journal, so a block the scanner flags
|
|
1183
|
+
// is not retired, and nothing has been written yet.
|
|
1184
|
+
if ((0, secretGuard_1.secretGuardEnabled)()) {
|
|
1185
|
+
await (0, secretGuard_1.assertStorable)([{ field: 'memory', text: block.raw }, { field: 'title', text: entryTitle }], 'retire');
|
|
1186
|
+
}
|
|
1187
|
+
const rest = withoutRanges(lines, block.ranges).join('\n');
|
|
1188
|
+
// The log's last entry takes the log with it. What is left is the engine's own header: no memory, yet it stayed
|
|
1189
|
+
// indexed as a note, an entity and a chunk, and with no entry in it no id could retire it (live dogfood,
|
|
1190
|
+
// 2026-10-03, D11). Its bytes go into the record, which restore writes back.
|
|
1191
|
+
const lastEntry = (0, dailyNote_1.isEmptyEngineDailyNote)(file.rel, rest);
|
|
1192
|
+
// Journal first: a crash before the rewrite leaves the entry in place and the record void.
|
|
1193
|
+
await appendJournal(ctx, {
|
|
1194
|
+
rid, id, kind: 'entry', path: file.rel, title: entryTitle, retiredAt: at, block: block.raw, copies: block.ranges.length,
|
|
1195
|
+
...entryNeighbours(blocks, block),
|
|
1196
|
+
...(lastEntry ? { remainder: rest } : {}),
|
|
1197
|
+
});
|
|
1198
|
+
if (lastEntry) {
|
|
1199
|
+
// Out of the index first, as a retired note is: a crash before the file goes leaves the entry in place, and
|
|
1200
|
+
// reconcile voids the record and indexes the file again.
|
|
1201
|
+
dropNoteFromIndex(ctx, file.indexed);
|
|
1202
|
+
await promises_1.default.unlink(file.real);
|
|
1203
|
+
}
|
|
1204
|
+
else {
|
|
1205
|
+
await atomicWrite(file.real, rest);
|
|
1206
|
+
}
|
|
1207
|
+
return { title: entryTitle, emptied: lastEntry };
|
|
1208
|
+
});
|
|
1209
|
+
if (!emptied)
|
|
1210
|
+
await (0, pipeline_1.indexFile)(ctx.db, file.indexed, ctx.indexOptions);
|
|
1211
|
+
recordEvent(ctx.db, file.rel, 'retire', id, title);
|
|
1212
|
+
return { ok: true, id, action: 'retired', title, at };
|
|
1213
|
+
}
|
|
1214
|
+
const title = ctx.db.prepare('SELECT title FROM notes WHERE path = ?').get(file.indexed)?.title
|
|
1215
|
+
?? (0, parser_1.parseNote)(file.indexed, (await readPrefix(file.real, MAP_READ_BYTES))?.text ?? '').title;
|
|
1216
|
+
// RENOR_SECRET_GUARD (default ON): the same rule for a whole note, before the retired directory exists.
|
|
1217
|
+
if ((0, secretGuard_1.secretGuardEnabled)()) {
|
|
1218
|
+
await (0, secretGuard_1.assertStorable)([{ field: 'memory', text: await readToScreen(file.real, 'retire') }, { field: 'title', text: title }], 'retire');
|
|
1219
|
+
}
|
|
1220
|
+
// Each retirement gets its own stored copy, so retiring a later note at the same path never
|
|
1221
|
+
// hides an earlier one.
|
|
1222
|
+
const storedRel = path_1.default.posix.join(RETIRED_FILES_DIR.split(path_1.default.sep).join('/'), `${rid.slice('retired:'.length)}`, file.rel);
|
|
1223
|
+
const storedDir = await ensureRealDir(rootReal, path_1.default.posix.dirname(storedRel));
|
|
1224
|
+
const storedReal = path_1.default.join(storedDir, path_1.default.posix.basename(storedRel));
|
|
1225
|
+
// Journal first, then leave retrieval, then move: an interruption anywhere leaves a state
|
|
1226
|
+
// reconcile() settles, never a memory nobody can restore.
|
|
1227
|
+
await appendJournal(ctx, { rid, id, kind: 'note', path: file.rel, title, retiredAt: at, storedAt: storedRel });
|
|
1228
|
+
// Under the file's lock, so an index pass already reading this note finishes before the drop,
|
|
1229
|
+
// and one that starts after finds the file gone instead of re-adding it.
|
|
1230
|
+
await (0, writer_1.withDailyFileLock)(file.indexed, async () => {
|
|
1231
|
+
if (expectedHash !== undefined && (0, hasher_1.hashContent)(await readWhole(file.real)) !== expectedHash) {
|
|
1232
|
+
// This retirement never took ownership: close its journal entry, preserving the edit.
|
|
1233
|
+
await appendJournal(ctx, { rid, id, kind: 'note', path: file.rel, title, retiredAt: at, storedAt: storedRel, voided: true });
|
|
1234
|
+
throw new Error('imported note changed; preserved');
|
|
1235
|
+
}
|
|
1236
|
+
dropNoteFromIndex(ctx, file.indexed);
|
|
1237
|
+
await promises_1.default.rename(file.real, storedReal);
|
|
1238
|
+
});
|
|
1239
|
+
recordEvent(ctx.db, file.rel, 'retire', id, title);
|
|
1240
|
+
return { ok: true, id, action: 'retired', title, at };
|
|
1241
|
+
});
|
|
1242
|
+
}
|
|
1243
|
+
/**
|
|
1244
|
+
* Restore by the retirement's id (from memory_map.retired) or by the memory id when exactly one
|
|
1245
|
+
* retirement of it is in effect. Idempotent: a restore whose bytes are already back (a retry after
|
|
1246
|
+
* an interruption) completes the journal and re-indexes instead of failing.
|
|
1247
|
+
*/
|
|
1248
|
+
function restoreMemory(ctx, id, options = {}) {
|
|
1249
|
+
if ((0, lifecycle_1.factEventId)(id) !== null)
|
|
1250
|
+
return (0, lifecycle_1.restoreFact)(ctx.db, ctx.memoryDir, ctx.ledgerRelPath, id, ctx.source, options.validateAfterWrite);
|
|
1251
|
+
return serialize(async () => {
|
|
1252
|
+
const before = await readJournal(ctx);
|
|
1253
|
+
const pick = (records) => {
|
|
1254
|
+
const active = activeRetired(records);
|
|
1255
|
+
if (id.startsWith('retired:'))
|
|
1256
|
+
return active.find((r) => r.rid === id) ?? null;
|
|
1257
|
+
parseMemoryId(id);
|
|
1258
|
+
const forMemory = active.filter((r) => r.id === id);
|
|
1259
|
+
if (forMemory.length > 1)
|
|
1260
|
+
throw (0, toolRefusal_1.refusal)('memory_already_retired', 'this memory was retired more than once; restore one retirement by its id from memory_map');
|
|
1261
|
+
return forMemory[0] ?? null;
|
|
1262
|
+
};
|
|
1263
|
+
const wanted = pick(before);
|
|
1264
|
+
await reconcile(ctx);
|
|
1265
|
+
const record = pick(await readJournal(ctx));
|
|
1266
|
+
if (!record) {
|
|
1267
|
+
// Settled by reconcile because its bytes were already back: that is a completed restore.
|
|
1268
|
+
if (wanted)
|
|
1269
|
+
return { ok: true, id: wanted.id, action: 'restored', title: wanted.title, at: new Date().toISOString() };
|
|
1270
|
+
throw (0, toolRefusal_1.refusal)('memory_not_found', 'no retired memory with that id');
|
|
1271
|
+
}
|
|
1272
|
+
const file = await resolveVaultFile(ctx, record.path, false);
|
|
1273
|
+
const rootReal = await promises_1.default.realpath(ctx.memoryDir);
|
|
1274
|
+
const at = new Date().toISOString();
|
|
1275
|
+
let rollback;
|
|
1276
|
+
if (record.kind === 'entry') {
|
|
1277
|
+
const day = dailyNote_1.DAILY_FILE_RE.exec(record.path)?.[2];
|
|
1278
|
+
// The block is stored exactly as it was in the file. Not trimmed: a statement that opens with a line break
|
|
1279
|
+
// has an entry line that ends in the separator space, and trimming it made the entry impossible to restore.
|
|
1280
|
+
if (!record.block || !day || !dailyNote_1.ENTRY_LINE_RE.test(record.block.split('\n')[0]))
|
|
1281
|
+
throw new Error('retired entry record is incomplete');
|
|
1282
|
+
// RENOR_SECRET_GUARD (default ON): restoring puts the text back into live memory, so it is screened like a save.
|
|
1283
|
+
if ((0, secretGuard_1.secretGuardEnabled)()) {
|
|
1284
|
+
await (0, secretGuard_1.assertStorable)([{ field: 'memory', text: record.block }, { field: 'title', text: record.title }], 'restore');
|
|
1285
|
+
}
|
|
1286
|
+
await ensureRealDir(rootReal, path_1.default.posix.dirname(record.path));
|
|
1287
|
+
await (0, writer_1.withDailyFileLock)(file.indexed, async () => {
|
|
1288
|
+
let existing = null;
|
|
1289
|
+
try {
|
|
1290
|
+
existing = await readWhole(file.real);
|
|
1291
|
+
}
|
|
1292
|
+
catch (err) {
|
|
1293
|
+
if (err.code !== 'ENOENT')
|
|
1294
|
+
throw err;
|
|
1295
|
+
}
|
|
1296
|
+
const copies = Array.from({ length: Math.max(1, record.copies ?? 1) }, () => record.block).join('\n');
|
|
1297
|
+
rollback = existing === null
|
|
1298
|
+
? async () => { await promises_1.default.rm(file.real, { force: true }); }
|
|
1299
|
+
: async () => { await atomicWrite(file.real, existing); };
|
|
1300
|
+
if (existing !== null) {
|
|
1301
|
+
const { lines, blocks } = parseEntries(record.path, existing);
|
|
1302
|
+
if (blocks.some((b) => b.id === record.id))
|
|
1303
|
+
return; // already back
|
|
1304
|
+
// Back to its place in the day, not the end of the file: a restore at the end put an earlier entry after later ones.
|
|
1305
|
+
const line = restoreLine(blocks, record, (0, dailyNote_1.parseEntryHeader)(record.block.split('\n')[0])?.time ?? '');
|
|
1306
|
+
if (line !== null) {
|
|
1307
|
+
lines.splice(line, 0, ...copies.split('\n'));
|
|
1308
|
+
await atomicWrite(file.real, lines.join('\n'));
|
|
1309
|
+
return;
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
const base = existing ?? record.remainder ?? (0, dailyNote_1.engineDailyHeader)(day);
|
|
1313
|
+
await atomicWrite(file.real, `${base.replace(/\n*$/, '')}\n${copies}`);
|
|
1314
|
+
});
|
|
1315
|
+
}
|
|
1316
|
+
else {
|
|
1317
|
+
if (!record.storedAt)
|
|
1318
|
+
throw new Error('retired note record is incomplete');
|
|
1319
|
+
const storedDir = path_1.default.join(rootReal, ...path_1.default.posix.dirname(record.storedAt).split('/'));
|
|
1320
|
+
if (!within(path_1.default.join(rootReal, RETIRED_FILES_DIR), path_1.default.join(storedDir, 'x')) || (await promises_1.default.realpath(storedDir)) !== storedDir) {
|
|
1321
|
+
throw (0, toolRefusal_1.refusal)('unsafe_path', 'retired note record points outside the retired directory');
|
|
1322
|
+
}
|
|
1323
|
+
const storedReal = path_1.default.join(storedDir, path_1.default.posix.basename(record.storedAt));
|
|
1324
|
+
if (await exists(file.real))
|
|
1325
|
+
throw (0, toolRefusal_1.refusal)('restore_conflict', 'a memory already exists at that path; restore would overwrite it');
|
|
1326
|
+
if ((0, secretGuard_1.secretGuardEnabled)()) {
|
|
1327
|
+
await (0, secretGuard_1.assertStorable)([{ field: 'memory', text: await readToScreen(storedReal, 'restore') }, { field: 'title', text: record.title }], 'restore');
|
|
1328
|
+
}
|
|
1329
|
+
const targetDir = await ensureRealDir(rootReal, path_1.default.posix.dirname(record.path));
|
|
1330
|
+
const targetReal = path_1.default.join(targetDir, path_1.default.posix.basename(record.path));
|
|
1331
|
+
await promises_1.default.rename(storedReal, targetReal);
|
|
1332
|
+
rollback = async () => { await promises_1.default.rename(targetReal, storedReal); };
|
|
1333
|
+
}
|
|
1334
|
+
try {
|
|
1335
|
+
await options.validateAfterWrite?.();
|
|
1336
|
+
}
|
|
1337
|
+
catch (err) {
|
|
1338
|
+
await rollback?.();
|
|
1339
|
+
throw err;
|
|
1340
|
+
}
|
|
1341
|
+
// Index before the journal says "restored": if indexing fails, the record stays active and a
|
|
1342
|
+
// retry completes the restore (see the idempotent path above).
|
|
1343
|
+
await (0, pipeline_1.indexFile)(ctx.db, file.indexed, ctx.indexOptions);
|
|
1344
|
+
await appendJournal(ctx, { ...record, restoredAt: at });
|
|
1345
|
+
recordEvent(ctx.db, record.path, 'restore', record.id, record.title);
|
|
1346
|
+
return { ok: true, id: record.id, action: 'restored', title: record.title, at };
|
|
1347
|
+
});
|
|
1348
|
+
}
|
|
1349
|
+
/** The directory holding one retirement's stored copy: `.trash/renor-retired/files/<rid without its prefix>`. */
|
|
1350
|
+
function storedCopyDir(rootReal, rid) {
|
|
1351
|
+
const hex = rid.startsWith('retired:') ? rid.slice('retired:'.length) : '';
|
|
1352
|
+
if (!/^[0-9a-f]{16}$/.test(hex))
|
|
1353
|
+
return null;
|
|
1354
|
+
return path_1.default.join(rootReal, RETIRED_FILES_DIR, hex);
|
|
1355
|
+
}
|
|
1356
|
+
/**
|
|
1357
|
+
* Remove the stored copy of every retirement the journal says was deleted. The journal is rewritten before any copy is
|
|
1358
|
+
* removed (purgeRetired), so a crash between the two leaves a copy no record points to: this finds it by the rid the
|
|
1359
|
+
* deletion's record keeps, and removes it. Only a plain directory inside the retired area is removed, never through a
|
|
1360
|
+
* link.
|
|
1361
|
+
*/
|
|
1362
|
+
async function removePurgedCopies(rootReal, journal) {
|
|
1363
|
+
for (const r of journal) {
|
|
1364
|
+
if (!r.purgedAt || r.kind !== 'note')
|
|
1365
|
+
continue;
|
|
1366
|
+
const dir = storedCopyDir(rootReal, r.rid);
|
|
1367
|
+
if (!dir)
|
|
1368
|
+
continue;
|
|
1369
|
+
let st;
|
|
1370
|
+
try {
|
|
1371
|
+
st = await promises_1.default.lstat(dir);
|
|
1372
|
+
}
|
|
1373
|
+
catch (err) {
|
|
1374
|
+
if (err.code === 'ENOENT')
|
|
1375
|
+
continue;
|
|
1376
|
+
throw err;
|
|
1377
|
+
}
|
|
1378
|
+
if (!st.isDirectory() || (await promises_1.default.realpath(path_1.default.dirname(dir))) !== path_1.default.dirname(dir))
|
|
1379
|
+
continue;
|
|
1380
|
+
// fs.rm removes a link inside the tree, never what it points to.
|
|
1381
|
+
await promises_1.default.rm(dir, { recursive: true, force: true });
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
/** Bytes of regular files below `dir`, never through a link (what the storage gauge counts of a stored copy). */
|
|
1385
|
+
async function bytesBelow(dir) {
|
|
1386
|
+
let total = 0;
|
|
1387
|
+
let entries;
|
|
1388
|
+
try {
|
|
1389
|
+
entries = await promises_1.default.readdir(dir, { withFileTypes: true });
|
|
1390
|
+
}
|
|
1391
|
+
catch (err) {
|
|
1392
|
+
if (err.code === 'ENOENT')
|
|
1393
|
+
return 0;
|
|
1394
|
+
throw err;
|
|
1395
|
+
}
|
|
1396
|
+
for (const entry of entries) {
|
|
1397
|
+
const abs = path_1.default.join(dir, entry.name);
|
|
1398
|
+
if (entry.isDirectory())
|
|
1399
|
+
total += await bytesBelow(abs);
|
|
1400
|
+
else if (entry.isFile() && entry.name.endsWith('.md'))
|
|
1401
|
+
total += (await promises_1.default.lstat(abs)).size;
|
|
1402
|
+
}
|
|
1403
|
+
return total;
|
|
1404
|
+
}
|
|
1405
|
+
/**
|
|
1406
|
+
* Permanently delete retired notes and daily-log entries: `target` is one retirement's id from memory_map.retired
|
|
1407
|
+
* (`retired:<hex>`), or `all` for every retired note and entry. With `all`, `only` (retirement ids, as the owner saw them)
|
|
1408
|
+
* limits the deletion to those, so a retirement made after the owner's list was read is not deleted unseen. A live memory
|
|
1409
|
+
* has no retirement id, so it can never be deleted in one step: it is retired first. A retired fact is kept in the fact
|
|
1410
|
+
* ledger, which is replayed to rebuild the index, and is refused here.
|
|
1411
|
+
*
|
|
1412
|
+
* What goes: a note's stored copy (so its bytes stop counting toward storage), an entry's text in the journal, every
|
|
1413
|
+
* journal record of the memory's earlier retirements that carried its text, the retire, restore and note events that
|
|
1414
|
+
* name it, cached evidence read from its file, a note's path in the import journal (.renor-imports), and every replacement
|
|
1415
|
+
* record that names it (replaced.jsonl). The search index
|
|
1416
|
+
* lost the memory when it was retired. What stays: one journal record per deletion (rid, kind, time) and one write event
|
|
1417
|
+
* (how many, of which kind), neither with an id, a path, a title or any text.
|
|
1418
|
+
*
|
|
1419
|
+
* Crash safety: events, cached evidence, import-journal entries and replacement records are dropped first (a crash then leaves
|
|
1420
|
+
* the memory retired and restorable, without its event titles, its undo entry or its replacement link); the journal rewrite is atomic and is the moment of
|
|
1421
|
+
* deletion (from then on nothing lists or restores it); a stored copy left behind by a crash after it is removed by the
|
|
1422
|
+
* next retire, restore or delete (removePurgedCopies), and until then it counts toward storage but nothing reads it.
|
|
1423
|
+
*/
|
|
1424
|
+
async function purgeRetired(ctx, target, only) {
|
|
1425
|
+
if ((0, lifecycle_1.factEventId)(target) !== null) {
|
|
1426
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'a retired fact stays in the fact ledger and cannot be deleted here; email privacy@renor.cloud to erase one');
|
|
1427
|
+
}
|
|
1428
|
+
if (target !== 'all' && !target.startsWith('retired:')) {
|
|
1429
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'pass the id of a retired memory from memory_map.retired (retired:<id>) or "all"; a live memory is retired first');
|
|
1430
|
+
}
|
|
1431
|
+
if (only !== undefined && target !== 'all') {
|
|
1432
|
+
throw (0, toolRefusal_1.refusal)('invalid_argument', 'a list of ids goes with "all" only: one id deletes one memory');
|
|
1433
|
+
}
|
|
1434
|
+
return serialize(async () => {
|
|
1435
|
+
await reconcile(ctx);
|
|
1436
|
+
const rootReal = await promises_1.default.realpath(ctx.memoryDir);
|
|
1437
|
+
const journal = await readJournal(ctx);
|
|
1438
|
+
const live = (await collectMemories(ctx)).seen;
|
|
1439
|
+
const completed = (await completedRetirements(ctx, live)).filter((r) => r.memory.kind !== 'fact');
|
|
1440
|
+
const listed = only === undefined ? null : new Set(only);
|
|
1441
|
+
const chosen = target === 'all'
|
|
1442
|
+
? completed.filter((r) => listed === null || listed.has(r.memory.id))
|
|
1443
|
+
: completed.filter((r) => r.memory.id === target);
|
|
1444
|
+
if (target !== 'all' && chosen.length === 0)
|
|
1445
|
+
throw (0, toolRefusal_1.refusal)('memory_not_found', 'no retired note or entry with that id');
|
|
1446
|
+
const at = new Date().toISOString();
|
|
1447
|
+
if (chosen.length === 0)
|
|
1448
|
+
return { ok: true, action: 'deleted', ids: [], freedBytes: 0, at };
|
|
1449
|
+
const rids = new Set(chosen.map((r) => r.memory.id));
|
|
1450
|
+
const memoryIds = new Set(chosen.map((r) => r.memory.memoryId));
|
|
1451
|
+
const byRid = new Map(activeRetired(journal).map((r) => [r.rid, r]));
|
|
1452
|
+
let freedBytes = 0;
|
|
1453
|
+
for (const rid of rids) {
|
|
1454
|
+
const dir = byRid.get(rid)?.kind === 'note' ? storedCopyDir(rootReal, rid) : null;
|
|
1455
|
+
if (dir)
|
|
1456
|
+
freedBytes += await bytesBelow(dir);
|
|
1457
|
+
}
|
|
1458
|
+
// 1. Events, cached evidence, import-journal entries and replacement records that carry its text or path. Idempotent, so a retry after a
|
|
1459
|
+
// crash repeats them harmlessly.
|
|
1460
|
+
const stems = new Set();
|
|
1461
|
+
await (0, forgetImported_1.forgetImportedNotes)(ctx.memoryDir, new Set(chosen.filter((r) => r.memory.kind === 'note').map((r) => r.memory.path)));
|
|
1462
|
+
await (0, replacedJournal_1.forgetReplacementsOf)(path_1.default.join(rootReal, exports.RETIRED_DIR), new Set(chosen.map((r) => r.memory.memoryId).filter((id) => typeof id === 'string')));
|
|
1463
|
+
ctx.db.transaction(() => {
|
|
1464
|
+
const named = ctx.db.prepare(`DELETE FROM write_events WHERE action IN ('retire', 'restore') AND json_valid(summary) AND json_extract(summary, '$.id') = ?`);
|
|
1465
|
+
const noted = ctx.db.prepare(`DELETE FROM write_events WHERE action = 'note' AND note_path IN (?, ?) AND summary = ?`);
|
|
1466
|
+
for (const memoryId of memoryIds)
|
|
1467
|
+
named.run(memoryId);
|
|
1468
|
+
for (const r of chosen) {
|
|
1469
|
+
const rel = r.memory.path;
|
|
1470
|
+
stems.add(rel);
|
|
1471
|
+
stems.add(path_1.default.resolve(ctx.memoryDir, ...rel.split('/')));
|
|
1472
|
+
if (r.memory.kind !== 'entry')
|
|
1473
|
+
continue;
|
|
1474
|
+
const record = byRid.get(r.memory.id);
|
|
1475
|
+
const text = record?.block ? parseEntries(rel, record.block).blocks[0]?.text : undefined;
|
|
1476
|
+
if (text !== undefined)
|
|
1477
|
+
noted.run(rel, rel.split('/').join(path_1.default.sep), text.slice(0, 240));
|
|
1478
|
+
}
|
|
1479
|
+
})();
|
|
1480
|
+
// A daily log is read whole by the evidence pass, so its live entries' extractions go too; the cache asks again.
|
|
1481
|
+
if (ctx.evidenceCachePath)
|
|
1482
|
+
(0, evidenceCache_1.dropEvidenceForStems)(ctx.evidenceCachePath, [...stems]);
|
|
1483
|
+
// 2. The journal, rewritten atomically: the moment of deletion. Every record of a chosen retirement goes, and so does
|
|
1484
|
+
// every ended retirement of the same memory (a restore or a void kept the entry's text); other retirements still in
|
|
1485
|
+
// effect stay, so a memory retired twice keeps its other retirement.
|
|
1486
|
+
const kept = journal.filter((r) => !rids.has(r.rid) && !(memoryIds.has(r.id) && !byRid.has(r.rid)));
|
|
1487
|
+
const tombstones = [...rids].map((rid) => ({
|
|
1488
|
+
rid, kind: byRid.get(rid).kind, purgedAt: at, voided: true,
|
|
1489
|
+
}));
|
|
1490
|
+
const dir = await ensureRealDir(rootReal, exports.RETIRED_DIR);
|
|
1491
|
+
const file = path_1.default.join(dir, path_1.default.basename(RETIRED_INDEX));
|
|
1492
|
+
const st = await promises_1.default.lstat(file);
|
|
1493
|
+
if (!st.isFile() || st.nlink !== 1)
|
|
1494
|
+
throw new Error(`refusing to rewrite ${JSON.stringify(RETIRED_INDEX)}: it is not a regular file with a single name`);
|
|
1495
|
+
await atomicWrite(file, [...kept, ...tombstones].map((r) => JSON.stringify(r) + '\n').join(''));
|
|
1496
|
+
// 3. The stored copies. A crash before this leaves them to removePurgedCopies.
|
|
1497
|
+
await removePurgedCopies(rootReal, tombstones);
|
|
1498
|
+
// 4. The visible audit line: what kind and how many, nothing else.
|
|
1499
|
+
const notes = tombstones.filter((r) => r.kind === 'note').length;
|
|
1500
|
+
const entries = tombstones.length - notes;
|
|
1501
|
+
const counted = [notes ? `${notes} note${notes === 1 ? '' : 's'}` : '', entries ? `${entries} entr${entries === 1 ? 'y' : 'ies'}` : '']
|
|
1502
|
+
.filter(Boolean).join(' and ');
|
|
1503
|
+
ctx.db.prepare('INSERT INTO write_events (note_path, action, summary, created_at, author) VALUES (?, ?, ?, ?, ?)')
|
|
1504
|
+
.run('_retired_', 'delete', `deleted permanently: ${counted}`, at, ctx.source ?? null);
|
|
1505
|
+
return { ok: true, action: 'deleted', ids: [...rids], freedBytes, at };
|
|
1506
|
+
});
|
|
1507
|
+
}
|
|
1508
|
+
/** Every entry of one daily log as stored (identical entries are one, as everywhere), in file order. */
|
|
1509
|
+
function dailyEntriesForExport(rel, content) {
|
|
1510
|
+
const day = dailyNote_1.DAILY_FILE_RE.exec(rel)?.[2];
|
|
1511
|
+
if (!day)
|
|
1512
|
+
return [];
|
|
1513
|
+
return parseEntries(rel, content).blocks.map((b) => ({ id: b.id, at: (0, dailyNote_1.entryInstant)(day, b), source: b.source, text: b.text }));
|
|
1514
|
+
}
|
|
1515
|
+
/**
|
|
1516
|
+
* Every completed retirement of a note or an entry with its full retained text, newest first. A note's text is read from
|
|
1517
|
+
* its stored copy (confined to the retired area); an entry's is the journal's block. Retired facts are not here: they
|
|
1518
|
+
* come from listRetiredFacts. An interrupted retirement whose memory never left is not retired and is not listed.
|
|
1519
|
+
*/
|
|
1520
|
+
async function retiredForExport(ctx) {
|
|
1521
|
+
const rootReal = await promises_1.default.realpath(ctx.memoryDir);
|
|
1522
|
+
const out = [];
|
|
1523
|
+
for (const r of activeRetired(await readJournal(ctx)).reverse()) {
|
|
1524
|
+
const day = dailyNote_1.DAILY_FILE_RE.exec(r.path)?.[2];
|
|
1525
|
+
if (r.kind === 'entry') {
|
|
1526
|
+
if (!r.block)
|
|
1527
|
+
continue;
|
|
1528
|
+
let live = false;
|
|
1529
|
+
try {
|
|
1530
|
+
const file = await resolveVaultFile(ctx, r.path, true);
|
|
1531
|
+
live = parseEntries(file.rel, await readWhole(file.real)).blocks.some((b) => b.id === r.id);
|
|
1532
|
+
}
|
|
1533
|
+
catch { /* the log is gone with its last entry: the entry is retired */ }
|
|
1534
|
+
if (live)
|
|
1535
|
+
continue;
|
|
1536
|
+
const header = (0, dailyNote_1.parseEntryHeader)(r.block.split('\n')[0]);
|
|
1537
|
+
out.push({
|
|
1538
|
+
rid: r.rid, memoryId: r.id, kind: 'entry', path: r.path, title: r.title, retiredAt: r.retiredAt, content: r.block,
|
|
1539
|
+
...(day && header ? { savedAt: (0, dailyNote_1.entryInstant)(day, header) } : {}),
|
|
1540
|
+
});
|
|
1541
|
+
continue;
|
|
1542
|
+
}
|
|
1543
|
+
if (!r.storedAt)
|
|
1544
|
+
continue;
|
|
1545
|
+
const stored = path_1.default.join(rootReal, ...r.storedAt.split('/'));
|
|
1546
|
+
if (!within(path_1.default.join(rootReal, RETIRED_FILES_DIR), stored))
|
|
1547
|
+
continue;
|
|
1548
|
+
try {
|
|
1549
|
+
if ((await promises_1.default.realpath(path_1.default.dirname(stored))) !== path_1.default.dirname(stored))
|
|
1550
|
+
continue;
|
|
1551
|
+
const handle = await openConfined(stored);
|
|
1552
|
+
let content;
|
|
1553
|
+
try {
|
|
1554
|
+
content = await handle.readFile('utf8');
|
|
1555
|
+
}
|
|
1556
|
+
finally {
|
|
1557
|
+
await handle.close();
|
|
1558
|
+
}
|
|
1559
|
+
out.push({ rid: r.rid, memoryId: r.id, kind: 'note', path: r.path, title: r.title, retiredAt: r.retiredAt, storedAt: r.storedAt, content });
|
|
1560
|
+
}
|
|
1561
|
+
catch (err) {
|
|
1562
|
+
if (err.code === 'ENOENT')
|
|
1563
|
+
continue; // never moved: still live
|
|
1564
|
+
throw err;
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
return out;
|
|
1568
|
+
}
|