renor-engine 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +75 -0
- package/README.md +105 -2
- package/dist/.tsbuildinfo +1 -0
- package/dist/BUILD.json +6 -0
- package/dist/api/apiAuth.js +43 -0
- package/dist/api/server.js +174 -0
- package/dist/buildIdentity.js +149 -0
- package/dist/callPhases.js +82 -0
- package/dist/cli-extractors.js +46 -0
- package/dist/cli.js +500 -0
- package/dist/config.js +65 -0
- package/dist/context/composer.js +320 -0
- package/dist/context/readHonesty.js +478 -0
- package/dist/context/rowStore.js +238 -0
- package/dist/conversation/formatter.js +81 -0
- package/dist/conversation/parser.js +193 -0
- package/dist/conversation/types.js +2 -0
- package/dist/conversation/writer.js +42 -0
- package/dist/db/client.js +149 -0
- package/dist/db/handoffTable.js +31 -0
- package/dist/db/indexSalvage.js +101 -0
- package/dist/db/migrations.js +670 -0
- package/dist/db/readCache.js +109 -0
- package/dist/db/schema.js +5 -0
- package/dist/db/sqlite.js +396 -0
- package/dist/db/statements.js +89 -0
- package/dist/digests/digestCache.js +146 -0
- package/dist/digests/digests.js +597 -0
- package/dist/digests/worker.js +350 -0
- package/dist/embeddings/caching.js +337 -0
- package/dist/embeddings/coalescing.js +177 -0
- package/dist/embeddings/interactive.js +103 -0
- package/dist/embeddings/local.js +70 -0
- package/dist/embeddings/openai.js +106 -0
- package/dist/embeddings/rebuild.js +166 -0
- package/dist/embeddings/simdScan.js +260 -0
- package/dist/embeddings/space.js +21 -0
- package/dist/embeddings/types.js +14 -0
- package/dist/embeddings/vectorIndex.js +674 -0
- package/dist/embeddings/vertex.js +334 -0
- package/dist/eval/artifacts.js +42 -0
- package/dist/eval/cacheReport.js +146 -0
- package/dist/eval/comparison.js +182 -0
- package/dist/eval/context700-candidate.js +145 -0
- package/dist/eval/evalCache.js +568 -0
- package/dist/eval/hashEmbedding.js +35 -0
- package/dist/eval/indexSnapshots.js +548 -0
- package/dist/eval/longmemeval.js +56 -0
- package/dist/eval/metrics.js +181 -0
- package/dist/eval/pageDump.js +67 -0
- package/dist/eval/renorbenchBaseline.js +486 -0
- package/dist/eval/renorbenchDocuments.js +134 -0
- package/dist/eval/renorbenchProduct.js +182 -0
- package/dist/eval/runLedger.js +279 -0
- package/dist/eval/runManifest.js +485 -0
- package/dist/eval/scoring.js +197 -0
- package/dist/eval/textTokens.js +17 -0
- package/dist/evidence/attach.js +264 -0
- package/dist/evidence/background.js +147 -0
- package/dist/evidence/evidenceCache.js +182 -0
- package/dist/evidence/gate.js +37 -0
- package/dist/evidence/keyPool.js +85 -0
- package/dist/evidence/ollamaChat.js +84 -0
- package/dist/evidence/optin.js +165 -0
- package/dist/evidence/options.js +151 -0
- package/dist/evidence/secondPass.js +294 -0
- package/dist/extraction/classifyCapture.js +34 -0
- package/dist/extraction/decision-model.js +174 -0
- package/dist/extraction/eventExtractor.js +113 -0
- package/dist/extraction/extractionClient.js +122 -0
- package/dist/extraction/factExtractor.js +46 -0
- package/dist/extraction/factsWritePath.js +179 -0
- package/dist/extraction/preferenceExtractor.js +150 -0
- package/dist/extraction/secretGuard.js +176 -0
- package/dist/extraction/secretScanner.js +665 -0
- package/dist/extraction/typedFacts.js +224 -0
- package/dist/extraction/writePathAdjudication.js +110 -0
- package/dist/importing/apply.js +672 -0
- package/dist/importing/bundle-write.js +80 -0
- package/dist/importing/cli.js +408 -0
- package/dist/importing/existing.js +51 -0
- package/dist/importing/facts.js +121 -0
- package/dist/importing/forgetImported.js +79 -0
- package/dist/importing/hosted.js +467 -0
- package/dist/importing/journalLog.js +96 -0
- package/dist/importing/markdown.js +98 -0
- package/dist/importing/plan.js +201 -0
- package/dist/importing/secretScreen.js +113 -0
- package/dist/importing/serialize.js +188 -0
- package/dist/importing/source.js +463 -0
- package/dist/importing/types.js +6 -0
- package/dist/indexer/chunker.js +319 -0
- package/dist/indexer/dailyEntries.js +63 -0
- package/dist/indexer/hasher.js +10 -0
- package/dist/indexer/laneRuns.js +48 -0
- package/dist/indexer/o200k.js +481 -0
- package/dist/indexer/o200k.json +8 -0
- package/dist/indexer/outsideRoot.js +77 -0
- package/dist/indexer/parser.js +170 -0
- package/dist/indexer/pipeline.js +988 -0
- package/dist/indexer/staleness.js +201 -0
- package/dist/indexer/tokenizer.js +77 -0
- package/dist/indexer/walker.js +170 -0
- package/dist/mcp/hive.js +88 -0
- package/dist/mcp/requestErrors.js +47 -0
- package/dist/mcp/server.js +426 -0
- package/dist/memory/agentLease.js +43 -0
- package/dist/memory/confinedWrite.js +315 -0
- package/dist/memory/controls.js +73 -0
- package/dist/memory/dailyNote.js +265 -0
- package/dist/memory/durableAppend.js +57 -0
- package/dist/memory/factConflicts.js +203 -0
- package/dist/memory/factTerms.js +321 -0
- package/dist/memory/handoff.js +363 -0
- package/dist/memory/hiveNameHeader.cjs +10 -0
- package/dist/memory/inventory.js +55 -0
- package/dist/memory/invisibleText.js +20 -0
- package/dist/memory/lifecycle.js +1991 -0
- package/dist/memory/limits.js +13 -0
- package/dist/memory/requests.js +183 -0
- package/dist/memory/sourceName.js +276 -0
- package/dist/memory/statusFacts.js +61 -0
- package/dist/memory/whatsNew.js +158 -0
- package/dist/memory/writer.js +175 -0
- package/dist/memory/writerConflicts.js +85 -0
- package/dist/memorymap/memoryMap.js +1594 -0
- package/dist/memorymap/replacedJournal.js +58 -0
- package/dist/memorymap/replacements.js +410 -0
- package/dist/mvp/activity.js +86 -0
- package/dist/mvp/agentMemory.js +1800 -0
- package/dist/mvp/answerCheck.js +211 -0
- package/dist/mvp/auth.js +531 -0
- package/dist/mvp/cli.js +75 -0
- package/dist/mvp/client-store.js +462 -0
- package/dist/mvp/compileCache.js +242 -0
- package/dist/mvp/compileSeed.js +33 -0
- package/dist/mvp/config.js +288 -0
- package/dist/mvp/connectionStatus.js +80 -0
- package/dist/mvp/contextDistinct.js +53 -0
- package/dist/mvp/contextProfile.js +87 -0
- package/dist/mvp/contextRelevance.js +413 -0
- package/dist/mvp/directorySurface.js +68 -0
- package/dist/mvp/engine.js +1078 -0
- package/dist/mvp/exportAll.js +374 -0
- package/dist/mvp/factLanePack.js +77 -0
- package/dist/mvp/factMarkers.js +65 -0
- package/dist/mvp/factsLane.js +203 -0
- package/dist/mvp/hiddenSpans.js +585 -0
- package/dist/mvp/history.js +414 -0
- package/dist/mvp/hitDates.js +31 -0
- package/dist/mvp/hitFloor.js +36 -0
- package/dist/mvp/identity.js +18 -0
- package/dist/mvp/idle.js +113 -0
- package/dist/mvp/log.js +118 -0
- package/dist/mvp/memoryDir.js +98 -0
- package/dist/mvp/posthogMcp.js +210 -0
- package/dist/mvp/readScope.js +104 -0
- package/dist/mvp/readUsefulness.js +222 -0
- package/dist/mvp/sanitize.js +48 -0
- package/dist/mvp/schemaRejections.js +91 -0
- package/dist/mvp/scope.js +127 -0
- package/dist/mvp/server.js +2604 -0
- package/dist/mvp/serverCard.js +38 -0
- package/dist/mvp/storedSize.js +139 -0
- package/dist/mvp/telemetry.js +474 -0
- package/dist/net/poolFetch.js +174 -0
- package/dist/recall/actionRecall.js +282 -0
- package/dist/recall/autoRecall.js +447 -0
- package/dist/recall/searchRecall.js +189 -0
- package/dist/retrieval/adviceLines.js +195 -0
- package/dist/retrieval/adviceQuestion.js +44 -0
- package/dist/retrieval/conversationShape.js +44 -0
- package/dist/retrieval/dailyEntryTime.js +55 -0
- package/dist/retrieval/entityLookup.js +162 -0
- package/dist/retrieval/factsFirst.js +188 -0
- package/dist/retrieval/keyword.js +84 -0
- package/dist/retrieval/ledgerEventTime.js +51 -0
- package/dist/retrieval/ledgerLiveness.js +83 -0
- package/dist/retrieval/ledgerPresentation.js +161 -0
- package/dist/retrieval/noteIdentity.js +57 -0
- package/dist/retrieval/orchestrator.js +1171 -0
- package/dist/retrieval/plainWording.js +236 -0
- package/dist/retrieval/queryHelpers.js +142 -0
- package/dist/retrieval/questionSpan.js +475 -0
- package/dist/retrieval/readerEvents.js +690 -0
- package/dist/retrieval/readerLayout.js +422 -0
- package/dist/retrieval/relativeTimeHints.js +113 -0
- package/dist/retrieval/semantic.js +67 -0
- package/dist/router/routerCache.js +106 -0
- package/dist/router/sessionRouter.js +726 -0
- package/dist/toolRefusal.js +120 -0
- package/dist/watcher/watcher.js +77 -0
- package/package.json +118 -4
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.toRelative = toRelative;
|
|
7
|
+
exports.isSelected = isSelected;
|
|
8
|
+
exports.readRenorIndexSource = readRenorIndexSource;
|
|
9
|
+
exports.readBundleSource = readBundleSource;
|
|
10
|
+
exports.readSource = readSource;
|
|
11
|
+
const fs_1 = __importDefault(require("fs"));
|
|
12
|
+
const path_1 = __importDefault(require("path"));
|
|
13
|
+
const client_1 = require("../db/client");
|
|
14
|
+
const types_1 = require("./types");
|
|
15
|
+
/** Relative path with forward slashes, or the input unchanged when it is not under the root. */
|
|
16
|
+
function toRelative(sourceRoot, absolutePath) {
|
|
17
|
+
const root = path_1.default.resolve(sourceRoot);
|
|
18
|
+
const abs = path_1.default.resolve(absolutePath);
|
|
19
|
+
const rel = path_1.default.relative(root, abs);
|
|
20
|
+
if (rel === '' || rel.startsWith('..') || path_1.default.isAbsolute(rel))
|
|
21
|
+
return absolutePath;
|
|
22
|
+
return rel.split(path_1.default.sep).join('/');
|
|
23
|
+
}
|
|
24
|
+
function matchesPrefix(relPath, prefixes) {
|
|
25
|
+
return prefixes.some(prefix => {
|
|
26
|
+
const clean = prefix.replace(/^(\.?\/)+/, '').replace(/\/+$/, '');
|
|
27
|
+
return relPath === clean || relPath.startsWith(clean + '/');
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
/** Selection is explicit: a path must pass `include` (when given) and survive `exclude`. */
|
|
31
|
+
function isSelected(relPath, filter) {
|
|
32
|
+
if (filter.include.length > 0 && !matchesPrefix(relPath, filter.include))
|
|
33
|
+
return false;
|
|
34
|
+
if (filter.exclude.length > 0 && matchesPrefix(relPath, filter.exclude))
|
|
35
|
+
return false;
|
|
36
|
+
return true;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Read a Renor-shaped index (any store that uses the engine's `notes`/`chunks`/`facts`
|
|
40
|
+
* tables, including another Renor workspace or the personal brain) as an import bundle.
|
|
41
|
+
*
|
|
42
|
+
* The database is opened `readonly` so a source can never be written by an import —
|
|
43
|
+
* including the WAL side files a write-capable open would touch.
|
|
44
|
+
*/
|
|
45
|
+
function readRenorIndexSource(options) {
|
|
46
|
+
// Through the sanctioned opener (`src/db/client.ts`), which enforces read-only at the
|
|
47
|
+
// SQLite level so an import can never write to the store it is reading.
|
|
48
|
+
const db = (0, client_1.openReadOnlyDb)(options.dbPath);
|
|
49
|
+
try {
|
|
50
|
+
const filter = {
|
|
51
|
+
include: options.include ?? [],
|
|
52
|
+
exclude: options.exclude ?? [],
|
|
53
|
+
limit: options.limit,
|
|
54
|
+
};
|
|
55
|
+
const allNotes = db.prepare('SELECT path, title, kind, modified_at, frontmatter_json FROM notes').all();
|
|
56
|
+
const selected = [];
|
|
57
|
+
let filteredOut = 0;
|
|
58
|
+
for (const row of allNotes) {
|
|
59
|
+
const rel = toRelative(options.sourceRoot, row.path);
|
|
60
|
+
if (!isSelected(rel, filter)) {
|
|
61
|
+
filteredOut += 1;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
// The lifecycle's synthetic `_mcp_` row is not a note document.
|
|
65
|
+
if (rel === '_mcp_') {
|
|
66
|
+
filteredOut += 1;
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
selected.push({ row, rel });
|
|
70
|
+
}
|
|
71
|
+
selected.sort((a, b) => a.rel.localeCompare(b.rel));
|
|
72
|
+
const limited = options.limit !== undefined && options.limit >= 0
|
|
73
|
+
? selected.slice(0, options.limit)
|
|
74
|
+
: selected;
|
|
75
|
+
const chunksByNote = new Map();
|
|
76
|
+
const entitiesByNote = new Map();
|
|
77
|
+
const contentsByNote = new Map();
|
|
78
|
+
const chunkStmt = db.prepare('SELECT note_path, heading_path, text, token_count, start_line, end_line FROM chunks WHERE note_path = ? ORDER BY id');
|
|
79
|
+
// The lifecycle sentinel owns canonical entities; carrying those would duplicate
|
|
80
|
+
// the destination's own lifecycle entity for the same subject.
|
|
81
|
+
const entityStmt = db.prepare(`SELECT type, canonical_name, aliases_json, confidence FROM entities
|
|
82
|
+
WHERE source_note = ? AND source_note != '_mcp_'`);
|
|
83
|
+
const contentRoot = path_1.default.resolve(options.contentRoot ?? options.sourceRoot);
|
|
84
|
+
for (const { row, rel } of limited) {
|
|
85
|
+
const rows = chunkStmt.all(row.path);
|
|
86
|
+
chunksByNote.set(row.path, rows.map(r => ({
|
|
87
|
+
headingPath: r.heading_path ?? '',
|
|
88
|
+
text: r.text,
|
|
89
|
+
tokenCount: r.token_count,
|
|
90
|
+
// The same span rule as the bundle adapter: a coordinate that cannot be a citation
|
|
91
|
+
// is dropped rather than inherited into the destination's `chunks` rows.
|
|
92
|
+
...normalizedSpan(r.start_line, r.end_line),
|
|
93
|
+
})));
|
|
94
|
+
const entityRows = entityStmt.all(row.path);
|
|
95
|
+
entitiesByNote.set(row.path, entityRows.map(e => ({
|
|
96
|
+
type: e.type,
|
|
97
|
+
canonicalName: e.canonical_name,
|
|
98
|
+
aliases: parseAliases(e.aliases_json),
|
|
99
|
+
confidence: typeof e.confidence === 'number' ? e.confidence : 1,
|
|
100
|
+
})));
|
|
101
|
+
// Prefer the source's own bytes: they are the source of truth, they keep the
|
|
102
|
+
// stored chunk line ranges truthful, and copying a document is not re-parsing it.
|
|
103
|
+
const original = readOriginalContent(contentRoot, rel);
|
|
104
|
+
if (original !== null)
|
|
105
|
+
contentsByNote.set(row.path, original);
|
|
106
|
+
}
|
|
107
|
+
const notes = limited.map(({ row, rel }) => ({
|
|
108
|
+
sourcePath: row.path,
|
|
109
|
+
relPath: rel,
|
|
110
|
+
title: row.title,
|
|
111
|
+
kind: row.kind ?? null,
|
|
112
|
+
modifiedAt: row.modified_at,
|
|
113
|
+
frontmatter: parseFrontmatter(row.frontmatter_json),
|
|
114
|
+
chunks: chunksByNote.get(row.path) ?? [],
|
|
115
|
+
entities: entitiesByNote.get(row.path) ?? [],
|
|
116
|
+
originalContent: contentsByNote.get(row.path),
|
|
117
|
+
}));
|
|
118
|
+
// Only CURRENT facts are carried: a superseded value is history the destination
|
|
119
|
+
// would have to replay in order, and the source's own ledger is the authority for
|
|
120
|
+
// its lifecycle. Carrying closed rows would duplicate history under new ids.
|
|
121
|
+
const factsAvailable = db.prepare('SELECT count(*) AS c FROM facts WHERE valid_to IS NULL').get().c;
|
|
122
|
+
const factRows = db.prepare(`SELECT e.canonical_name AS subject, f.predicate AS predicate, f.object_text AS object_text,
|
|
123
|
+
f.scope AS scope, f.valid_from AS valid_from, f.valid_to AS valid_to,
|
|
124
|
+
f.source_path AS source_path
|
|
125
|
+
FROM facts f
|
|
126
|
+
JOIN entities e ON e.id = f.subject_entity_id
|
|
127
|
+
WHERE f.valid_to IS NULL AND f.object_text IS NOT NULL AND e.canonical_name IS NOT NULL
|
|
128
|
+
ORDER BY f.valid_from DESC, f.id DESC`).all();
|
|
129
|
+
const selectedNotePaths = new Set(limited.map(({ row }) => row.path));
|
|
130
|
+
const facts = [];
|
|
131
|
+
for (const row of factRows) {
|
|
132
|
+
const isLedgerFact = row.source_path === '_mcp_';
|
|
133
|
+
// Ledger-derived facts are the owner's explicitly stated memory: a small, global,
|
|
134
|
+
// high-value set. They are carried by default (the point of an import is to see
|
|
135
|
+
// what Renor already holds) and skipped only when the caller excludes them.
|
|
136
|
+
if (isLedgerFact) {
|
|
137
|
+
if (matchesPrefix('_mcp_', filter.exclude) || matchesPrefix('ledger', filter.exclude))
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
else if (!selectedNotePaths.has(row.source_path)) {
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
facts.push({
|
|
144
|
+
subject: String(row.subject),
|
|
145
|
+
predicate: String(row.predicate),
|
|
146
|
+
object: String(row.object_text),
|
|
147
|
+
scope: row.scope ?? 'global',
|
|
148
|
+
validFrom: row.valid_from,
|
|
149
|
+
sourcePath: row.source_path,
|
|
150
|
+
origin: isLedgerFact ? 'ledger' : 'note',
|
|
151
|
+
ledgerSource: isLedgerFact ? 'import:renor-index' : 'import:renor-index',
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
format: types_1.BUNDLE_FORMAT,
|
|
156
|
+
sourceLabel: options.label ?? 'renor-index',
|
|
157
|
+
notes,
|
|
158
|
+
facts,
|
|
159
|
+
stats: {
|
|
160
|
+
notesAvailable: allNotes.length,
|
|
161
|
+
notesSelected: notes.length,
|
|
162
|
+
notesFilteredOut: filteredOut + (selected.length - limited.length),
|
|
163
|
+
notesWithOriginalBytes: notes.filter(n => n.originalContent !== undefined).length,
|
|
164
|
+
chunksSelected: notes.reduce((sum, n) => sum + n.chunks.length, 0),
|
|
165
|
+
factsAvailable,
|
|
166
|
+
factsSelected: facts.length,
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
finally {
|
|
171
|
+
db.close();
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* The source note's original bytes, when the content tree is reachable. Returns null
|
|
176
|
+
* when the file is absent, so the caller falls back to reconstruction and reports it.
|
|
177
|
+
*/
|
|
178
|
+
function readOriginalContent(contentRoot, relPath) {
|
|
179
|
+
const candidate = path_1.default.resolve(contentRoot, relPath);
|
|
180
|
+
const root = path_1.default.resolve(contentRoot);
|
|
181
|
+
const rel = path_1.default.relative(root, candidate);
|
|
182
|
+
if (rel === '' || rel.startsWith('..') || path_1.default.isAbsolute(rel))
|
|
183
|
+
return null;
|
|
184
|
+
try {
|
|
185
|
+
return fs_1.default.readFileSync(candidate, 'utf8');
|
|
186
|
+
}
|
|
187
|
+
catch {
|
|
188
|
+
return null;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
function parseAliases(raw) {
|
|
192
|
+
try {
|
|
193
|
+
const parsed = JSON.parse(raw);
|
|
194
|
+
return Array.isArray(parsed) ? parsed.filter((v) => typeof v === 'string') : [];
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
return [];
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
function parseFrontmatter(raw) {
|
|
201
|
+
try {
|
|
202
|
+
const parsed = JSON.parse(raw);
|
|
203
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
204
|
+
? parsed
|
|
205
|
+
: {};
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
return {};
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Read a `renor-memory-export/v1` JSONL bundle written by any provider.
|
|
213
|
+
* One JSON record per line: `{"type":"note",…}` / `{"type":"fact",…}` / `{"type":"meta",…}`.
|
|
214
|
+
*
|
|
215
|
+
* `filter` is applied HERE, in the adapter, exactly as the index adapter applies it —
|
|
216
|
+
* `include`/`exclude` select note records by `relPath`, `limit` caps the selection, and
|
|
217
|
+
* `_mcp_`/`ledger` in `exclude` drops the owner's ledger-derived facts. An adapter that
|
|
218
|
+
* returned everything would make the CLI's selection flags a silent no-op, and
|
|
219
|
+
* `--exclude Private` would import the private notes anyway. `filter` is optional so the
|
|
220
|
+
* adapter can still be used as a plain bundle reader.
|
|
221
|
+
*/
|
|
222
|
+
function readBundleSource(filePath, filter = { include: [], exclude: [] }) {
|
|
223
|
+
const include = filter.include ?? [];
|
|
224
|
+
const exclude = filter.exclude ?? [];
|
|
225
|
+
const raw = fs_1.default.readFileSync(filePath, 'utf8');
|
|
226
|
+
const parsedNotes = [];
|
|
227
|
+
const parsedFacts = [];
|
|
228
|
+
let sourceLabel = path_1.default.basename(filePath, path_1.default.extname(filePath));
|
|
229
|
+
let format = types_1.BUNDLE_FORMAT;
|
|
230
|
+
const lines = raw.split('\n');
|
|
231
|
+
for (let i = 0; i < lines.length; i++) {
|
|
232
|
+
const line = lines[i].trim();
|
|
233
|
+
if (!line)
|
|
234
|
+
continue;
|
|
235
|
+
let record;
|
|
236
|
+
try {
|
|
237
|
+
record = JSON.parse(line);
|
|
238
|
+
}
|
|
239
|
+
catch {
|
|
240
|
+
throw new Error(`bundle line ${i + 1} is not valid JSON`);
|
|
241
|
+
}
|
|
242
|
+
if (record?.type === 'meta') {
|
|
243
|
+
if (typeof record.format === 'string')
|
|
244
|
+
format = record.format;
|
|
245
|
+
if (typeof record.source === 'string')
|
|
246
|
+
sourceLabel = record.source;
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
if (record?.type === 'note') {
|
|
250
|
+
// Every field that reaches canonical bytes is validated where it is read: `String(x)`
|
|
251
|
+
// accepted anything, so a malformed record became the literal text `[object Object]`,
|
|
252
|
+
// which is then written into Markdown, indexed, and (for a fact) recorded in the
|
|
253
|
+
// ledger as though it were memory.
|
|
254
|
+
const note = `note ${bundleLabel(record.relPath ?? record.path)}`;
|
|
255
|
+
const at = bundleWhere(note, i);
|
|
256
|
+
const relPathFromRecord = record.relPath ?? record.path;
|
|
257
|
+
parsedNotes.push({
|
|
258
|
+
sourcePath: bundleString(record.path, '', 'path', at),
|
|
259
|
+
relPath: bundleString(relPathFromRecord, '', 'relPath', at),
|
|
260
|
+
title: bundleString(record.title, '', 'title', at),
|
|
261
|
+
kind: typeof record.kind === 'string' ? record.kind : null,
|
|
262
|
+
modifiedAt: bundleString(record.modifiedAt, '', 'modifiedAt', at),
|
|
263
|
+
frontmatter: record.frontmatter && typeof record.frontmatter === 'object' ? record.frontmatter : {},
|
|
264
|
+
chunks: Array.isArray(record.chunks)
|
|
265
|
+
? record.chunks.map((c, index) => {
|
|
266
|
+
const chunk = bundleWhere(note, i, `chunk ${index}`);
|
|
267
|
+
return {
|
|
268
|
+
headingPath: bundleString(c?.headingPath, '', 'headingPath', chunk),
|
|
269
|
+
text: bundleString(c?.text, '', 'text', chunk),
|
|
270
|
+
tokenCount: bundleNumber(c?.tokenCount, 0, 'tokenCount', chunk),
|
|
271
|
+
// Spans must survive the round trip: they are the only fallback for a chunk
|
|
272
|
+
// whose text cannot be located, so dropping them would silently make a
|
|
273
|
+
// bundle-sourced import behave differently from a direct index read. A span
|
|
274
|
+
// that cannot be a citation — negative, fractional, or reversed — is dropped
|
|
275
|
+
// here instead of being inherited as a coordinate.
|
|
276
|
+
...normalizedSpan(c?.startLine, c?.endLine),
|
|
277
|
+
};
|
|
278
|
+
})
|
|
279
|
+
: [],
|
|
280
|
+
entities: Array.isArray(record.entities)
|
|
281
|
+
? record.entities.map((e, index) => {
|
|
282
|
+
const entity = bundleWhere(note, i, `entity ${index}`);
|
|
283
|
+
return {
|
|
284
|
+
type: bundleString(e?.type, 'note', 'type', entity),
|
|
285
|
+
canonicalName: bundleString(e?.canonicalName, '', 'canonicalName', entity),
|
|
286
|
+
aliases: Array.isArray(e?.aliases)
|
|
287
|
+
? e.aliases.map((a, aliasIndex) => (bundleString(a, '', `alias ${aliasIndex}`, entity)))
|
|
288
|
+
: [],
|
|
289
|
+
confidence: bundleNumber(e?.confidence, 1, 'confidence', entity),
|
|
290
|
+
};
|
|
291
|
+
})
|
|
292
|
+
: [],
|
|
293
|
+
originalContent: typeof record.originalContent === 'string' ? record.originalContent : undefined,
|
|
294
|
+
});
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
if (record?.type === 'fact') {
|
|
298
|
+
// `String(x)` on a fact field fabricated `[object Object]` as a canonical ledger value.
|
|
299
|
+
const at = bundleWhere('fact', i);
|
|
300
|
+
parsedFacts.push({
|
|
301
|
+
subject: bundleString(record.subject, '', 'subject', at),
|
|
302
|
+
predicate: bundleString(record.predicate, '', 'predicate', at),
|
|
303
|
+
object: bundleString(record.object, '', 'object', at),
|
|
304
|
+
scope: String(record.scope ?? 'global'),
|
|
305
|
+
validFrom: String(record.validFrom ?? ''),
|
|
306
|
+
sourcePath: String(record.sourcePath ?? ''),
|
|
307
|
+
origin: record.origin === 'ledger' ? 'ledger' : 'note',
|
|
308
|
+
eventId: typeof record.eventId === 'string' ? record.eventId : undefined,
|
|
309
|
+
closedEvent: (record.closedEvent ?? null), // carried unvalidated, as before
|
|
310
|
+
ledgerSource: typeof record.ledgerSource === 'string' ? record.ledgerSource : undefined,
|
|
311
|
+
});
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
throw new Error(`bundle line ${i + 1} has unknown record type ${JSON.stringify(record?.type)}`);
|
|
315
|
+
}
|
|
316
|
+
if (format !== types_1.BUNDLE_FORMAT) {
|
|
317
|
+
throw new Error(`unsupported bundle format ${JSON.stringify(format)}; expected ${types_1.BUNDLE_FORMAT}`);
|
|
318
|
+
}
|
|
319
|
+
// Selection, in the same order and with the same rule the index adapter uses: the index
|
|
320
|
+
// adapter sorts by relative path BEFORE applying `limit`, so a bundle is sorted here too
|
|
321
|
+
// and `--limit N` selects the same records whichever order the provider wrote them in.
|
|
322
|
+
const selection = { include, exclude };
|
|
323
|
+
const selected = [];
|
|
324
|
+
let notesFilteredOut = 0;
|
|
325
|
+
for (const note of parsedNotes) {
|
|
326
|
+
if (!isSelected(note.relPath, selection)) {
|
|
327
|
+
notesFilteredOut += 1;
|
|
328
|
+
continue;
|
|
329
|
+
}
|
|
330
|
+
selected.push(note);
|
|
331
|
+
}
|
|
332
|
+
selected.sort((a, b) => a.relPath.localeCompare(b.relPath));
|
|
333
|
+
const notes = filter.limit !== undefined && filter.limit >= 0
|
|
334
|
+
? selected.slice(0, filter.limit)
|
|
335
|
+
: selected;
|
|
336
|
+
notesFilteredOut += selected.length - notes.length;
|
|
337
|
+
// Facts follow the notes: a note-derived fact for a note that was not selected is not
|
|
338
|
+
// carried, and the owner's ledger-derived facts (`_mcp_`) are carried by default and
|
|
339
|
+
// dropped only when explicitly excluded — the same rule the index adapter applies.
|
|
340
|
+
// Both a note's source path and its relative path count as naming it, because a
|
|
341
|
+
// provider may write either.
|
|
342
|
+
const selectedPaths = new Set();
|
|
343
|
+
for (const note of notes) {
|
|
344
|
+
selectedPaths.add(note.sourcePath);
|
|
345
|
+
selectedPaths.add(note.relPath);
|
|
346
|
+
}
|
|
347
|
+
const facts = [];
|
|
348
|
+
for (const fact of parsedFacts) {
|
|
349
|
+
if (fact.sourcePath === '_mcp_') {
|
|
350
|
+
if (matchesPrefix('_mcp_', exclude) || matchesPrefix('ledger', exclude))
|
|
351
|
+
continue;
|
|
352
|
+
facts.push(fact);
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
// A fact that names no path at all carries no note association that selection could
|
|
356
|
+
// be applied to; one that names a path must belong to a selected note.
|
|
357
|
+
if (fact.sourcePath !== '' && !selectedPaths.has(fact.sourcePath))
|
|
358
|
+
continue;
|
|
359
|
+
facts.push(fact);
|
|
360
|
+
}
|
|
361
|
+
return {
|
|
362
|
+
format,
|
|
363
|
+
sourceLabel,
|
|
364
|
+
notes,
|
|
365
|
+
facts,
|
|
366
|
+
stats: {
|
|
367
|
+
notesAvailable: parsedNotes.length,
|
|
368
|
+
notesSelected: notes.length,
|
|
369
|
+
notesFilteredOut,
|
|
370
|
+
notesWithOriginalBytes: notes.filter(n => n.originalContent !== undefined).length,
|
|
371
|
+
chunksSelected: notes.reduce((sum, n) => sum + n.chunks.length, 0),
|
|
372
|
+
factsAvailable: parsedFacts.length,
|
|
373
|
+
factsSelected: facts.length,
|
|
374
|
+
},
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
/** A label for a record whose own identity may be malformed, safe to print in an error. */
|
|
378
|
+
function bundleLabel(raw) {
|
|
379
|
+
return typeof raw === 'string' ? JSON.stringify(raw) : JSON.stringify(raw ?? '');
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* A string field of a bundle record, validated where the record is read.
|
|
383
|
+
*
|
|
384
|
+
* An absent/null field keeps its default, so existing bundles stay readable; a
|
|
385
|
+
* present-but-non-string field is refused, naming the line, the record and the field.
|
|
386
|
+
*/
|
|
387
|
+
function bundleString(raw, fallback, field, where) {
|
|
388
|
+
if (raw === undefined || raw === null)
|
|
389
|
+
return fallback;
|
|
390
|
+
if (typeof raw !== 'string') {
|
|
391
|
+
throw new Error(`${where} has an invalid ${field} ${JSON.stringify(raw)}: expected a string`);
|
|
392
|
+
}
|
|
393
|
+
return raw;
|
|
394
|
+
}
|
|
395
|
+
/** Where a bundle record's field lives, so a malformed value is named precisely. */
|
|
396
|
+
function bundleWhere(origin, lineIndex, part) {
|
|
397
|
+
return `bundle line ${lineIndex + 1}: ${origin}${part ? ` ${part}` : ''}`;
|
|
398
|
+
}
|
|
399
|
+
/**
|
|
400
|
+
* A line span, or nothing when it cannot be a citation.
|
|
401
|
+
*
|
|
402
|
+
* A negative or fractional span was copied verbatim and could be inherited into
|
|
403
|
+
* `chunks.start_line`/`end_line` and handed back as a citation range, so a span that is not
|
|
404
|
+
* a non-negative, ordered integer pair is treated exactly like "text not located": unknown.
|
|
405
|
+
*/
|
|
406
|
+
function normalizedSpan(startLine, endLine) {
|
|
407
|
+
if (typeof startLine !== 'number' || typeof endLine !== 'number')
|
|
408
|
+
return {};
|
|
409
|
+
if (!Number.isInteger(startLine) || !Number.isInteger(endLine))
|
|
410
|
+
return {};
|
|
411
|
+
if (startLine < 0 || endLine < startLine)
|
|
412
|
+
return {};
|
|
413
|
+
return { startLine, endLine };
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* A numeric field of a bundle record, validated where the record is read.
|
|
417
|
+
*
|
|
418
|
+
* `Number(record.field ?? default)` accepted anything: a malformed `tokenCount` became
|
|
419
|
+
* `NaN`, travelled through the plan, and only failed at the write layer as
|
|
420
|
+
* `NOT NULL constraint failed: chunks.token_count` — after canonical note bytes had been
|
|
421
|
+
* written, and reported as a *ledger replay* rejection. Invalid input is a property of the
|
|
422
|
+
* bundle, so it is rejected here, before any write, naming the line, the note and the
|
|
423
|
+
* field. An absent/null field keeps its default, so existing bundles stay readable.
|
|
424
|
+
*/
|
|
425
|
+
function bundleNumber(raw, fallback, field, where) {
|
|
426
|
+
if (raw === undefined || raw === null)
|
|
427
|
+
return fallback;
|
|
428
|
+
const value = typeof raw === 'number'
|
|
429
|
+
? raw
|
|
430
|
+
: typeof raw === 'string' && raw.trim() !== '' ? Number(raw) : Number.NaN;
|
|
431
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
432
|
+
throw new Error(`${where} has an invalid ${field} ${JSON.stringify(raw)}: expected a non-negative finite number`);
|
|
433
|
+
}
|
|
434
|
+
return value;
|
|
435
|
+
}
|
|
436
|
+
/** Open a source from a `kind:path` spec. */
|
|
437
|
+
function readSource(spec, filter) {
|
|
438
|
+
const separator = spec.indexOf(':');
|
|
439
|
+
if (separator < 0) {
|
|
440
|
+
throw new Error('source must be given as kind:path (for example renor-index:/path/index.db)');
|
|
441
|
+
}
|
|
442
|
+
const kind = spec.slice(0, separator);
|
|
443
|
+
const target = spec.slice(separator + 1);
|
|
444
|
+
if (kind === 'renor-index') {
|
|
445
|
+
return readRenorIndexSource({
|
|
446
|
+
dbPath: target,
|
|
447
|
+
sourceRoot: filter.sourceRoot ?? path_1.default.dirname(path_1.default.dirname(target)),
|
|
448
|
+
contentRoot: filter.contentRoot,
|
|
449
|
+
include: filter.include,
|
|
450
|
+
exclude: filter.exclude,
|
|
451
|
+
limit: filter.limit,
|
|
452
|
+
label: filter.label,
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
if (kind === 'bundle') {
|
|
456
|
+
return readBundleSource(target, {
|
|
457
|
+
include: filter.include,
|
|
458
|
+
exclude: filter.exclude,
|
|
459
|
+
limit: filter.limit,
|
|
460
|
+
});
|
|
461
|
+
}
|
|
462
|
+
throw new Error(`unknown source kind ${JSON.stringify(kind)}; expected renor-index or bundle`);
|
|
463
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/** Portable memory bundle: the interface every import source must produce. */
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.BUNDLE_FORMAT = void 0;
|
|
5
|
+
/** The bundle format this module reads and writes. */
|
|
6
|
+
exports.BUNDLE_FORMAT = 'renor-memory-export/v1';
|