renor-engine 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -661
- package/NOTICE +8 -6
- package/README.md +13 -8
- package/dist/.tsbuildinfo +1 -0
- package/dist/api/server.js +10 -8
- package/dist/buildIdentity.js +18 -1
- package/dist/callPhases.js +82 -0
- package/dist/cli.js +36 -1
- package/dist/context/rowStore.js +1 -1
- package/dist/conversation/parser.js +40 -36
- package/dist/db/client.js +6 -4
- package/dist/db/handoffTable.js +31 -0
- package/dist/db/indexSalvage.js +24 -4
- package/dist/db/migrations.js +30 -2
- package/dist/db/sqlite.js +396 -0
- package/dist/digests/digestCache.js +6 -2
- package/dist/digests/digests.js +3 -3
- package/dist/digests/worker.js +3 -1
- package/dist/embeddings/caching.js +5 -5
- package/dist/embeddings/local.js +5 -3
- package/dist/embeddings/openai.js +4 -3
- package/dist/embeddings/simdScan.js +3 -1
- package/dist/embeddings/vectorIndex.js +0 -1
- package/dist/embeddings/vertex.js +123 -13
- package/dist/eval/cacheReport.js +7 -0
- package/dist/eval/context700-candidate.js +5 -5
- package/dist/eval/evalCache.js +46 -3
- package/dist/eval/indexSnapshots.js +549 -0
- package/dist/eval/pageDump.js +67 -0
- package/dist/eval/runManifest.js +6 -0
- package/dist/eval/textTokens.js +17 -0
- package/dist/evidence/attach.js +0 -3
- package/dist/evidence/evidenceCache.js +3 -3
- package/dist/evidence/ollamaChat.js +1 -1
- package/dist/evidence/options.js +46 -25
- package/dist/evidence/secondPass.js +8 -170
- package/dist/extraction/factExtractor.js +1 -1
- package/dist/extraction/jevDecisionModel.js +307 -0
- package/dist/extraction/preferenceExtractor.js +1 -4
- package/dist/extraction/secretGuard.js +7 -3
- package/dist/extraction/typedFacts.js +1 -1
- package/dist/importing/apply.js +42 -14
- package/dist/importing/cli.js +9 -8
- package/dist/importing/existing.js +2 -0
- package/dist/importing/facts.js +0 -2
- package/dist/importing/plan.js +0 -5
- package/dist/importing/source.js +1 -1
- package/dist/indexer/parser.js +4 -1
- package/dist/indexer/pipeline.js +99 -279
- package/dist/indexer/walker.js +11 -25
- package/dist/learning/cli.js +113 -0
- package/dist/learning/schema.js +144 -0
- package/dist/learning/settings.js +45 -0
- package/dist/learning/store.js +267 -0
- package/dist/learning/tap.js +401 -0
- package/dist/mcp/server.js +36 -6
- package/dist/memory/dreamer/adjudicate.js +215 -0
- package/dist/memory/dreamer/apply.js +120 -0
- package/dist/memory/dreamer/budget.js +64 -0
- package/dist/memory/dreamer/candidates.js +149 -0
- package/dist/memory/dreamer/collection.js +104 -0
- package/dist/memory/dreamer/dreamer.js +352 -0
- package/dist/memory/dreamer/review.js +111 -0
- package/dist/memory/factConflicts.js +2 -0
- package/dist/memory/handoff.js +363 -0
- package/dist/memory/lifecycle.js +338 -80
- package/dist/memory/sourceName.js +77 -1
- package/dist/memory/statusFacts.js +61 -0
- package/dist/memory/writer.js +3 -2
- package/dist/memorymap/memoryMap.js +39 -13
- package/dist/memorymap/replacements.js +8 -3
- package/dist/mvp/agentMemory.js +109 -50
- package/dist/mvp/answerCheck.js +211 -0
- package/dist/mvp/cli.js +14 -1
- package/dist/mvp/compileCache.js +3 -10
- package/dist/mvp/compileSeed.js +9 -3
- package/dist/mvp/config.js +4 -0
- package/dist/mvp/contextDistinct.js +53 -0
- package/dist/mvp/contextProfile.js +87 -0
- package/dist/mvp/directorySurface.js +68 -0
- package/dist/mvp/engine.js +39 -26
- package/dist/mvp/exportAll.js +36 -2
- package/dist/mvp/factMarkers.js +18 -2
- package/dist/mvp/history.js +5 -4
- package/dist/mvp/log.js +118 -0
- package/dist/mvp/server.js +759 -136
- package/dist/mvp/serverCard.js +12 -3
- package/dist/mvp/telemetry.js +35 -2
- package/dist/net/poolFetch.js +174 -0
- package/dist/recall/actionRecall.js +282 -0
- package/dist/recall/autoRecall.js +447 -0
- package/dist/recall/searchRecall.js +189 -0
- package/dist/retrieval/entityLookup.js +53 -4
- package/dist/retrieval/orchestrator.js +148 -10
- package/dist/retrieval/plainWording.js +236 -0
- package/dist/retrieval/rankingTrace.js +184 -0
- package/dist/retrieval/readerLayout.js +143 -224
- package/dist/router/routerCache.js +2 -2
- package/dist/sharing/cli.js +181 -0
- package/dist/sharing/examples.js +136 -0
- package/dist/sharing/fields.js +225 -0
- package/dist/sharing/loop.js +80 -0
- package/dist/sharing/project.js +327 -0
- package/dist/sharing/send.js +466 -0
- package/dist/sharing/state.js +222 -0
- package/dist/sharing/wording.js +48 -0
- package/dist/watcher/watcher.js +5 -3
- package/package.json +5 -7
- package/dist/BUILD.json +0 -6
- package/dist/importing/index.js +0 -29
package/dist/api/server.js
CHANGED
|
@@ -17,6 +17,8 @@ const options_1 = require("../evidence/options");
|
|
|
17
17
|
const composer_1 = require("../context/composer");
|
|
18
18
|
const readHonesty_1 = require("../context/readHonesty");
|
|
19
19
|
const buildIdentity_1 = require("../buildIdentity");
|
|
20
|
+
const log_1 = require("../mvp/log");
|
|
21
|
+
const log = (0, log_1.createLogger)('engine');
|
|
20
22
|
function createServer(db, vectorIndex, provider, vaultPath, priorityPaths, memoryDir, ledgerPath, options) {
|
|
21
23
|
// Fail closed: there is no unauthenticated way to build this server.
|
|
22
24
|
const apiToken = options?.apiToken?.trim();
|
|
@@ -48,8 +50,8 @@ function createServer(db, vectorIndex, provider, vaultPath, priorityPaths, memor
|
|
|
48
50
|
// at build time by scripts/ci/stamp-build.mjs and read through the shared,
|
|
49
51
|
// non-throwing helper (src/buildIdentity.ts) that the MVP's GET /version also
|
|
50
52
|
// uses. Deploy drift must be observable; the manifest never throws.
|
|
51
|
-
app.get('/api/health', (
|
|
52
|
-
res.json({ ok: true, name: 'renor', version:
|
|
53
|
+
app.get('/api/health', (_req, res) => {
|
|
54
|
+
res.json({ ok: true, name: 'renor', version: buildIdentity_1.ENGINE_VERSION, ...(0, buildIdentity_1.readBuildIdentity)() });
|
|
53
55
|
});
|
|
54
56
|
app.post('/retrieve-context', async (req, res) => {
|
|
55
57
|
const { query, topK: rawTopK = 20 } = req.body;
|
|
@@ -100,7 +102,7 @@ function createServer(db, vectorIndex, provider, vaultPath, priorityPaths, memor
|
|
|
100
102
|
});
|
|
101
103
|
}
|
|
102
104
|
catch (e) {
|
|
103
|
-
res.status(500).json({ error: e.message });
|
|
105
|
+
res.status(500).json({ error: e instanceof Error ? e.message : String(e) });
|
|
104
106
|
}
|
|
105
107
|
});
|
|
106
108
|
app.get('/entity/:name', (req, res) => {
|
|
@@ -124,7 +126,7 @@ function createServer(db, vectorIndex, provider, vaultPath, priorityPaths, memor
|
|
|
124
126
|
res.json({ ok: true });
|
|
125
127
|
}
|
|
126
128
|
catch (e) {
|
|
127
|
-
res.status(500).json({ error: e.message });
|
|
129
|
+
res.status(500).json({ error: e instanceof Error ? e.message : String(e) });
|
|
128
130
|
}
|
|
129
131
|
});
|
|
130
132
|
// Lifecycle-backed durable memory (ledger-first, supersedes, idempotent, returns receipt).
|
|
@@ -143,7 +145,7 @@ function createServer(db, vectorIndex, provider, vaultPath, priorityPaths, memor
|
|
|
143
145
|
// an escaping ledger path — still throw plain Errors and stay 500.
|
|
144
146
|
if (e instanceof lifecycle_1.LifecycleInputError)
|
|
145
147
|
return void res.status(400).json({ error: e.message });
|
|
146
|
-
res.status(500).json({ error: e.message });
|
|
148
|
+
res.status(500).json({ error: e instanceof Error ? e.message : String(e) });
|
|
147
149
|
}
|
|
148
150
|
});
|
|
149
151
|
// Facts by entity NAME (agents don't know DB ids). ?asOf=ISO&history=1
|
|
@@ -155,7 +157,7 @@ function createServer(db, vectorIndex, provider, vaultPath, priorityPaths, memor
|
|
|
155
157
|
}));
|
|
156
158
|
}
|
|
157
159
|
catch (e) {
|
|
158
|
-
res.status(500).json({ error: e.message });
|
|
160
|
+
res.status(500).json({ error: e instanceof Error ? e.message : String(e) });
|
|
159
161
|
}
|
|
160
162
|
});
|
|
161
163
|
// One generic answer for anything a route did not handle: Express's own error page names the absolute path of the
|
|
@@ -163,9 +165,9 @@ function createServer(db, vectorIndex, provider, vaultPath, priorityPaths, memor
|
|
|
163
165
|
app.use((err, _req, res, next) => {
|
|
164
166
|
if (res.headersSent)
|
|
165
167
|
return next(err);
|
|
166
|
-
const status = Number.isInteger(err
|
|
168
|
+
const status = typeof err?.status === 'number' && Number.isInteger(err.status) && err.status >= 400 && err.status < 500 ? err.status : 500;
|
|
167
169
|
if (status >= 500)
|
|
168
|
-
|
|
170
|
+
log.error('serve_request_failed', { status }, err);
|
|
169
171
|
res.status(status).json({ error: status >= 500 ? 'internal error' : 'bad request' });
|
|
170
172
|
});
|
|
171
173
|
return app;
|
package/dist/buildIdentity.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.BUILD_MANIFEST_PATH = exports.MAX_BUILD_MANIFEST_BYTES = void 0;
|
|
3
|
+
exports.BUILD_MANIFEST_PATH = exports.ENGINE_VERSION = exports.MAX_BUILD_MANIFEST_BYTES = void 0;
|
|
4
4
|
exports.readBuildIdentity = readBuildIdentity;
|
|
5
5
|
const node_fs_1 = require("node:fs");
|
|
6
6
|
const node_path_1 = require("node:path");
|
|
@@ -46,6 +46,23 @@ function resolveEngineRoot() {
|
|
|
46
46
|
}
|
|
47
47
|
return null;
|
|
48
48
|
}
|
|
49
|
+
/**
|
|
50
|
+
* The engine package's version, read from its package.json. Every surface that announces a version (the MCP
|
|
51
|
+
* initialize reply, GET /api/health, `renor --version`) takes it from here, so none can drift from the package.
|
|
52
|
+
* "unknown" when the file cannot be read or holds no semver-shaped version, never a guess.
|
|
53
|
+
*/
|
|
54
|
+
exports.ENGINE_VERSION = (() => {
|
|
55
|
+
const root = resolveEngineRoot();
|
|
56
|
+
if (!root)
|
|
57
|
+
return 'unknown';
|
|
58
|
+
try {
|
|
59
|
+
const version = JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(root, 'package.json'), 'utf8')).version;
|
|
60
|
+
return typeof version === 'string' && /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version) ? version : 'unknown';
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return 'unknown';
|
|
64
|
+
}
|
|
65
|
+
})();
|
|
49
66
|
/**
|
|
50
67
|
* Absolute path of the stamped manifest, or null when the package root cannot
|
|
51
68
|
* be located (then identity is simply unknown — never a guess).
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CallPhases = exports.PHASE_NAMES = void 0;
|
|
4
|
+
exports.isPhasedTool = isPhasedTool;
|
|
5
|
+
exports.recordPhase = recordPhase;
|
|
6
|
+
exports.recordedPhase = recordedPhase;
|
|
7
|
+
exports.markRetrieved = markRetrieved;
|
|
8
|
+
const node_async_hooks_1 = require("node:async_hooks");
|
|
9
|
+
/**
|
|
10
|
+
* Where the time of one read call went, for the engine's log (telemetry.ts `phases` on a `get_context` or `search_memory` line).
|
|
11
|
+
* Milliseconds, a number per phase and nothing else: no text of the call, its query or its result ever goes through here.
|
|
12
|
+
*
|
|
13
|
+
* embed the embedding provider's whole call for the query (token, requests, waits between retries); absent when the
|
|
14
|
+
* query's vector came from the query cache, so a missing `embed` says that no call to the service was made
|
|
15
|
+
* token of embed: waiting for an access token (about 0 when one was in hand)
|
|
16
|
+
* connect of embed: opening a connection to the service (TCP, TLS, address lookup); absent when a connection was reused
|
|
17
|
+
* call of embed: the requests themselves once connected (send, the service's own time, receive)
|
|
18
|
+
* retrieve the matches and facts: from the start of the call to the point they are in hand, less embed
|
|
19
|
+
* reply the answer built from them: from that point to the end of the call
|
|
20
|
+
*
|
|
21
|
+
* A call's phases are collected in an `AsyncLocalStorage` scope that the tool wrapper opens (server.ts), so the provider that
|
|
22
|
+
* measures the embedding needs no argument for it, and a call made outside a scope (a CLI, an eval, a test) records nothing.
|
|
23
|
+
* Requests made together add up, so the phases of such a call can sum to more than its wall time.
|
|
24
|
+
*/
|
|
25
|
+
exports.PHASE_NAMES = ['embed', 'token', 'connect', 'call', 'retrieve', 'reply'];
|
|
26
|
+
/** The tools that report their phases: the two reads whose latency the product is judged on. */
|
|
27
|
+
const PHASED_TOOLS = ['get_context', 'search_memory'];
|
|
28
|
+
function isPhasedTool(tool) {
|
|
29
|
+
return PHASED_TOOLS.includes(tool);
|
|
30
|
+
}
|
|
31
|
+
const scope = new node_async_hooks_1.AsyncLocalStorage();
|
|
32
|
+
class CallPhases {
|
|
33
|
+
totals = new Map();
|
|
34
|
+
startedAt = performance.now();
|
|
35
|
+
retrievedAt;
|
|
36
|
+
embedAtRetrieved = 0;
|
|
37
|
+
/** Runs `work` with this as the call whose phases are recorded. */
|
|
38
|
+
run(work) {
|
|
39
|
+
return scope.run(this, work);
|
|
40
|
+
}
|
|
41
|
+
add(phase, ms) {
|
|
42
|
+
if (Number.isFinite(ms) && ms >= 0)
|
|
43
|
+
this.totals.set(phase, (this.totals.get(phase) ?? 0) + ms);
|
|
44
|
+
}
|
|
45
|
+
total(phase) {
|
|
46
|
+
return this.totals.get(phase) ?? 0;
|
|
47
|
+
}
|
|
48
|
+
/** The matches and facts are in hand: what is left of the call is the answer being built. The first mark counts. */
|
|
49
|
+
markRetrieved() {
|
|
50
|
+
if (this.retrievedAt !== undefined)
|
|
51
|
+
return;
|
|
52
|
+
this.retrievedAt = performance.now();
|
|
53
|
+
this.embedAtRetrieved = this.total('embed');
|
|
54
|
+
}
|
|
55
|
+
/** What the call recorded, rounded to a tenth of a millisecond; `retrieve` and `reply` only for a call that reached the mark. */
|
|
56
|
+
report() {
|
|
57
|
+
const out = {};
|
|
58
|
+
for (const [phase, ms] of this.totals)
|
|
59
|
+
out[phase] = ms;
|
|
60
|
+
if (this.retrievedAt !== undefined) {
|
|
61
|
+
out.retrieve = Math.max(0, this.retrievedAt - this.startedAt - this.embedAtRetrieved);
|
|
62
|
+
out.reply = Math.max(0, performance.now() - this.retrievedAt);
|
|
63
|
+
}
|
|
64
|
+
for (const phase of exports.PHASE_NAMES)
|
|
65
|
+
if (out[phase] !== undefined)
|
|
66
|
+
out[phase] = Math.round(out[phase] * 10) / 10;
|
|
67
|
+
return out;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
exports.CallPhases = CallPhases;
|
|
71
|
+
/** Adds to the current call's phase, if a call is being recorded. */
|
|
72
|
+
function recordPhase(phase, ms) {
|
|
73
|
+
scope.getStore()?.add(phase, ms);
|
|
74
|
+
}
|
|
75
|
+
/** What the current call has recorded so far for a phase (0 when no call is being recorded). */
|
|
76
|
+
function recordedPhase(phase) {
|
|
77
|
+
return scope.getStore()?.total(phase) ?? 0;
|
|
78
|
+
}
|
|
79
|
+
/** The matches and facts of the current call are in hand (see CallPhases.markRetrieved). */
|
|
80
|
+
function markRetrieved() {
|
|
81
|
+
scope.getStore()?.markRetrieved();
|
|
82
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -5,6 +5,7 @@ const commander_1 = require("commander");
|
|
|
5
5
|
const config_1 = require("./config");
|
|
6
6
|
const client_1 = require("./db/client");
|
|
7
7
|
const schema_1 = require("./db/schema");
|
|
8
|
+
const sqlite_1 = require("./db/sqlite");
|
|
8
9
|
const lifecycle_1 = require("./memory/lifecycle");
|
|
9
10
|
const pipeline_1 = require("./indexer/pipeline");
|
|
10
11
|
const watcher_1 = require("./watcher/watcher");
|
|
@@ -34,7 +35,9 @@ const hashEmbedding_1 = require("./eval/hashEmbedding");
|
|
|
34
35
|
const parser_1 = require("./conversation/parser");
|
|
35
36
|
const writer_1 = require("./conversation/writer");
|
|
36
37
|
const sessionRouter_1 = require("./router/sessionRouter");
|
|
38
|
+
const cli_1 = require("./sharing/cli");
|
|
37
39
|
const digests_1 = require("./digests/digests");
|
|
40
|
+
const buildIdentity_1 = require("./buildIdentity");
|
|
38
41
|
/**
|
|
39
42
|
* `serving`: the process is the long-running service (serve, mcp), the only one that may set an unterminated ledger tail
|
|
40
43
|
* aside at start-up. Every other command refuses such a tail and leaves the ledger alone (replayLedger).
|
|
@@ -180,7 +183,7 @@ function printNotIndexed(names, labelWidth, tooLarge = []) {
|
|
|
180
183
|
if (names.length > 10)
|
|
181
184
|
console.log(` ! …and ${names.length - 10} more`);
|
|
182
185
|
}
|
|
183
|
-
const program = new commander_1.Command().name("renor").version(
|
|
186
|
+
const program = new commander_1.Command().name("renor").version(buildIdentity_1.ENGINE_VERSION);
|
|
184
187
|
program
|
|
185
188
|
.command("index")
|
|
186
189
|
.description("Index full vault")
|
|
@@ -447,6 +450,31 @@ program
|
|
|
447
450
|
process.exit(1);
|
|
448
451
|
}
|
|
449
452
|
});
|
|
453
|
+
program
|
|
454
|
+
.command("telemetry <action> [argument]")
|
|
455
|
+
.description("Choose whether to share anonymous signals or training data with Renor (off by default): status | signals | data | off | delete | preview [level] | send | fields [level]")
|
|
456
|
+
.option("--yes", "do not ask: you have read the words shown and confirm you are 18 or older")
|
|
457
|
+
.option("--local", "delete: also empty the local learning log")
|
|
458
|
+
.option("--dir <path>", "the sharing folder (default: RENOR_SHARING_DIR, else ~/.config/renor)")
|
|
459
|
+
.option("--memory-dir <path>", "the memory folder whose learning log is read (default: MVP_MEMORY_DIR)")
|
|
460
|
+
.action(async (action, argument, opts) => {
|
|
461
|
+
const readline = await import("node:readline");
|
|
462
|
+
const code = await (0, cli_1.runTelemetry)(action, argument, opts, {
|
|
463
|
+
out: (line) => process.stdout.write(`${line}\n`),
|
|
464
|
+
err: (line) => process.stderr.write(`${line}\n`),
|
|
465
|
+
env: process.env,
|
|
466
|
+
ask: (question) => {
|
|
467
|
+
if (!process.stdin.isTTY)
|
|
468
|
+
return Promise.resolve(null);
|
|
469
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
470
|
+
return new Promise((resolve) => rl.question(question, (answer) => {
|
|
471
|
+
rl.close();
|
|
472
|
+
resolve(/^y(es)?$/i.test(answer.trim()));
|
|
473
|
+
}));
|
|
474
|
+
},
|
|
475
|
+
});
|
|
476
|
+
process.exit(code);
|
|
477
|
+
});
|
|
450
478
|
program
|
|
451
479
|
.command("log-conversation")
|
|
452
480
|
.description("Log a Claude Code session to Obsidian vault (invoked by Stop hook)")
|
|
@@ -488,4 +516,11 @@ program
|
|
|
488
516
|
process.exit(0);
|
|
489
517
|
}
|
|
490
518
|
});
|
|
519
|
+
// One line naming the fix, before any command runs, when this machine cannot run the engine's SQLite: otherwise an MCP
|
|
520
|
+
// client only sees a server that exited.
|
|
521
|
+
const runtimeProblem = (0, sqlite_1.sqliteRuntimeProblem)();
|
|
522
|
+
if (runtimeProblem) {
|
|
523
|
+
process.stderr.write(`renor-engine: ${runtimeProblem}\n`);
|
|
524
|
+
process.exit(1);
|
|
525
|
+
}
|
|
491
526
|
program.parse();
|
package/dist/context/rowStore.js
CHANGED
|
@@ -116,7 +116,7 @@ function assertProvenance(row) {
|
|
|
116
116
|
throw new ProvenanceError(`structured row (${row.rowKind}) from ${row.sourcePath} rejected: missing source_chunk_id`);
|
|
117
117
|
}
|
|
118
118
|
}
|
|
119
|
-
function parsePayload(
|
|
119
|
+
function parsePayload(_rowKind, raw) {
|
|
120
120
|
return JSON.parse(raw);
|
|
121
121
|
}
|
|
122
122
|
/**
|
|
@@ -30,6 +30,21 @@ function resolveJSONLPath(sessionId, cwd) {
|
|
|
30
30
|
}
|
|
31
31
|
return file;
|
|
32
32
|
}
|
|
33
|
+
const isText = (v) => typeof v === "string" && v !== "";
|
|
34
|
+
/** A message's content blocks: an array of objects, or a non-blank string as one text block. Anything else in the array is dropped. */
|
|
35
|
+
function contentBlocks(raw) {
|
|
36
|
+
if (Array.isArray(raw))
|
|
37
|
+
return raw.filter((b) => b !== null && typeof b === "object");
|
|
38
|
+
return typeof raw === "string" && raw.trim() ? [{ type: "text", text: raw }] : [];
|
|
39
|
+
}
|
|
40
|
+
/** The text of the text blocks, joined; a block whose text is not a string adds nothing. */
|
|
41
|
+
function textOf(content) {
|
|
42
|
+
return content
|
|
43
|
+
.filter(b => b.type === "text" && typeof b.text === "string")
|
|
44
|
+
.map(b => b.text)
|
|
45
|
+
.join("\n")
|
|
46
|
+
.trim();
|
|
47
|
+
}
|
|
33
48
|
/**
|
|
34
49
|
* Определяет, является ли сообщение пользователя внутренним (системным).
|
|
35
50
|
* Внутренние сообщения не включаются в итоговый разговор:
|
|
@@ -43,7 +58,8 @@ function isInternalUserMessage(content) {
|
|
|
43
58
|
if (content.every((b) => b.type === "tool_result"))
|
|
44
59
|
return true;
|
|
45
60
|
// Внутренние теги Claude Code CLI
|
|
46
|
-
const
|
|
61
|
+
const first = content.find((b) => b.type === "text")?.text;
|
|
62
|
+
const firstText = typeof first === "string" ? first : "";
|
|
47
63
|
if (firstText.startsWith("<local-command") ||
|
|
48
64
|
firstText.startsWith("<command-name>"))
|
|
49
65
|
return true;
|
|
@@ -70,7 +86,7 @@ async function waitForAiTitle(jsonlPath, retries = 8, intervalMs = 2000) {
|
|
|
70
86
|
for (const line of raw.split("\n")) {
|
|
71
87
|
try {
|
|
72
88
|
const entry = JSON.parse(line);
|
|
73
|
-
if (entry
|
|
89
|
+
if (entry?.type === "ai-title" && typeof entry.aiTitle === "string" && entry.aiTitle) {
|
|
74
90
|
return entry.aiTitle;
|
|
75
91
|
}
|
|
76
92
|
}
|
|
@@ -96,22 +112,27 @@ async function parseConversation(jsonlPath, sessionId, cwd, options = {}) {
|
|
|
96
112
|
let timeStartFound = false;
|
|
97
113
|
const turns = [];
|
|
98
114
|
for (const line of lines) {
|
|
99
|
-
let
|
|
115
|
+
let parsed;
|
|
100
116
|
try {
|
|
101
|
-
|
|
117
|
+
parsed = JSON.parse(line);
|
|
102
118
|
}
|
|
103
119
|
catch {
|
|
104
120
|
continue;
|
|
105
121
|
}
|
|
122
|
+
// A line of the file is any JSON value; only an object can be an entry.
|
|
123
|
+
if (parsed === null || typeof parsed !== "object")
|
|
124
|
+
continue;
|
|
125
|
+
const entry = parsed;
|
|
106
126
|
// Извлекаем заголовок сессии из специальной записи ai-title
|
|
107
|
-
if (entry.type === "ai-title" && entry.aiTitle) {
|
|
127
|
+
if (entry.type === "ai-title" && typeof entry.aiTitle === "string" && entry.aiTitle) {
|
|
108
128
|
title = entry.aiTitle;
|
|
109
129
|
continue;
|
|
110
130
|
}
|
|
111
131
|
// Время начала = первый dequeue из очереди операций
|
|
112
132
|
if (!timeStartFound &&
|
|
113
133
|
entry.type === "queue-operation" &&
|
|
114
|
-
entry.operation === "dequeue"
|
|
134
|
+
entry.operation === "dequeue" &&
|
|
135
|
+
typeof entry.timestamp === "string") {
|
|
115
136
|
timeStart = entry.timestamp;
|
|
116
137
|
timeStartFound = true;
|
|
117
138
|
continue;
|
|
@@ -122,53 +143,36 @@ async function parseConversation(jsonlPath, sessionId, cwd, options = {}) {
|
|
|
122
143
|
continue;
|
|
123
144
|
if (entry.type === "user" && entry.message?.role === "user") {
|
|
124
145
|
// content может быть строкой (короткие сессии) или массивом блоков
|
|
125
|
-
const
|
|
126
|
-
const content = Array.isArray(rawContent)
|
|
127
|
-
? rawContent
|
|
128
|
-
: typeof rawContent === "string" && rawContent.trim()
|
|
129
|
-
? [{ type: "text", text: rawContent }]
|
|
130
|
-
: [];
|
|
146
|
+
const content = contentBlocks(entry.message.content);
|
|
131
147
|
if (isInternalUserMessage(content))
|
|
132
148
|
continue;
|
|
133
|
-
const text = content
|
|
134
|
-
.filter((b) => b.type === "text")
|
|
135
|
-
.map((b) => b.text)
|
|
136
|
-
.join("\n")
|
|
137
|
-
.trim();
|
|
149
|
+
const text = textOf(content);
|
|
138
150
|
if (!text)
|
|
139
151
|
continue;
|
|
140
152
|
turns.push({ role: "user", text, toolCalls: [] });
|
|
141
153
|
continue;
|
|
142
154
|
}
|
|
143
155
|
if (entry.type === "assistant" && entry.message?.role === "assistant") {
|
|
144
|
-
const
|
|
145
|
-
const
|
|
146
|
-
? rawContent
|
|
147
|
-
: typeof rawContent === "string" && rawContent.trim()
|
|
148
|
-
? [{ type: "text", text: rawContent }]
|
|
149
|
-
: [];
|
|
150
|
-
const text = content
|
|
151
|
-
.filter((b) => b.type === "text")
|
|
152
|
-
.map((b) => b.text)
|
|
153
|
-
.join("\n")
|
|
154
|
-
.trim();
|
|
156
|
+
const content = contentBlocks(entry.message.content);
|
|
157
|
+
const text = textOf(content);
|
|
155
158
|
const toolCalls = [];
|
|
156
159
|
for (const block of content) {
|
|
157
160
|
if (block.type !== "tool_use")
|
|
158
161
|
continue;
|
|
159
|
-
|
|
162
|
+
const input = (block.input ?? {});
|
|
163
|
+
if (block.name === "Edit" && isText(input.file_path)) {
|
|
160
164
|
toolCalls.push({
|
|
161
165
|
kind: "edit",
|
|
162
|
-
filePath:
|
|
163
|
-
oldString:
|
|
164
|
-
newString:
|
|
166
|
+
filePath: input.file_path,
|
|
167
|
+
oldString: typeof input.old_string === "string" ? input.old_string : "",
|
|
168
|
+
newString: typeof input.new_string === "string" ? input.new_string : "",
|
|
165
169
|
});
|
|
166
170
|
}
|
|
167
|
-
else if (block.name === "Write" &&
|
|
168
|
-
toolCalls.push({ kind: "write", filePath:
|
|
171
|
+
else if (block.name === "Write" && isText(input.file_path)) {
|
|
172
|
+
toolCalls.push({ kind: "write", filePath: input.file_path });
|
|
169
173
|
}
|
|
170
|
-
else if (block.name === "Bash" &&
|
|
171
|
-
toolCalls.push({ kind: "bash", command:
|
|
174
|
+
else if (block.name === "Bash" && isText(input.command)) {
|
|
175
|
+
toolCalls.push({ kind: "bash", command: input.command });
|
|
172
176
|
}
|
|
173
177
|
}
|
|
174
178
|
if (!text && toolCalls.length === 0)
|
package/dist/db/client.js
CHANGED
|
@@ -6,11 +6,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.openDb = openDb;
|
|
7
7
|
exports.openIndexDb = openIndexDb;
|
|
8
8
|
exports.openReadOnlyDb = openReadOnlyDb;
|
|
9
|
-
const
|
|
9
|
+
const sqlite_1 = __importDefault(require("./sqlite"));
|
|
10
10
|
const fs_1 = __importDefault(require("fs"));
|
|
11
11
|
const path_1 = __importDefault(require("path"));
|
|
12
12
|
const migrations_1 = require("./migrations");
|
|
13
13
|
const indexSalvage_1 = require("./indexSalvage");
|
|
14
|
+
const log_1 = require("../mvp/log");
|
|
15
|
+
const log = (0, log_1.createLogger)('engine');
|
|
14
16
|
/**
|
|
15
17
|
* Map up to this much of the database file into memory for reads (`PRAGMA mmap_size`). SQLite maps only
|
|
16
18
|
* what exists, so a small database is untouched; the ceiling is address space, not memory.
|
|
@@ -35,7 +37,7 @@ function openDb(dbPath) {
|
|
|
35
37
|
fs_1.default.mkdirSync(path_1.default.dirname(dbPath), { recursive: true, mode: 0o700 });
|
|
36
38
|
fs_1.default.closeSync(fs_1.default.openSync(dbPath, fs_1.default.constants.O_CREAT | fs_1.default.constants.O_WRONLY, 0o600));
|
|
37
39
|
}
|
|
38
|
-
const db = new
|
|
40
|
+
const db = new sqlite_1.default(dbPath);
|
|
39
41
|
try {
|
|
40
42
|
db.pragma('journal_mode = WAL');
|
|
41
43
|
db.pragma('foreign_keys = ON');
|
|
@@ -87,7 +89,7 @@ function openIndexDb(dbPath, ledgerPath) {
|
|
|
87
89
|
const db = openDb(dbPath);
|
|
88
90
|
(0, migrations_1.runMigrations)(db);
|
|
89
91
|
const { restored, rejected } = (0, indexSalvage_1.restoreIndexTables)(db, salvage);
|
|
90
|
-
|
|
92
|
+
log.error('index_rebuilt_from_ledger', { index: path_1.default.basename(dbPath), damage, movedTo: path_1.default.basename(aside), salvage: (0, indexSalvage_1.describeSalvage)(salvage, restored, rejected) });
|
|
91
93
|
return db;
|
|
92
94
|
}
|
|
93
95
|
/** The rows the ledger does not hold, read from the damaged file, read-only (openReadOnlyDb: the file is not written). */
|
|
@@ -141,7 +143,7 @@ function openReadOnlyDb(dbPath) {
|
|
|
141
143
|
if (!fs_1.default.existsSync(dbPath)) {
|
|
142
144
|
throw new Error(`database not found: ${dbPath}`);
|
|
143
145
|
}
|
|
144
|
-
const db = new
|
|
146
|
+
const db = new sqlite_1.default(dbPath, { readonly: true, fileMustExist: true });
|
|
145
147
|
db.pragma('foreign_keys = ON');
|
|
146
148
|
return db;
|
|
147
149
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.HANDOFF_TABLE = void 0;
|
|
4
|
+
exports.ensureHandoffTable = ensureHandoffTable;
|
|
5
|
+
exports.hasHandoffTable = hasHandoffTable;
|
|
6
|
+
/**
|
|
7
|
+
* The handoff note's tables (memory/handoff.ts): the notes, and the one-row high-water mark of the clock they expire
|
|
8
|
+
* by. They are not a migration: a workspace with RENOR_HANDOFF_NOTE off never gets them, so its schema and its bytes
|
|
9
|
+
* stay those of a workspace without this code. They are created on the first write, read or expiry sweep, all of which
|
|
10
|
+
* run only with the flag on, and by index salvage only when the damaged index had the notes.
|
|
11
|
+
*/
|
|
12
|
+
exports.HANDOFF_TABLE = 'handoff_notes';
|
|
13
|
+
const DDL = `CREATE TABLE IF NOT EXISTS handoff_notes (
|
|
14
|
+
repo TEXT NOT NULL,
|
|
15
|
+
branch TEXT NOT NULL,
|
|
16
|
+
note TEXT NOT NULL,
|
|
17
|
+
writer TEXT NOT NULL,
|
|
18
|
+
written_at TEXT NOT NULL,
|
|
19
|
+
expires_at TEXT NOT NULL,
|
|
20
|
+
PRIMARY KEY (repo, branch)
|
|
21
|
+
);
|
|
22
|
+
CREATE TABLE IF NOT EXISTS handoff_clock (
|
|
23
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
24
|
+
high_water INTEGER NOT NULL
|
|
25
|
+
)`;
|
|
26
|
+
function ensureHandoffTable(db) {
|
|
27
|
+
db.exec(DDL);
|
|
28
|
+
}
|
|
29
|
+
function hasHandoffTable(db) {
|
|
30
|
+
return db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(exports.HANDOFF_TABLE) !== undefined;
|
|
31
|
+
}
|
package/dist/db/indexSalvage.js
CHANGED
|
@@ -5,6 +5,7 @@ exports.unreadableSalvage = unreadableSalvage;
|
|
|
5
5
|
exports.salvageIndexTables = salvageIndexTables;
|
|
6
6
|
exports.restoreIndexTables = restoreIndexTables;
|
|
7
7
|
exports.describeSalvage = describeSalvage;
|
|
8
|
+
const handoffTable_1 = require("./handoffTable");
|
|
8
9
|
/**
|
|
9
10
|
* The index is rebuilt from the ledger, but a few of its tables hold state that no ledger event rebuilds: the pending
|
|
10
11
|
* agent requests, the what's-new cursors, and the connection names the workspace has seen (known_agents). When the
|
|
@@ -23,7 +24,7 @@ function unreadableSalvage(reason) {
|
|
|
23
24
|
/** Reads what an open (read-only) damaged index still gives. Never throws: a table that does not read says why. */
|
|
24
25
|
function salvageIndexTables(db) {
|
|
25
26
|
const salvage = { rows: {}, unreadable: {} };
|
|
26
|
-
for (const table of
|
|
27
|
+
for (const table of tablesIn(db)) {
|
|
27
28
|
const rows = [];
|
|
28
29
|
salvage.rows[table] = rows;
|
|
29
30
|
try {
|
|
@@ -43,7 +44,10 @@ function salvageIndexTables(db) {
|
|
|
43
44
|
function restoreIndexTables(db, salvage) {
|
|
44
45
|
const restored = {};
|
|
45
46
|
const rejected = {};
|
|
46
|
-
|
|
47
|
+
// The handoff table is created on first use (db/handoffTable.ts), so the rebuilt index has it only when it is owed rows.
|
|
48
|
+
if ((salvage.rows[handoffTable_1.HANDOFF_TABLE]?.length ?? 0) > 0)
|
|
49
|
+
(0, handoffTable_1.ensureHandoffTable)(db);
|
|
50
|
+
for (const table of tablesOf(salvage)) {
|
|
47
51
|
restored[table] = 0;
|
|
48
52
|
rejected[table] = 0;
|
|
49
53
|
for (const row of salvage.rows[table] ?? []) {
|
|
@@ -63,8 +67,8 @@ function restoreIndexTables(db, salvage) {
|
|
|
63
67
|
}
|
|
64
68
|
/** One sentence for the log: what came back from the damaged index, and what did not. */
|
|
65
69
|
function describeSalvage(salvage, restored, rejected) {
|
|
66
|
-
const kept =
|
|
67
|
-
const lost =
|
|
70
|
+
const kept = tablesOf(salvage).map(t => `${t} ${restored[t] ?? 0}`).join(', ');
|
|
71
|
+
const lost = tablesOf(salvage)
|
|
68
72
|
.map(t => {
|
|
69
73
|
const parts = [];
|
|
70
74
|
if (rejected[t])
|
|
@@ -79,3 +83,19 @@ function describeSalvage(salvage, restored, rejected) {
|
|
|
79
83
|
function errorText(err) {
|
|
80
84
|
return err instanceof Error ? err.message : String(err);
|
|
81
85
|
}
|
|
86
|
+
/**
|
|
87
|
+
* The tables to salvage from a damaged index: INDEX_ONLY_TABLES, then the handoff notes when that index has their table.
|
|
88
|
+
* A workspace with the handoff note off never has it, so its salvage, and the log sentence, are what they were without it.
|
|
89
|
+
*/
|
|
90
|
+
function tablesIn(db) {
|
|
91
|
+
let handoff = false;
|
|
92
|
+
try {
|
|
93
|
+
handoff = (0, handoffTable_1.hasHandoffTable)(db);
|
|
94
|
+
}
|
|
95
|
+
catch { /* an unreadable schema: the other tables say why */ }
|
|
96
|
+
return handoff ? [...exports.INDEX_ONLY_TABLES, handoffTable_1.HANDOFF_TABLE] : [...exports.INDEX_ONLY_TABLES];
|
|
97
|
+
}
|
|
98
|
+
/** The tables a salvage carries: INDEX_ONLY_TABLES, then the handoff notes when they were salvaged. */
|
|
99
|
+
function tablesOf(salvage) {
|
|
100
|
+
return handoffTable_1.HANDOFF_TABLE in salvage.rows ? [...exports.INDEX_ONLY_TABLES, handoffTable_1.HANDOFF_TABLE] : [...exports.INDEX_ONLY_TABLES];
|
|
101
|
+
}
|
package/dist/db/migrations.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.runMigrations = runMigrations;
|
|
4
|
+
const log_1 = require("../mvp/log");
|
|
5
|
+
const log = (0, log_1.createLogger)('engine', { stderr: true });
|
|
4
6
|
const migrations = [
|
|
5
7
|
{
|
|
6
8
|
version: 1,
|
|
@@ -85,7 +87,7 @@ const migrations = [
|
|
|
85
87
|
{
|
|
86
88
|
version: 3,
|
|
87
89
|
name: 'fts5_chunks',
|
|
88
|
-
up: (
|
|
90
|
+
up: () => {
|
|
89
91
|
// Intentionally left as no-op — superseded by migration 4
|
|
90
92
|
},
|
|
91
93
|
},
|
|
@@ -611,6 +613,32 @@ const migrations = [
|
|
|
611
613
|
`);
|
|
612
614
|
},
|
|
613
615
|
},
|
|
616
|
+
{
|
|
617
|
+
version: 27,
|
|
618
|
+
name: '27_erased_facts',
|
|
619
|
+
up: (db) => {
|
|
620
|
+
// Content-free receipts for fact erasures. The authoritative tombstone is in the ledger; this projection makes
|
|
621
|
+
// retries and restore refusals cheap without retaining subject, predicate, object, statement, scope, or source.
|
|
622
|
+
db.exec(`CREATE TABLE IF NOT EXISTS erased_facts (
|
|
623
|
+
event_id TEXT PRIMARY KEY,
|
|
624
|
+
original_digest TEXT NOT NULL,
|
|
625
|
+
erased_at TEXT NOT NULL
|
|
626
|
+
)`);
|
|
627
|
+
},
|
|
628
|
+
},
|
|
629
|
+
{
|
|
630
|
+
version: 28,
|
|
631
|
+
name: '28_erased_facts_closes',
|
|
632
|
+
up: (db) => {
|
|
633
|
+
// The closing link of an erasure (event ids and a time, no content), kept here as well as in the tombstone: a retry that
|
|
634
|
+
// has to redo a lost rewrite cannot read it from the facts tables, which a restart may have reconciled without it.
|
|
635
|
+
const cols = db.prepare('PRAGMA table_info(erased_facts)').all();
|
|
636
|
+
if (!cols.some(c => c.name === 'closes'))
|
|
637
|
+
db.exec('ALTER TABLE erased_facts ADD COLUMN closes TEXT');
|
|
638
|
+
if (!cols.some(c => c.name === 'closed_at'))
|
|
639
|
+
db.exec('ALTER TABLE erased_facts ADD COLUMN closed_at TEXT');
|
|
640
|
+
},
|
|
641
|
+
},
|
|
614
642
|
];
|
|
615
643
|
function runMigrations(db) {
|
|
616
644
|
db.prepare(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)`).run();
|
|
@@ -637,6 +665,6 @@ function runMigrations(db) {
|
|
|
637
665
|
}
|
|
638
666
|
})();
|
|
639
667
|
currentVersion = migration.version;
|
|
640
|
-
|
|
668
|
+
log.info('migration_applied', { version: migration.version, name: migration.name });
|
|
641
669
|
}
|
|
642
670
|
}
|