renor-engine 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +73 -0
- package/README.md +105 -2
- package/dist/BUILD.json +6 -0
- package/dist/api/apiAuth.js +43 -0
- package/dist/api/server.js +172 -0
- package/dist/buildIdentity.js +132 -0
- package/dist/cli-extractors.js +46 -0
- package/dist/cli.js +491 -0
- package/dist/config.js +65 -0
- package/dist/context/composer.js +320 -0
- package/dist/context/readHonesty.js +478 -0
- package/dist/context/rowStore.js +238 -0
- package/dist/conversation/formatter.js +81 -0
- package/dist/conversation/parser.js +189 -0
- package/dist/conversation/types.js +2 -0
- package/dist/conversation/writer.js +42 -0
- package/dist/db/client.js +147 -0
- package/dist/db/indexSalvage.js +81 -0
- package/dist/db/migrations.js +642 -0
- package/dist/db/readCache.js +109 -0
- package/dist/db/schema.js +5 -0
- package/dist/db/statements.js +89 -0
- package/dist/digests/digestCache.js +142 -0
- package/dist/digests/digests.js +597 -0
- package/dist/digests/worker.js +348 -0
- package/dist/embeddings/caching.js +337 -0
- package/dist/embeddings/coalescing.js +177 -0
- package/dist/embeddings/interactive.js +103 -0
- package/dist/embeddings/local.js +68 -0
- package/dist/embeddings/openai.js +105 -0
- package/dist/embeddings/rebuild.js +166 -0
- package/dist/embeddings/simdScan.js +258 -0
- package/dist/embeddings/space.js +21 -0
- package/dist/embeddings/types.js +14 -0
- package/dist/embeddings/vectorIndex.js +675 -0
- package/dist/embeddings/vertex.js +224 -0
- package/dist/eval/artifacts.js +42 -0
- package/dist/eval/cacheReport.js +139 -0
- package/dist/eval/comparison.js +182 -0
- package/dist/eval/context700-candidate.js +145 -0
- package/dist/eval/evalCache.js +525 -0
- package/dist/eval/hashEmbedding.js +35 -0
- package/dist/eval/longmemeval.js +56 -0
- package/dist/eval/metrics.js +181 -0
- package/dist/eval/renorbenchBaseline.js +486 -0
- package/dist/eval/renorbenchDocuments.js +134 -0
- package/dist/eval/renorbenchProduct.js +182 -0
- package/dist/eval/runLedger.js +279 -0
- package/dist/eval/runManifest.js +479 -0
- package/dist/eval/scoring.js +197 -0
- package/dist/evidence/attach.js +267 -0
- package/dist/evidence/background.js +147 -0
- package/dist/evidence/evidenceCache.js +182 -0
- package/dist/evidence/gate.js +37 -0
- package/dist/evidence/keyPool.js +85 -0
- package/dist/evidence/ollamaChat.js +84 -0
- package/dist/evidence/optin.js +165 -0
- package/dist/evidence/options.js +130 -0
- package/dist/evidence/secondPass.js +456 -0
- package/dist/extraction/classifyCapture.js +34 -0
- package/dist/extraction/decision-model.js +174 -0
- package/dist/extraction/eventExtractor.js +113 -0
- package/dist/extraction/extractionClient.js +122 -0
- package/dist/extraction/factExtractor.js +46 -0
- package/dist/extraction/factsWritePath.js +179 -0
- package/dist/extraction/preferenceExtractor.js +153 -0
- package/dist/extraction/secretGuard.js +176 -0
- package/dist/extraction/secretScanner.js +665 -0
- package/dist/extraction/typedFacts.js +224 -0
- package/dist/extraction/writePathAdjudication.js +110 -0
- package/dist/importing/apply.js +644 -0
- package/dist/importing/bundle-write.js +80 -0
- package/dist/importing/cli.js +407 -0
- package/dist/importing/existing.js +49 -0
- package/dist/importing/facts.js +123 -0
- package/dist/importing/forgetImported.js +79 -0
- package/dist/importing/hosted.js +467 -0
- package/dist/importing/index.js +29 -0
- package/dist/importing/journalLog.js +96 -0
- package/dist/importing/markdown.js +98 -0
- package/dist/importing/plan.js +206 -0
- package/dist/importing/secretScreen.js +113 -0
- package/dist/importing/serialize.js +188 -0
- package/dist/importing/source.js +463 -0
- package/dist/importing/types.js +6 -0
- package/dist/indexer/chunker.js +319 -0
- package/dist/indexer/dailyEntries.js +63 -0
- package/dist/indexer/hasher.js +10 -0
- package/dist/indexer/laneRuns.js +48 -0
- package/dist/indexer/o200k.js +481 -0
- package/dist/indexer/o200k.json +8 -0
- package/dist/indexer/outsideRoot.js +77 -0
- package/dist/indexer/parser.js +167 -0
- package/dist/indexer/pipeline.js +1168 -0
- package/dist/indexer/staleness.js +201 -0
- package/dist/indexer/tokenizer.js +77 -0
- package/dist/indexer/walker.js +184 -0
- package/dist/mcp/hive.js +88 -0
- package/dist/mcp/requestErrors.js +47 -0
- package/dist/mcp/server.js +396 -0
- package/dist/memory/agentLease.js +43 -0
- package/dist/memory/confinedWrite.js +315 -0
- package/dist/memory/controls.js +73 -0
- package/dist/memory/dailyNote.js +265 -0
- package/dist/memory/durableAppend.js +57 -0
- package/dist/memory/factConflicts.js +203 -0
- package/dist/memory/factTerms.js +321 -0
- package/dist/memory/hiveNameHeader.cjs +10 -0
- package/dist/memory/inventory.js +55 -0
- package/dist/memory/invisibleText.js +20 -0
- package/dist/memory/lifecycle.js +1756 -0
- package/dist/memory/limits.js +13 -0
- package/dist/memory/requests.js +183 -0
- package/dist/memory/sourceName.js +200 -0
- package/dist/memory/whatsNew.js +158 -0
- package/dist/memory/writer.js +175 -0
- package/dist/memory/writerConflicts.js +85 -0
- package/dist/memorymap/memoryMap.js +1568 -0
- package/dist/memorymap/replacedJournal.js +58 -0
- package/dist/memorymap/replacements.js +405 -0
- package/dist/mvp/activity.js +86 -0
- package/dist/mvp/agentMemory.js +1744 -0
- package/dist/mvp/auth.js +531 -0
- package/dist/mvp/cli.js +73 -0
- package/dist/mvp/client-store.js +462 -0
- package/dist/mvp/compileCache.js +249 -0
- package/dist/mvp/compileSeed.js +27 -0
- package/dist/mvp/config.js +288 -0
- package/dist/mvp/connectionStatus.js +80 -0
- package/dist/mvp/contextRelevance.js +413 -0
- package/dist/mvp/engine.js +1065 -0
- package/dist/mvp/exportAll.js +359 -0
- package/dist/mvp/factLanePack.js +77 -0
- package/dist/mvp/factMarkers.js +49 -0
- package/dist/mvp/factsLane.js +203 -0
- package/dist/mvp/hiddenSpans.js +585 -0
- package/dist/mvp/history.js +413 -0
- package/dist/mvp/hitDates.js +31 -0
- package/dist/mvp/hitFloor.js +36 -0
- package/dist/mvp/identity.js +18 -0
- package/dist/mvp/idle.js +113 -0
- package/dist/mvp/memoryDir.js +98 -0
- package/dist/mvp/posthogMcp.js +210 -0
- package/dist/mvp/readScope.js +104 -0
- package/dist/mvp/readUsefulness.js +222 -0
- package/dist/mvp/sanitize.js +48 -0
- package/dist/mvp/schemaRejections.js +91 -0
- package/dist/mvp/scope.js +127 -0
- package/dist/mvp/server.js +2089 -0
- package/dist/mvp/serverCard.js +29 -0
- package/dist/mvp/storedSize.js +139 -0
- package/dist/mvp/telemetry.js +441 -0
- package/dist/retrieval/adviceLines.js +195 -0
- package/dist/retrieval/adviceQuestion.js +44 -0
- package/dist/retrieval/conversationShape.js +44 -0
- package/dist/retrieval/dailyEntryTime.js +55 -0
- package/dist/retrieval/entityLookup.js +113 -0
- package/dist/retrieval/factsFirst.js +188 -0
- package/dist/retrieval/keyword.js +84 -0
- package/dist/retrieval/ledgerEventTime.js +51 -0
- package/dist/retrieval/ledgerLiveness.js +83 -0
- package/dist/retrieval/ledgerPresentation.js +161 -0
- package/dist/retrieval/noteIdentity.js +57 -0
- package/dist/retrieval/orchestrator.js +1086 -0
- package/dist/retrieval/queryHelpers.js +142 -0
- package/dist/retrieval/questionSpan.js +475 -0
- package/dist/retrieval/readerEvents.js +690 -0
- package/dist/retrieval/readerLayout.js +503 -0
- package/dist/retrieval/relativeTimeHints.js +113 -0
- package/dist/retrieval/semantic.js +67 -0
- package/dist/router/routerCache.js +106 -0
- package/dist/router/sessionRouter.js +726 -0
- package/dist/toolRefusal.js +120 -0
- package/dist/watcher/watcher.js +75 -0
- package/package.json +120 -4
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ProvenanceError = void 0;
|
|
4
|
+
exports.normDedupeValue = normDedupeValue;
|
|
5
|
+
exports.matchForm = matchForm;
|
|
6
|
+
exports.splitWords = splitWords;
|
|
7
|
+
exports.dedupeKeyFor = dedupeKeyFor;
|
|
8
|
+
exports.insertStructuredRows = insertStructuredRows;
|
|
9
|
+
exports.deleteStructuredRowsForSource = deleteStructuredRowsForSource;
|
|
10
|
+
exports.deleteStructuredRowsForSourceKind = deleteStructuredRowsForSourceKind;
|
|
11
|
+
exports.readCurrentRowsForNotes = readCurrentRowsForNotes;
|
|
12
|
+
const orchestrator_1 = require("../retrieval/orchestrator");
|
|
13
|
+
class ProvenanceError extends Error {
|
|
14
|
+
}
|
|
15
|
+
exports.ProvenanceError = ProvenanceError;
|
|
16
|
+
/** NFKD → fold Latin diacritics → keep letters/numbers/marks from every script → lowercase → collapse.
|
|
17
|
+
*
|
|
18
|
+
* Letters and numbers are kept from EVERY script (`\p{L}`/`\p{N}`), never an
|
|
19
|
+
* ASCII range. The previous `[^\x20-\x7E]` + `[^a-z0-9]` pair deleted every
|
|
20
|
+
* non-Latin character, so two distinct memories written in Chinese, Japanese,
|
|
21
|
+
* Cyrillic, Greek, Arabic, Hebrew or Devanagari both normalized to `''` and
|
|
22
|
+
* shared one identity — and `decision-model` then called one `identical` to the
|
|
23
|
+
* other, folding the candidate into the existing row's confirm_count instead of
|
|
24
|
+
* storing it. Silent data loss.
|
|
25
|
+
*
|
|
26
|
+
* Combining marks are folded ONLY after a Latin base letter, which is what keeps
|
|
27
|
+
* the existing accent folding (`Café` → `cafe`) without destroying the marks
|
|
28
|
+
* that ARE letters in other scripts. A blanket `\p{M}` strip looks equivalent on
|
|
29
|
+
* Latin but merges meaning-changing pairs in scripts this bug report names:
|
|
30
|
+
* `が`/`か`, `ぱ`/`は`, `की`/`क`, `मिल`/`मील`, `น้ำ`/`นำ` (measured; pinned by test).
|
|
31
|
+
* To go back to the blanket strip, delete only the lookbehind (making the two
|
|
32
|
+
* mark clauses `\p{M}` in both places).
|
|
33
|
+
*
|
|
34
|
+
* `\p{L}` is `a-zA-Z` and `\p{N}` is `0-9` within ASCII, so ASCII identities are
|
|
35
|
+
* unchanged. `\p{C}` (control, format, private-use, unassigned) is REMOVED rather
|
|
36
|
+
* than treated as a separator, matching what the old `[^\x20-\x7E]` filter did to
|
|
37
|
+
* those characters — so every ASCII input stays byte-identical, including one
|
|
38
|
+
* carrying a control character. That preserves one pre-existing quirk: a control
|
|
39
|
+
* character between two word characters glues them (`'a\u0000b'` → `ab`);
|
|
40
|
+
* separating instead would move ASCII identities, so it is left out of scope.
|
|
41
|
+
*
|
|
42
|
+
* Values with no letters or numbers (punctuation, symbols, emoji) still normalize
|
|
43
|
+
* to `''`, exactly as before. Dataset-free. */
|
|
44
|
+
function normDedupeValue(value) {
|
|
45
|
+
return value
|
|
46
|
+
.normalize('NFKD')
|
|
47
|
+
.replace(/(?<=\p{Script=Latin})\p{M}+/gu, '')
|
|
48
|
+
.toLowerCase()
|
|
49
|
+
.replace(/\p{C}+/gu, '')
|
|
50
|
+
.replace(/[^\p{L}\p{N}\p{M}]+/gu, ' ')
|
|
51
|
+
.trim();
|
|
52
|
+
}
|
|
53
|
+
/** The read side's word separator: the SAME class `normDedupeValue` uses above, so a query
|
|
54
|
+
* is cut into words exactly as the text it must match is. `\p{L}`/`\p{N}`/`\p{M}` cover
|
|
55
|
+
* every script; both are exactly `a-z`/`0-9`/nothing within ASCII, so an ASCII query is
|
|
56
|
+
* cut into byte-identical words.
|
|
57
|
+
*
|
|
58
|
+
* Splitting on the ASCII range instead (`[^a-z0-9]`) deletes every non-ASCII letter, and a
|
|
59
|
+
* word that is *entirely* non-ASCII is deleted whole rather than shortened: `não` became
|
|
60
|
+
* `n`+`o`, `東京` became nothing, `東京都` became nothing. A read then lost the identity of
|
|
61
|
+
* the very facts it was written to find — measured on this repo's read paths in
|
|
62
|
+
* `tests/context/readSideUnicodeTokens.test.ts`: a `東京都` query scored every row 0, so
|
|
63
|
+
* `大阪に通っている` (newer, irrelevant) outranked `東京都に住んでいる`, and the query
|
|
64
|
+
* scoped to no row at all, returning the whole table.
|
|
65
|
+
*
|
|
66
|
+
* `numeric` additionally keeps `.` and `%` inside a word, so a measurement stays one word
|
|
67
|
+
* ("3.5%", "42 Wh"). Both variants were the pre-existing ASCII classes plus `\p{L}\p{N}\p{M}`
|
|
68
|
+
* widened from `a-z0-9`, so no ASCII word changes.
|
|
69
|
+
*
|
|
70
|
+
* Deliberately NOT NFKD-folded, unlike `normDedupeValue`: the read side matches its terms
|
|
71
|
+
* with `includes()` against the RAW stored text (`rowSearchText` + `indexOf`), so folding
|
|
72
|
+
* here would stop a term from matching the very row it names. Marks are kept as word
|
|
73
|
+
* characters for the same reason the write side keeps them for non-Latin scripts. */
|
|
74
|
+
const WORD_SEPARATOR = /[^\p{L}\p{N}\p{M}]+/u;
|
|
75
|
+
const WORD_SEPARATOR_NUMERIC = /[^\p{L}\p{N}\p{M}.%]+/u;
|
|
76
|
+
/** The one form the read side matches on: NFC, then lowercased. Applied to BOTH the query
|
|
77
|
+
* and the text it is matched against, so a query matches a stored fact however either was
|
|
78
|
+
* spelled. Stored payloads are never normalized (they are written as given), and NFC is
|
|
79
|
+
* *not* the same string as NFD — macOS-authored text, pasted filenames and Obsidian on Mac
|
|
80
|
+
* all produce NFD — so an NFC query term would otherwise miss an NFD fact (`café` vs
|
|
81
|
+
* `café`) that a raw substring test cannot bridge. ASCII is NFC-stable, so this is
|
|
82
|
+
* identity on every ASCII input. NOT NFKD and NOT accent-folded: the match is a raw
|
|
83
|
+
* `includes()`, so folding a term would stop it matching the very row it names, which is
|
|
84
|
+
* also why the write side's `normDedupeValue` folding cannot be reused here. */
|
|
85
|
+
function matchForm(text) {
|
|
86
|
+
return (text ?? '').normalize('NFC').toLowerCase();
|
|
87
|
+
}
|
|
88
|
+
/** Lowercased NFC words of `text` in order, repeats included — callers de-duplicate. A query
|
|
89
|
+
* and the stored text it is matched against are both put in `matchForm`, so the cut cannot
|
|
90
|
+
* differ by side. */
|
|
91
|
+
function splitWords(text, numeric = false) {
|
|
92
|
+
return matchForm(text).split(numeric ? WORD_SEPARATOR_NUMERIC : WORD_SEPARATOR);
|
|
93
|
+
}
|
|
94
|
+
/** dedupe_key per ADR §4: fact (value, object), event (when, event), pref (statement). */
|
|
95
|
+
function dedupeKeyFor(row) {
|
|
96
|
+
switch (row.rowKind) {
|
|
97
|
+
case 'fact': {
|
|
98
|
+
const p = row.payload;
|
|
99
|
+
return `fact|${normDedupeValue(p.value)}|${normDedupeValue(p.object)}`;
|
|
100
|
+
}
|
|
101
|
+
case 'event': {
|
|
102
|
+
const p = row.payload;
|
|
103
|
+
return `event|${normDedupeValue(p.when)}|${normDedupeValue(p.event)}`;
|
|
104
|
+
}
|
|
105
|
+
case 'preference': {
|
|
106
|
+
const p = row.payload;
|
|
107
|
+
return `pref|${normDedupeValue(p.statement)}`;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
function assertProvenance(row) {
|
|
112
|
+
if (!row.sourcePath || !row.sourcePath.trim()) {
|
|
113
|
+
throw new ProvenanceError(`structured row (${row.rowKind}) rejected: missing source_path`);
|
|
114
|
+
}
|
|
115
|
+
if (!Number.isInteger(row.sourceChunkId) || row.sourceChunkId <= 0) {
|
|
116
|
+
throw new ProvenanceError(`structured row (${row.rowKind}) from ${row.sourcePath} rejected: missing source_chunk_id`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
function parsePayload(rowKind, raw) {
|
|
120
|
+
return JSON.parse(raw);
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Insert rows with identity-keyed supersession (ADR-0003 §4.1). A new row
|
|
124
|
+
* supersedes a current row only when they share (row_kind, scope, dedupe_key)
|
|
125
|
+
* AND both content dates are known AND the new source is not older. When
|
|
126
|
+
* either date is NULL (unknown), supersession never fires: both rows stay
|
|
127
|
+
* current and the composer orders them deterministically.
|
|
128
|
+
*
|
|
129
|
+
* Same-identity double-insert from the SAME source cannot occur: migration
|
|
130
|
+
* v11's partial UNIQUE index (current rows only) closes the check-then-act
|
|
131
|
+
* race across `await model.decide`, and a lost race is re-checked here and
|
|
132
|
+
* skipped (the winning writer's row already covers the identity) instead of
|
|
133
|
+
* double-inserting. Cross-note coexistence (different source_path) is
|
|
134
|
+
* preserved by the index's source_path column.
|
|
135
|
+
*/
|
|
136
|
+
function insertStructuredRows(db, rows, opts = {}) {
|
|
137
|
+
const insert = db.prepare(`
|
|
138
|
+
INSERT INTO structured_rows
|
|
139
|
+
(row_kind, source_path, source_hash, source_chunk_id, scope, extraction_version,
|
|
140
|
+
valid_from, valid_to, payload_json, dedupe_key, created_at)
|
|
141
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?)
|
|
142
|
+
`);
|
|
143
|
+
// A content date is only as late as its file's write time. Rows stored before this bound existed are bounded the same
|
|
144
|
+
// way wherever they are read, including here, so a stale future date cannot outrank a normal later update.
|
|
145
|
+
const findCurrent = db.prepare(`
|
|
146
|
+
SELECT r.id, r.valid_from, n.modified_at
|
|
147
|
+
FROM structured_rows r LEFT JOIN notes n ON n.path = r.source_path
|
|
148
|
+
WHERE r.row_kind = ? AND r.scope = ? AND r.dedupe_key = ? AND r.valid_to IS NULL
|
|
149
|
+
`);
|
|
150
|
+
const writeTime = db.prepare('SELECT modified_at FROM notes WHERE path = ?');
|
|
151
|
+
const supersede = db.prepare('UPDATE structured_rows SET valid_to = ? WHERE id = ?');
|
|
152
|
+
db.transaction(() => {
|
|
153
|
+
for (const row of rows) {
|
|
154
|
+
assertProvenance(row);
|
|
155
|
+
const validFrom = (0, orchestrator_1.boundContentDate)(row.validFrom, writeTime.get(row.sourcePath)?.modified_at);
|
|
156
|
+
const dedupeKey = dedupeKeyFor(row);
|
|
157
|
+
const scope = row.scope ?? 'global';
|
|
158
|
+
const now = new Date().toISOString();
|
|
159
|
+
if (validFrom !== null && !opts.skipSupersession) {
|
|
160
|
+
for (const existing of findCurrent.all(row.rowKind, scope, dedupeKey)) {
|
|
161
|
+
// Supersede only on identity collision with known dates on both
|
|
162
|
+
// sides; the newer (or same-date re-extraction) source wins.
|
|
163
|
+
const existingFrom = (0, orchestrator_1.boundContentDate)(existing.valid_from, existing.modified_at);
|
|
164
|
+
if (existingFrom !== null && validFrom >= existingFrom) {
|
|
165
|
+
supersede.run(validFrom, existing.id);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
try {
|
|
170
|
+
insert.run(row.rowKind, row.sourcePath, row.sourceHash, row.sourceChunkId, scope, row.extractionVersion, validFrom, JSON.stringify(row.payload), dedupeKey, now);
|
|
171
|
+
}
|
|
172
|
+
catch (err) {
|
|
173
|
+
const code = err?.code ?? '';
|
|
174
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
175
|
+
if (code === 'SQLITE_CONSTRAINT_UNIQUE' || /UNIQUE constraint failed/.test(message)) {
|
|
176
|
+
// Lost a same-identity insert race against a concurrent writer for
|
|
177
|
+
// this source (migration v11 partial UNIQUE index). Re-check: a
|
|
178
|
+
// current row already covers the identity from this source — skip
|
|
179
|
+
// instead of double-inserting; anything else is a real error.
|
|
180
|
+
const dupe = db
|
|
181
|
+
.prepare(`SELECT id FROM structured_rows
|
|
182
|
+
WHERE row_kind = ? AND scope = ? AND dedupe_key = ? AND source_path = ? AND valid_to IS NULL LIMIT 1`)
|
|
183
|
+
.get(row.rowKind, scope, dedupeKey, row.sourcePath);
|
|
184
|
+
if (dupe)
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
throw err;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
})();
|
|
191
|
+
}
|
|
192
|
+
/** Edit path (ADR-0003 §4.2): re-index of a note deletes its rows; re-extraction repopulates. */
|
|
193
|
+
function deleteStructuredRowsForSource(db, sourcePath) {
|
|
194
|
+
db.prepare('DELETE FROM structured_rows WHERE source_path = ?').run(sourcePath);
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Same edit path, scoped to one lane.
|
|
198
|
+
*
|
|
199
|
+
* The whole-source delete removes every row kind for a note. With more than
|
|
200
|
+
* one lane armed, the second lane's cleanup deleted the first lane's fresh
|
|
201
|
+
* rows and the third deleted both. The cache key is per lane
|
|
202
|
+
* (source_hash + extraction_version), so stale-row cleanup is too.
|
|
203
|
+
*/
|
|
204
|
+
function deleteStructuredRowsForSourceKind(db, sourcePath, rowKind) {
|
|
205
|
+
db.prepare('DELETE FROM structured_rows WHERE source_path = ? AND row_kind = ?')
|
|
206
|
+
.run(sourcePath, rowKind);
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Current-only read joined to the live notes table: a row whose source_path no
|
|
210
|
+
* longer resolves to a live note is never returned (provenance guard), even if
|
|
211
|
+
* a connection without the FK pragma leaked orphans into the table.
|
|
212
|
+
*/
|
|
213
|
+
function readCurrentRowsForNotes(db, notePaths) {
|
|
214
|
+
if (notePaths.length === 0)
|
|
215
|
+
return [];
|
|
216
|
+
const placeholders = notePaths.map(() => '?').join(', ');
|
|
217
|
+
const rows = db.prepare(`SELECT r.id, r.row_kind, r.source_path, r.source_hash, r.source_chunk_id,
|
|
218
|
+
r.scope, r.extraction_version, r.valid_from, r.valid_to, r.payload_json, r.dedupe_key,
|
|
219
|
+
r.confirm_count, n.modified_at
|
|
220
|
+
FROM structured_rows r
|
|
221
|
+
JOIN notes n ON n.path = r.source_path
|
|
222
|
+
WHERE r.valid_to IS NULL AND r.source_path IN (${placeholders})
|
|
223
|
+
ORDER BY r.id`).all(...notePaths);
|
|
224
|
+
return rows.map((r) => ({
|
|
225
|
+
id: r.id,
|
|
226
|
+
rowKind: r.row_kind,
|
|
227
|
+
sourcePath: r.source_path,
|
|
228
|
+
sourceChunkId: r.source_chunk_id,
|
|
229
|
+
sourceHash: r.source_hash,
|
|
230
|
+
scope: r.scope,
|
|
231
|
+
extractionVersion: r.extraction_version,
|
|
232
|
+
validFrom: (0, orchestrator_1.boundContentDate)(r.valid_from ?? null, r.modified_at),
|
|
233
|
+
validTo: r.valid_to ?? null,
|
|
234
|
+
payload: parsePayload(r.row_kind, r.payload_json),
|
|
235
|
+
dedupeKey: r.dedupe_key,
|
|
236
|
+
confirmCount: r.confirm_count ?? 1,
|
|
237
|
+
}));
|
|
238
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.buildFilename = buildFilename;
|
|
7
|
+
exports.formatConversation = formatConversation;
|
|
8
|
+
const crypto_1 = __importDefault(require("crypto"));
|
|
9
|
+
// Возвращает YYYY-MM-DD из ISO-строки
|
|
10
|
+
function isoToDate(iso) {
|
|
11
|
+
return iso.substring(0, 10);
|
|
12
|
+
}
|
|
13
|
+
// Возвращает HH:mm из ISO-строки
|
|
14
|
+
function isoToTime(iso) {
|
|
15
|
+
return iso.substring(11, 16);
|
|
16
|
+
}
|
|
17
|
+
// A file name is at most 255 bytes; the date prefix and ".md" take about 22 of them, so the title gets this many UTF-8 bytes.
|
|
18
|
+
const MAX_TITLE_BYTES = 120;
|
|
19
|
+
// Убирает символы, недопустимые в именах файлов, и управляющие символы (NUL ломает путь); длинное название обрезается
|
|
20
|
+
// и получает короткий хеш полного названия, чтобы разные разговоры не совпали по имени файла.
|
|
21
|
+
function sanitizeTitle(title) {
|
|
22
|
+
const clean = title.replace(/[/\\:*?"<>|\x00-\x1f\x7f]/g, "-").trim();
|
|
23
|
+
if (Buffer.byteLength(clean, "utf8") <= MAX_TITLE_BYTES)
|
|
24
|
+
return clean;
|
|
25
|
+
let kept = "";
|
|
26
|
+
for (const ch of clean) {
|
|
27
|
+
if (Buffer.byteLength(kept + ch, "utf8") > MAX_TITLE_BYTES)
|
|
28
|
+
break;
|
|
29
|
+
kept += ch;
|
|
30
|
+
}
|
|
31
|
+
const digest = crypto_1.default.createHash("sha256").update(title).digest("hex").slice(0, 8);
|
|
32
|
+
return `${kept.trimEnd()}-${digest}`;
|
|
33
|
+
}
|
|
34
|
+
function buildFilename(title, isoTime) {
|
|
35
|
+
const date = isoToDate(isoTime);
|
|
36
|
+
const time = isoToTime(isoTime).replace(":", "-");
|
|
37
|
+
return `${date} ${time} - ${sanitizeTitle(title)}.md`;
|
|
38
|
+
}
|
|
39
|
+
function formatToolCall(tc) {
|
|
40
|
+
if (tc.kind === "edit") {
|
|
41
|
+
const diffLines = [
|
|
42
|
+
...tc.oldString.split("\n").map((l) => `> - ${l}`),
|
|
43
|
+
...tc.newString.split("\n").map((l) => `> + ${l}`),
|
|
44
|
+
].join("\n");
|
|
45
|
+
return `> **Edit** \`${tc.filePath}\`\n> \`\`\`diff\n${diffLines}\n> \`\`\``;
|
|
46
|
+
}
|
|
47
|
+
if (tc.kind === "write") {
|
|
48
|
+
return `> **Write** \`${tc.filePath}\``;
|
|
49
|
+
}
|
|
50
|
+
return `> **Bash** \`${tc.command}\``;
|
|
51
|
+
}
|
|
52
|
+
function formatTurn(turn) {
|
|
53
|
+
const parts = [];
|
|
54
|
+
if (turn.text) {
|
|
55
|
+
const label = turn.role === "user" ? "User" : "Assistant";
|
|
56
|
+
parts.push(`**${label}:** ${turn.text}`);
|
|
57
|
+
}
|
|
58
|
+
for (const tc of turn.toolCalls) {
|
|
59
|
+
parts.push(formatToolCall(tc));
|
|
60
|
+
}
|
|
61
|
+
return parts.join("\n\n");
|
|
62
|
+
}
|
|
63
|
+
function formatConversation(conv) {
|
|
64
|
+
const date = isoToDate(conv.timeStart);
|
|
65
|
+
const timeStart = isoToTime(conv.timeStart);
|
|
66
|
+
// Убираем переносы строк, чтобы не сломать YAML-фронтматтер
|
|
67
|
+
const safeSessionId = conv.sessionId.replace(/[\r\n]/g, " ");
|
|
68
|
+
const safeProject = conv.project.replace(/[\r\n]/g, " ");
|
|
69
|
+
const frontmatter = [
|
|
70
|
+
"---",
|
|
71
|
+
"kind: conversation",
|
|
72
|
+
`date: ${date}`,
|
|
73
|
+
`time_start: ${timeStart}`,
|
|
74
|
+
`session_id: ${safeSessionId}`,
|
|
75
|
+
`project: ${safeProject}`,
|
|
76
|
+
`title: ${JSON.stringify(conv.title)}`,
|
|
77
|
+
"---",
|
|
78
|
+
].join("\n");
|
|
79
|
+
const body = conv.turns.map(formatTurn).filter(Boolean).join("\n\n");
|
|
80
|
+
return `${frontmatter}\n\n${body}\n`;
|
|
81
|
+
}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.encodeProjectPath = encodeProjectPath;
|
|
7
|
+
exports.resolveJSONLPath = resolveJSONLPath;
|
|
8
|
+
exports.parseConversation = parseConversation;
|
|
9
|
+
const promises_1 = __importDefault(require("fs/promises"));
|
|
10
|
+
const path_1 = __importDefault(require("path"));
|
|
11
|
+
const os_1 = __importDefault(require("os"));
|
|
12
|
+
/**
|
|
13
|
+
* Кодирует путь проекта в формат имени директории Claude Code:
|
|
14
|
+
* слэши заменяются на дефисы (как в ~/.claude/projects/).
|
|
15
|
+
*/
|
|
16
|
+
function encodeProjectPath(cwd) {
|
|
17
|
+
return cwd.replace(/\//g, "-");
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Возвращает абсолютный путь к JSONL-файлу сессии в директории проектов Claude Code.
|
|
21
|
+
*/
|
|
22
|
+
function resolveJSONLPath(sessionId, cwd) {
|
|
23
|
+
const encoded = encodeProjectPath(cwd);
|
|
24
|
+
const projects = path_1.default.join(os_1.default.homedir(), ".claude", "projects");
|
|
25
|
+
const file = path_1.default.join(projects, encoded, `${sessionId}.jsonl`);
|
|
26
|
+
// The session id comes from the hook payload and names a file: a separator or a `..` that leaves the projects
|
|
27
|
+
// directory is refused, never read (a transcript of another file would otherwise be written to the vault).
|
|
28
|
+
if (/[\\/]/.test(sessionId) || !file.startsWith(projects + path_1.default.sep)) {
|
|
29
|
+
throw new Error("session_id must name a file in the Claude Code projects directory");
|
|
30
|
+
}
|
|
31
|
+
return file;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Определяет, является ли сообщение пользователя внутренним (системным).
|
|
35
|
+
* Внутренние сообщения не включаются в итоговый разговор:
|
|
36
|
+
* - только tool_result блоки (ответ инструмента, не пользователь)
|
|
37
|
+
* - сообщения с тегами CLI (local-command, command-name)
|
|
38
|
+
*/
|
|
39
|
+
function isInternalUserMessage(content) {
|
|
40
|
+
if (content.length === 0)
|
|
41
|
+
return true;
|
|
42
|
+
// Только tool_result блоки — ответ инструмента, не настоящий пользователь
|
|
43
|
+
if (content.every((b) => b.type === "tool_result"))
|
|
44
|
+
return true;
|
|
45
|
+
// Внутренние теги Claude Code CLI
|
|
46
|
+
const firstText = content.find((b) => b.type === "text")?.text ?? "";
|
|
47
|
+
if (firstText.startsWith("<local-command") ||
|
|
48
|
+
firstText.startsWith("<command-name>"))
|
|
49
|
+
return true;
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Ждёт появления записи ai-title в JSONL-файле с повторными попытками.
|
|
54
|
+
* Claude Code генерирует заголовок асинхронно после завершения сессии,
|
|
55
|
+
* поэтому Stop hook может сработать раньше, чем заголовок записан.
|
|
56
|
+
*
|
|
57
|
+
* @param jsonlPath - путь к .jsonl файлу
|
|
58
|
+
* @param retries - максимальное количество попыток (каждая с задержкой 2с)
|
|
59
|
+
*/
|
|
60
|
+
async function waitForAiTitle(jsonlPath, retries = 8, intervalMs = 2000) {
|
|
61
|
+
for (let i = 0; i < retries; i++) {
|
|
62
|
+
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
|
63
|
+
let raw;
|
|
64
|
+
try {
|
|
65
|
+
raw = await promises_1.default.readFile(jsonlPath, "utf8");
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
for (const line of raw.split("\n")) {
|
|
71
|
+
try {
|
|
72
|
+
const entry = JSON.parse(line);
|
|
73
|
+
if (entry.type === "ai-title" && entry.aiTitle) {
|
|
74
|
+
return entry.aiTitle;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Парсит JSONL-файл сессии Claude Code и возвращает структурированный разговор.
|
|
86
|
+
*
|
|
87
|
+
* @param jsonlPath - путь к .jsonl файлу сессии
|
|
88
|
+
* @param sessionId - идентификатор сессии (используется как fallback заголовок)
|
|
89
|
+
* @param cwd - рабочая директория проекта
|
|
90
|
+
*/
|
|
91
|
+
async function parseConversation(jsonlPath, sessionId, cwd, options = {}) {
|
|
92
|
+
const raw = await promises_1.default.readFile(jsonlPath, "utf8");
|
|
93
|
+
const lines = raw.split("\n").filter((l) => l.trim());
|
|
94
|
+
let title = sessionId;
|
|
95
|
+
let timeStart = new Date().toISOString();
|
|
96
|
+
let timeStartFound = false;
|
|
97
|
+
const turns = [];
|
|
98
|
+
for (const line of lines) {
|
|
99
|
+
let entry;
|
|
100
|
+
try {
|
|
101
|
+
entry = JSON.parse(line);
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
// Извлекаем заголовок сессии из специальной записи ai-title
|
|
107
|
+
if (entry.type === "ai-title" && entry.aiTitle) {
|
|
108
|
+
title = entry.aiTitle;
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
// Время начала = первый dequeue из очереди операций
|
|
112
|
+
if (!timeStartFound &&
|
|
113
|
+
entry.type === "queue-operation" &&
|
|
114
|
+
entry.operation === "dequeue") {
|
|
115
|
+
timeStart = entry.timestamp;
|
|
116
|
+
timeStartFound = true;
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
// Пропускаем мета-сообщения (системные уведомления Claude Code)
|
|
120
|
+
// и сайдчейн-записи (внутренние подразговоры оркестратора)
|
|
121
|
+
if (entry.isMeta || entry.isSidechain)
|
|
122
|
+
continue;
|
|
123
|
+
if (entry.type === "user" && entry.message?.role === "user") {
|
|
124
|
+
// content может быть строкой (короткие сессии) или массивом блоков
|
|
125
|
+
const rawContent = entry.message.content;
|
|
126
|
+
const content = Array.isArray(rawContent)
|
|
127
|
+
? rawContent
|
|
128
|
+
: typeof rawContent === "string" && rawContent.trim()
|
|
129
|
+
? [{ type: "text", text: rawContent }]
|
|
130
|
+
: [];
|
|
131
|
+
if (isInternalUserMessage(content))
|
|
132
|
+
continue;
|
|
133
|
+
const text = content
|
|
134
|
+
.filter((b) => b.type === "text")
|
|
135
|
+
.map((b) => b.text)
|
|
136
|
+
.join("\n")
|
|
137
|
+
.trim();
|
|
138
|
+
if (!text)
|
|
139
|
+
continue;
|
|
140
|
+
turns.push({ role: "user", text, toolCalls: [] });
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
if (entry.type === "assistant" && entry.message?.role === "assistant") {
|
|
144
|
+
const rawContent = entry.message.content;
|
|
145
|
+
const content = Array.isArray(rawContent)
|
|
146
|
+
? rawContent
|
|
147
|
+
: typeof rawContent === "string" && rawContent.trim()
|
|
148
|
+
? [{ type: "text", text: rawContent }]
|
|
149
|
+
: [];
|
|
150
|
+
const text = content
|
|
151
|
+
.filter((b) => b.type === "text")
|
|
152
|
+
.map((b) => b.text)
|
|
153
|
+
.join("\n")
|
|
154
|
+
.trim();
|
|
155
|
+
const toolCalls = [];
|
|
156
|
+
for (const block of content) {
|
|
157
|
+
if (block.type !== "tool_use")
|
|
158
|
+
continue;
|
|
159
|
+
if (block.name === "Edit" && block.input?.file_path) {
|
|
160
|
+
toolCalls.push({
|
|
161
|
+
kind: "edit",
|
|
162
|
+
filePath: block.input.file_path,
|
|
163
|
+
oldString: block.input.old_string ?? "",
|
|
164
|
+
newString: block.input.new_string ?? "",
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
else if (block.name === "Write" && block.input?.file_path) {
|
|
168
|
+
toolCalls.push({ kind: "write", filePath: block.input.file_path });
|
|
169
|
+
}
|
|
170
|
+
else if (block.name === "Bash" && block.input?.command) {
|
|
171
|
+
toolCalls.push({ kind: "bash", command: block.input.command });
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
if (!text && toolCalls.length === 0)
|
|
175
|
+
continue;
|
|
176
|
+
turns.push({ role: "assistant", text, toolCalls });
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
// Если заголовок не найден — ждём: Claude Code записывает ai-title асинхронно.
|
|
180
|
+
// Ждём только если в JSONL есть queue-operation (признак полноценной сессии).
|
|
181
|
+
// Короткие сессии без queue-operation ai-title не получают.
|
|
182
|
+
const { maxRetries = 8, retryIntervalMs = 2000 } = options;
|
|
183
|
+
if (title === sessionId && timeStartFound && maxRetries > 0) {
|
|
184
|
+
const aiTitle = await waitForAiTitle(jsonlPath, maxRetries, retryIntervalMs);
|
|
185
|
+
if (aiTitle)
|
|
186
|
+
title = aiTitle;
|
|
187
|
+
}
|
|
188
|
+
return { sessionId, title, timeStart, project: cwd, turns };
|
|
189
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.writeConversation = writeConversation;
|
|
7
|
+
const path_1 = __importDefault(require("path"));
|
|
8
|
+
const formatter_1 = require("./formatter");
|
|
9
|
+
const secretGuard_1 = require("../extraction/secretGuard");
|
|
10
|
+
const confinedWrite_1 = require("../memory/confinedWrite");
|
|
11
|
+
/**
|
|
12
|
+
* Записывает разговор в файл markdown в хранилище.
|
|
13
|
+
* @param vaultPath - корневой путь хранилища
|
|
14
|
+
* @param conversationsDir - относительный путь к папке разговоров (например, "Claude Code/Conversations")
|
|
15
|
+
* @param conv - распарсенный разговор
|
|
16
|
+
* @returns абсолютный путь к созданному файлу
|
|
17
|
+
* @throws Error если обнаружена попытка обхода пути
|
|
18
|
+
*/
|
|
19
|
+
async function writeConversation(vaultPath, conversationsDir, conv) {
|
|
20
|
+
const filename = (0, formatter_1.buildFilename)(conv.title, conv.timeStart);
|
|
21
|
+
const dir = path_1.default.join(vaultPath, conversationsDir);
|
|
22
|
+
// Защита от path traversal через имя файла
|
|
23
|
+
const filePath = path_1.default.join(dir, filename);
|
|
24
|
+
const resolvedPath = path_1.default.resolve(filePath);
|
|
25
|
+
const resolvedBase = path_1.default.resolve(dir);
|
|
26
|
+
if (!resolvedPath.startsWith(resolvedBase + path_1.default.sep) &&
|
|
27
|
+
resolvedPath !== resolvedBase) {
|
|
28
|
+
throw new Error("Path traversal detected in conversation filename");
|
|
29
|
+
}
|
|
30
|
+
// RENOR_SECRET_GUARD (default ON): a transcript that looks like it holds a secret is not written, and the
|
|
31
|
+
// directory is not created. The caller (the log-conversation hook) reports the refusal and exits cleanly.
|
|
32
|
+
if ((0, secretGuard_1.secretGuardEnabled)()) {
|
|
33
|
+
await (0, secretGuard_1.assertStorable)([
|
|
34
|
+
{ field: "conversation", text: (0, formatter_1.formatConversation)(conv) },
|
|
35
|
+
{ field: "conversation file name", text: filename },
|
|
36
|
+
]);
|
|
37
|
+
}
|
|
38
|
+
const content = (0, formatter_1.formatConversation)(conv);
|
|
39
|
+
// A link, a hard link or a FIFO planted in the vault is refused, never followed (memory/confinedWrite.ts).
|
|
40
|
+
await (0, confinedWrite_1.writeConfined)(vaultPath, path_1.default.relative(path_1.default.resolve(vaultPath), resolvedPath), content);
|
|
41
|
+
return filePath;
|
|
42
|
+
}
|