renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,98 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.readMarkdownSource = readMarkdownSource;
7
- exports.parseMarkdownImportNote = parseMarkdownImportNote;
8
- const fs_1 = __importDefault(require("fs"));
9
- const path_1 = __importDefault(require("path"));
10
- const walker_1 = require("../indexer/walker");
11
- const parser_1 = require("../indexer/parser");
12
- const chunker_1 = require("../indexer/chunker");
13
- const source_1 = require("./source");
14
- const controls_1 = require("../memory/controls");
15
- const types_1 = require("./types");
16
- /** Plain vault input: use the shipped walker/parser/chunker, without a source index. */
17
- async function readMarkdownSource(folder, filter) {
18
- const root = fs_1.default.realpathSync(folder);
19
- if (!fs_1.default.statSync(root).isDirectory())
20
- throw new Error('Markdown source must be a folder');
21
- const nonUtf8Names = [];
22
- const files = (await (0, walker_1.walkVault)(root, { onNonUtf8Markdown: name => nonUtf8Names.push(name) })).sort((a, b) => (0, source_1.toRelative)(root, a).localeCompare((0, source_1.toRelative)(root, b)));
23
- const selected = files.filter(file => (0, source_1.isSelected)((0, source_1.toRelative)(root, file), filter));
24
- const limited = filter.limit === undefined ? selected : selected.slice(0, filter.limit);
25
- const notes = limited.map(file => {
26
- const relPath = (0, source_1.toRelative)(root, file);
27
- // Refuse a file replaced with a symlink after walking, and do not follow a
28
- // substituted ancestor outside the chosen vault.
29
- const resolved = fs_1.default.realpathSync(file);
30
- const relative = path_1.default.relative(root, resolved);
31
- if (relative === '..' || relative.startsWith(`..${path_1.default.sep}`) || path_1.default.isAbsolute(relative)) {
32
- throw new Error(`Markdown source escapes the chosen folder: ${relPath}`);
33
- }
34
- const fd = fs_1.default.openSync(file, fs_1.default.constants.O_RDONLY | fs_1.default.constants.O_NOFOLLOW);
35
- try {
36
- const stat = fs_1.default.fstatSync(fd);
37
- if (!stat.isFile())
38
- throw new Error(`Markdown source is not a regular file: ${relPath}`);
39
- const bytes = fs_1.default.readFileSync(fd);
40
- try {
41
- new TextDecoder('utf-8', { fatal: true }).decode(bytes);
42
- }
43
- catch {
44
- throw new Error(`Markdown source is not UTF-8: ${relPath}`);
45
- }
46
- const content = bytes.toString('utf8');
47
- if (content.includes('\0'))
48
- throw new Error(`Markdown source contains binary data: ${relPath}`);
49
- return parseMarkdownImportNote(file, relPath, content, stat.mtime.toISOString());
50
- }
51
- finally {
52
- fs_1.default.closeSync(fd);
53
- }
54
- });
55
- return {
56
- format: types_1.BUNDLE_FORMAT, sourceLabel: path_1.default.basename(root), notes, facts: [],
57
- // A name the selection excludes anyway is not reported; every other one could not be imported.
58
- nonUtf8Names: nonUtf8Names.filter(name => (0, source_1.isSelected)(name, filter)).sort(),
59
- stats: {
60
- notesAvailable: files.length, notesSelected: notes.length,
61
- // Everything the selection left out, the --limit cut included, as the other adapters count it.
62
- notesFilteredOut: files.length - limited.length, notesWithOriginalBytes: notes.length,
63
- chunksSelected: notes.reduce((sum, note) => sum + note.chunks.length, 0),
64
- factsAvailable: 0, factsSelected: 0,
65
- },
66
- };
67
- }
68
- /** Shared adapter for local files and bounded hosted Markdown uploads. */
69
- function parseMarkdownImportNote(sourcePath, relPath, content, modifiedAt) {
70
- // A control character is refused as it is in a note or a statement (memory/controls.ts): the note is
71
- // written into the vault and indexed, so what this funnel admits the workspace holds afterwards. One
72
- // rule for a folder import and a hosted upload both.
73
- (0, controls_1.assertNoUnstorableControls)(`imported note ${relPath}`, content);
74
- // gray-matter ships a JavaScript engine that evaluates frontmatter. Remote
75
- // Markdown is data: admit only default/YAML or JSON before calling the parser.
76
- const opening = content.replace(/^\uFEFF/, '').match(/^---([^\r\n]*)/);
77
- if (opening && !['', 'yaml', 'json'].includes(opening[1].trim())) {
78
- throw new Error(`Unsupported Markdown frontmatter language: ${relPath}. Use YAML between --- lines, or remove the block.`);
79
- }
80
- let parsed;
81
- try {
82
- parsed = (0, parser_1.parseNote)(sourcePath, content);
83
- }
84
- catch {
85
- // Do not expose private YAML text in an error.
86
- throw new Error(`Cannot parse Markdown frontmatter: ${relPath}. Fix the YAML between the --- lines or remove that block, then send again.`);
87
- }
88
- return {
89
- sourcePath, relPath, title: parsed.title,
90
- kind: typeof parsed.frontmatter.kind === 'string' ? parsed.frontmatter.kind : null,
91
- modifiedAt, frontmatter: parsed.frontmatter, originalContent: content,
92
- // Spans are body-relative; the renderer locates chunks in original bytes.
93
- chunks: (0, chunker_1.chunkNote)(parsed).map(chunk => ({
94
- headingPath: chunk.headingPath, text: chunk.text, tokenCount: chunk.tokenCount,
95
- })),
96
- entities: [],
97
- };
98
- }
@@ -1,201 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.IMPORT_ROOT = void 0;
4
- exports.streamKey = streamKey;
5
- exports.assertImportNamespace = assertImportNamespace;
6
- exports.buildImportPlan = buildImportPlan;
7
- const hasher_1 = require("../indexer/hasher");
8
- const queryHelpers_1 = require("../retrieval/queryHelpers");
9
- const orchestrator_1 = require("../retrieval/orchestrator");
10
- const facts_1 = require("./facts");
11
- const scope_1 = require("../mvp/scope");
12
- const serialize_1 = require("./serialize");
13
- const secretGuard_1 = require("../extraction/secretGuard");
14
- const secretScreen_1 = require("./secretScreen");
15
- const sourceName_1 = require("../memory/sourceName");
16
- /**
17
- * The ledger source of a carried fact. A bundle file can name any source, and so can the `--event-source` flag, so without
18
- * this an imported fact could read as the agent that saved it (its author is what get_context shows and marks); an import
19
- * always says it is one.
20
- */
21
- const importedLedgerSource = (carried) => ((0, sourceName_1.isImportSource)(carried) ? carried : `${sourceName_1.IMPORT_SOURCE_PREFIX}${carried}`);
22
- /**
23
- * Normalized stream identity — the SAME rule the lifecycle uses to decide a stream.
24
- *
25
- * `identityKey` applies the lifecycle's Unicode subject fold (NFC + case, explicit
26
- * aliases unified) so a stream the lifecycle considers one subject is skipped rather
27
- * than carried — carrying it either failed the whole import (older value, replay
28
- * monotonicity) or silently superseded the destination's owner-stated value (newer
29
- * value). The predicate is kept **exact**, matching `currentFact`'s `predicate = ?`:
30
- * lowercasing it here collapsed `Primary Goal` and `primary goal`, two streams the
31
- * lifecycle holds separately, so a real stream was silently dropped. The scope is
32
- * normalized exactly as the ledger/replay normalizes it, so raw variants cannot
33
- * collide later, and keyed by its slug: `scope:x`, `project:x` and `x` are one
34
- * stream, whose current value a write closes whatever spelling it was saved
35
- * with (scopeSpellings). Nothing here rewrites the bytes that are emitted or hashed.
36
- */
37
- function streamKey(subject, predicate, scope) {
38
- const normalized = (0, facts_1.tryNormalizeScope)(scope);
39
- const key = normalized ? (0, scope_1.scopeSpellings)(normalized)[0] : scope.trim().toLowerCase();
40
- return `${(0, queryHelpers_1.identityKey)(subject)}\u0000${predicate}\u0000${key}`;
41
- }
42
- /** Every import writes under this folder: a read marks a memory there as imported (mvp/agentMemory.ts, IMPORTED_PATH). */
43
- exports.IMPORT_ROOT = 'Imported';
44
- /**
45
- * Refuses a destination outside Imported/. The import mark is read off a note's path, so an import written elsewhere read
46
- * as the workspace's own notes, and one written where the engine keeps its daily log (`Memory`, or the root) turned an
47
- * imported file's entry lines into saves by whatever source the file named.
48
- */
49
- function assertImportNamespace(namespace) {
50
- if ((0, serialize_1.destinationRelPath)(namespace, 'x').split('/')[0] !== exports.IMPORT_ROOT) {
51
- throw new Error(`the import destination must be under ${exports.IMPORT_ROOT}/ (got ${JSON.stringify(namespace)}): a read marks an imported memory by that folder`);
52
- }
53
- }
54
- function buildImportPlan(bundle, options) {
55
- assertImportNamespace(options.namespace);
56
- const existingStreams = options.existingStreams ?? new Set();
57
- const destinationNow = options.now ?? new Date().toISOString();
58
- const notes = [];
59
- const seenPaths = new Set();
60
- let duplicateNotePath = 0;
61
- let emptyNoteBody = 0;
62
- let notesFromOriginalBytes = 0;
63
- for (const note of bundle.notes) {
64
- const relPath = (0, serialize_1.destinationRelPath)(options.namespace, note.relPath);
65
- if (seenPaths.has(relPath)) {
66
- duplicateNotePath += 1;
67
- continue;
68
- }
69
- seenPaths.add(relPath);
70
- const rendered = (0, serialize_1.renderNoteMarkdown)(note);
71
- if (!rendered.content.trim()) {
72
- emptyNoteBody += 1;
73
- continue;
74
- }
75
- if (rendered.fromOriginalBytes)
76
- notesFromOriginalBytes += 1;
77
- notes.push({
78
- relPath,
79
- content: rendered.content,
80
- contentHash: (0, hasher_1.hashContent)(rendered.content),
81
- fromOriginalBytes: rendered.fromOriginalBytes,
82
- chunkLines: rendered.chunkLines,
83
- chunksNotLocated: rendered.chunksNotLocated,
84
- chunksSpanInherited: rendered.chunksSpanInherited,
85
- note,
86
- });
87
- }
88
- const skipEvents = new Map();
89
- const bump = (reason) => skipEvents.set(reason, (skipEvents.get(reason) ?? 0) + 1);
90
- const byStream = new Map();
91
- for (const fact of bundle.facts) {
92
- const key = streamKey(fact.subject, fact.predicate, fact.scope);
93
- const list = byStream.get(key);
94
- if (list)
95
- list.push(fact);
96
- else
97
- byStream.set(key, [fact]);
98
- }
99
- let multiValuedFactsDropped = 0;
100
- const events = [];
101
- const emittedStreams = new Set();
102
- for (const [key, facts] of byStream) {
103
- // The lifecycle layer holds at most ONE current value per (subject, predicate, scope).
104
- // The engine deliberately allows note-derived facts to be multi-valued, so a stream
105
- // with several distinct current values cannot become one current fact. Picking a
106
- // winner would misrepresent which value is current — and row order is not a
107
- // decision — so the whole stream is reported and dropped.
108
- const distinctObjects = new Set(facts.map(f => f.object));
109
- if (distinctObjects.size > 1) {
110
- multiValuedFactsDropped += facts.length;
111
- bump('multi-valued stream: the lifecycle holds one current value per subject+predicate');
112
- continue;
113
- }
114
- if (existingStreams.has(key)) {
115
- bump('destination already represents this subject+predicate');
116
- continue;
117
- }
118
- if (emittedStreams.has(key)) {
119
- bump('duplicate stream inside the bundle');
120
- continue;
121
- }
122
- const fact = facts[0];
123
- const validation = (0, facts_1.validateCarriedFact)({
124
- subject: fact.subject,
125
- predicate: fact.predicate,
126
- object: fact.object,
127
- scope: fact.scope,
128
- validFrom: fact.validFrom,
129
- });
130
- if (!validation.ok || !validation.scope || !validation.at) {
131
- bump(validation.reason ?? 'invalid fact');
132
- continue;
133
- }
134
- // Bound the bundle's claimed time by the destination's clock BEFORE the id is derived, so the planned event, the
135
- // stored valid_from and every read of it are bounded. Same rule and tolerance as every other content-derived clock.
136
- const at = (0, orchestrator_1.clampSessionDate)(validation.at, destinationNow) ?? destinationNow;
137
- // The event id is derived from every field replay compares, so a re-run against an
138
- // unchanged source re-appends nothing and a changed source yields a new event
139
- // instead of mutating one that replay already recorded.
140
- const id = (0, facts_1.deterministicEventId)({
141
- label: options.label,
142
- subject: fact.subject,
143
- predicate: fact.predicate,
144
- scope: validation.scope,
145
- object: fact.object,
146
- at,
147
- });
148
- events.push({
149
- id,
150
- at,
151
- kind: 'store',
152
- subject: fact.subject,
153
- predicate: fact.predicate,
154
- object: fact.object,
155
- scope: validation.scope,
156
- source: importedLedgerSource(fact.ledgerSource ?? options.eventSource),
157
- });
158
- emittedStreams.add(key);
159
- }
160
- // RENOR_SECRET_GUARD (default ON): a note or fact that looks like a secret is left out of the plan, with the
161
- // reason, so the dry run shows it and every other note is still imported. With the guard off none of this runs.
162
- let guardSkips = null;
163
- if ((0, secretGuard_1.secretGuardEnabled)()) {
164
- const screened = (0, secretScreen_1.leaveOutFlagged)(notes, events);
165
- // In place and without spreading: a bundle can hold more notes than a call can take as arguments.
166
- notes.length = 0;
167
- for (const kept of screened.notes)
168
- notes.push(kept);
169
- events.length = 0;
170
- for (const kept of screened.events)
171
- events.push(kept);
172
- notesFromOriginalBytes = notes.filter(n => n.fromOriginalBytes).length;
173
- guardSkips = screened.skips;
174
- }
175
- // Deterministic append order, so two runs produce an identical ledger.
176
- events.sort((a, b) => streamKey(a.subject, a.predicate, a.scope).localeCompare(streamKey(b.subject, b.predicate, b.scope)));
177
- const skippedFacts = [...skipEvents.entries()]
178
- .map(([reason, count]) => ({ reason, count }))
179
- .sort((a, b) => b.count - a.count);
180
- return {
181
- sourceLabel: bundle.sourceLabel,
182
- namespace: options.namespace,
183
- notes,
184
- events,
185
- skipped: { duplicateNotePath, emptyNoteBody, events: skippedFacts, ...(guardSkips ? { secretGuard: guardSkips } : {}) },
186
- counts: {
187
- notesInBundle: bundle.notes.length,
188
- chunksInBundle: bundle.stats.chunksSelected,
189
- factsInBundle: bundle.facts.length,
190
- selectedNotes: notes.length,
191
- selectedChunks: notes.reduce((sum, n) => sum + n.note.chunks.length, 0),
192
- notesFromOriginalBytes,
193
- notesReconstructed: notes.length - notesFromOriginalBytes,
194
- chunksNotLocated: notes.reduce((sum, n) => sum + n.chunksNotLocated, 0),
195
- chunksSpanInherited: notes.reduce((sum, n) => sum + n.chunksSpanInherited, 0),
196
- eventsToAppend: events.length,
197
- multiValuedFactsDropped,
198
- factsSkipped: skippedFacts.reduce((sum, s) => sum + s.count, 0),
199
- },
200
- };
201
- }
@@ -1,113 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ImportRefusedError = void 0;
4
- exports.screenNote = screenNote;
5
- exports.screenEvent = screenEvent;
6
- exports.screenImportPlan = screenImportPlan;
7
- exports.refuseFlaggedPlan = refuseFlaggedPlan;
8
- exports.leaveOutFlagged = leaveOutFlagged;
9
- const secretGuard_1 = require("../extraction/secretGuard");
10
- /** Every string `applyImportPlan` would store for this note. */
11
- function noteTexts(planned) {
12
- const note = planned.note;
13
- const texts = [
14
- { field: 'path', text: planned.relPath },
15
- { field: 'content', text: planned.content },
16
- { field: 'title', text: note.title },
17
- { field: 'kind', text: note.kind ?? '' },
18
- { field: 'frontmatter', text: JSON.stringify(note.frontmatter ?? {}) },
19
- ];
20
- for (const chunk of note.chunks) {
21
- texts.push({ field: 'chunk', text: chunk.text }, { field: 'chunk heading', text: chunk.headingPath });
22
- }
23
- for (const entity of note.entities) {
24
- texts.push({ field: 'entity', text: entity.canonicalName }, { field: 'entity', text: entity.type });
25
- for (const alias of entity.aliases ?? [])
26
- texts.push({ field: 'entity', text: alias });
27
- }
28
- return texts;
29
- }
30
- /** Every string `applyImportPlan` would append to the ledger for this fact. */
31
- function eventTexts(event) {
32
- return [
33
- { field: 'subject, predicate and object', text: `${event.subject} ${event.predicate}: ${event.object}` },
34
- { field: 'source', text: event.source },
35
- ];
36
- }
37
- function screenNote(planned) {
38
- const hit = (0, secretGuard_1.findSensitive)(noteTexts(planned));
39
- if (hit === null)
40
- return null;
41
- // The path was scanned first: if it is what was flagged, it must not be printed.
42
- return { ref: hit.field === 'path' ? null : planned.relPath, field: hit.field, reason: hit.reason };
43
- }
44
- function screenEvent(event) {
45
- const hit = (0, secretGuard_1.findSensitive)(eventTexts(event));
46
- return hit === null ? null : { ref: event.id, field: hit.field, reason: hit.reason };
47
- }
48
- /** What a plan still holds that the guard would refuse. Pure: it changes nothing. */
49
- function screenImportPlan(plan) {
50
- const notes = [];
51
- for (const planned of plan.notes) {
52
- const skip = screenNote(planned);
53
- if (skip)
54
- notes.push(skip);
55
- }
56
- const events = [];
57
- for (const event of plan.events) {
58
- const skip = screenEvent(event);
59
- if (skip)
60
- events.push(skip);
61
- }
62
- return { notes, events };
63
- }
64
- const describeSkip = (skip) => `${skip.ref ?? '(path withheld)'} (${skip.field}: ${skip.reason})`;
65
- /** The apply-time refusal. Counts and names only: the message carries no text from the plan. */
66
- class ImportRefusedError extends Error {
67
- skips;
68
- constructor(skips) {
69
- const all = [...skips.notes, ...skips.events];
70
- const shown = all.slice(0, 5).map(describeSkip).join('; ');
71
- super(`import refused: ${skips.notes.length} note(s) and ${skips.events.length} fact(s) in this plan appear to contain a credential `
72
- + 'or personal identifier, and the secret guard (RENOR_SECRET_GUARD) is on, so nothing was written. '
73
- + 'Build the plan again (it leaves them out) or switch the guard off. '
74
- + `First: ${shown}${all.length > 5 ? `; and ${all.length - 5} more` : ''}`);
75
- this.name = 'ImportRefusedError';
76
- this.skips = skips;
77
- }
78
- }
79
- exports.ImportRefusedError = ImportRefusedError;
80
- /** Throws ImportRefusedError, and writes nothing, when the plan still holds anything the guard would refuse. */
81
- function refuseFlaggedPlan(plan) {
82
- const skips = screenImportPlan(plan);
83
- if (skips.notes.length === 0 && skips.events.length === 0)
84
- return;
85
- for (const skip of [...skips.notes, ...skips.events])
86
- (0, secretGuard_1.logSkip)(skip.field, skip.reason);
87
- throw new ImportRefusedError(skips);
88
- }
89
- /** The plan-time half: split `notes` and `events` into what stays and what is left out, in order. */
90
- function leaveOutFlagged(notes, events) {
91
- const skips = { notes: [], events: [] };
92
- const keptNotes = [];
93
- for (const planned of notes) {
94
- const skip = screenNote(planned);
95
- if (skip) {
96
- skips.notes.push(skip);
97
- (0, secretGuard_1.logSkip)(skip.field, skip.reason);
98
- }
99
- else
100
- keptNotes.push(planned);
101
- }
102
- const keptEvents = [];
103
- for (const event of events) {
104
- const skip = screenEvent(event);
105
- if (skip) {
106
- skips.events.push(skip);
107
- (0, secretGuard_1.logSkip)(skip.field, skip.reason);
108
- }
109
- else
110
- keptEvents.push(event);
111
- }
112
- return { notes: keptNotes, events: keptEvents, skips };
113
- }
@@ -1,188 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.renderNoteMarkdown = renderNoteMarkdown;
4
- exports.destinationRelPath = destinationRelPath;
5
- /**
6
- * Produce the bytes to write for a note.
7
- *
8
- * The source's **original Markdown is preferred whenever it is reachable**: it is the
9
- * source of truth, and carrying a document is not re-parsing it (the constraint on this
10
- * path is that no re-parse, re-chunk or re-embed is run — not that a document may not
11
- * be carried).
12
- *
13
- * Reconstruction is the fallback for an index whose content tree is unavailable. It
14
- * concatenates the source's own chunk texts in order with the heading structure their
15
- * `heading_path` values already describe — not a re-parse, but a projection: it is
16
- * lossy for YAML comments and key order, and a caller that falls back is told so by
17
- * `fromOriginalBytes: false`.
18
- *
19
- * Chunk spans are **located in the bytes actually written**, not copied from the source
20
- * index. Measured on a real 22,774-chunk index, only 475 of 594 sampled stored spans
21
- * contained their own chunk text (the engine's paragraph splitter accumulates its
22
- * offsets approximately), while 593 of 594 chunk texts were found in the note body. A
23
- * copied span would therefore mis-cite a fifth of chunks; a located span is true of the
24
- * file being written by construction.
25
- */
26
- function renderNoteMarkdown(note) {
27
- const fromOriginalBytes = typeof note.originalContent === 'string';
28
- const content = fromOriginalBytes
29
- ? note.originalContent
30
- : reconstructNoteMarkdown(note);
31
- const located = locateChunkLines(content, note.chunks);
32
- let chunksNotLocated = 0;
33
- let chunksSpanInherited = 0;
34
- const chunkLines = located.lines.map((span, index) => {
35
- if (located.found[index])
36
- return span;
37
- chunksNotLocated += 1;
38
- const chunk = note.chunks[index];
39
- // The source's stored span describes the source's own bytes. That is exactly what is
40
- // being written on the original-bytes path, so it may be inherited there. On the
41
- // reconstruction path the written file has different bytes (frontmatter re-rendered,
42
- // headings materialised, empty chunks dropped), so an inherited coordinate would
43
- // point at unrelated text — report it as unknown instead.
44
- if (fromOriginalBytes && typeof chunk?.startLine === 'number' && typeof chunk.endLine === 'number') {
45
- chunksSpanInherited += 1;
46
- return { startLine: chunk.startLine, endLine: chunk.endLine };
47
- }
48
- return { startLine: 0, endLine: 0 };
49
- });
50
- return { content, chunkLines, fromOriginalBytes, chunksNotLocated, chunksSpanInherited };
51
- }
52
- /** Half-open line spans for each chunk, found by locating its text in the content. */
53
- function locateChunkLines(content, chunks) {
54
- const lines = [];
55
- const found = [];
56
- const newlinesBefore = newlineCounter(content);
57
- let cursor = 0;
58
- for (const chunk of chunks) {
59
- const needle = chunk.text.replace(/[ \t]+$/gm, '').trim();
60
- if (!needle) {
61
- lines.push({ startLine: 0, endLine: 0 });
62
- found.push(false);
63
- continue;
64
- }
65
- let at = content.indexOf(needle, cursor);
66
- if (at < 0)
67
- at = content.indexOf(needle);
68
- if (at < 0) {
69
- lines.push({ startLine: 0, endLine: 0 });
70
- found.push(false);
71
- continue;
72
- }
73
- const startLine = newlinesBefore(at);
74
- lines.push({ startLine, endLine: startLine + needle.split('\n').length });
75
- found.push(true);
76
- cursor = at + needle.length;
77
- }
78
- return { lines, found };
79
- }
80
- /**
81
- * Counts the newlines before an offset, resuming from where the previous count stopped.
82
- *
83
- * Chunks are located in document order, so each count reads only the text since the last one
84
- * and the pass over a note is linear in its size. Rescanning from offset 0 for every chunk cost
85
- * chunks x bytes: a 13 MB note with 8,000 chunks took minutes to preview. An offset before the
86
- * last stop (a chunk found out of order) is reached from whichever end is nearer, so no count
87
- * reads more characters than the rescan did.
88
- */
89
- function newlineCounter(content) {
90
- let stop = 0; // `newlines` is exact for content[0, stop)
91
- let newlines = 0;
92
- return index => {
93
- if (index >= stop)
94
- newlines += countNewlines(content, stop, index);
95
- else if (index < stop - index)
96
- newlines = countNewlines(content, 0, index);
97
- else
98
- newlines -= countNewlines(content, index, stop);
99
- stop = index;
100
- return newlines;
101
- };
102
- }
103
- function countNewlines(content, from, to) {
104
- let count = 0;
105
- for (let i = from; i < to; i += 1) {
106
- if (content.charCodeAt(i) === 10)
107
- count += 1;
108
- }
109
- return count;
110
- }
111
- /**
112
- * Rebuild a note body from its chunks, materialising heading paths that the chunk text
113
- * does not itself carry. The lines are joined verbatim — no blank-line collapsing —
114
- * because the chunk spans are computed against these exact bytes.
115
- */
116
- function reconstructNoteMarkdown(note) {
117
- const lines = [];
118
- const frontmatter = renderFrontmatter(note.frontmatter);
119
- if (frontmatter.length > 0)
120
- lines.push('---', ...frontmatter, '---', '');
121
- let stack = [];
122
- const first = note.chunks[0];
123
- if (first && !/^#{1,3}\s/.test(first.text.trimStart()) && note.title) {
124
- lines.push(`# ${note.title}`, '');
125
- stack = [note.title];
126
- }
127
- for (const chunk of note.chunks) {
128
- const text = chunk.text.replace(/[ \t]+$/gm, '').trim();
129
- if (!text)
130
- continue;
131
- const startsWithHeading = /^#{1,3}\s/.test(text);
132
- const headingStack = chunk.headingPath
133
- ? chunk.headingPath.split(' > ').map(s => s.trim()).filter(Boolean)
134
- : [];
135
- if (startsWithHeading) {
136
- if (headingStack.length > 0)
137
- stack = headingStack;
138
- }
139
- else if (headingStack.length > 0) {
140
- let common = 0;
141
- while (common < Math.min(headingStack.length, stack.length) && headingStack[common] === stack[common])
142
- common += 1;
143
- for (let level = common; level < headingStack.length; level += 1) {
144
- lines.push(`${'#'.repeat(Math.min(level + 1, 3))} ${headingStack[level]}`);
145
- }
146
- stack = headingStack.slice();
147
- }
148
- if (lines.length > 0 && lines[lines.length - 1] !== '')
149
- lines.push('');
150
- lines.push(...text.split('\n'));
151
- }
152
- return lines.length > 0 ? lines.join('\n') + '\n' : '';
153
- }
154
- /**
155
- * Frontmatter as YAML key/scalar lines. Values are JSON-encoded, which is a valid YAML
156
- * scalar or flow collection for every JSON value, so a round-trip through the engine's
157
- * own frontmatter parser cannot produce malformed YAML.
158
- */
159
- function renderFrontmatter(frontmatter) {
160
- const out = [];
161
- for (const [key, value] of Object.entries(frontmatter)) {
162
- if (!/^[A-Za-z_][A-Za-z0-9_-]*$/.test(key))
163
- continue;
164
- if (value === undefined)
165
- continue;
166
- const encoded = JSON.stringify(value);
167
- if (encoded === undefined || encoded.length > 4096)
168
- continue;
169
- out.push(`${key}: ${encoded}`);
170
- }
171
- return out;
172
- }
173
- /** `namespace/relPath`, with every segment sanitized so a source path cannot escape. */
174
- function destinationRelPath(namespace, sourceRelPath) {
175
- const cleanNamespace = sanitizeSegments(namespace);
176
- const clean = sanitizeSegments(sourceRelPath);
177
- if (!clean)
178
- throw new Error(`source path ${JSON.stringify(sourceRelPath)} has no usable segments`);
179
- return cleanNamespace ? `${cleanNamespace}/${clean}` : clean;
180
- }
181
- function sanitizeSegments(value) {
182
- return value
183
- .split('/')
184
- .map(segment => segment.trim())
185
- .filter(segment => segment !== '' && segment !== '.' && segment !== '..')
186
- .map(segment => segment.replace(/[\u0000-\u001F\u007F]/g, '_'))
187
- .join('/');
188
- }