renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,113 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.EVENTS_EXTRACTION_VERSION = exports.EVENTS_MODEL = exports.repairMalformedUnicodeEscapes = exports.stripExtractorCodeFence = exports.ExtractorParseError = void 0;
4
- exports.chronicleEnabled = chronicleEnabled;
5
- exports.extractEvents = extractEvents;
6
- exports.extractEventsWithRetry = extractEventsWithRetry;
7
- const typedFacts_1 = require("./typedFacts");
8
- Object.defineProperty(exports, "ExtractorParseError", { enumerable: true, get: function () { return typedFacts_1.ExtractorParseError; } });
9
- Object.defineProperty(exports, "stripExtractorCodeFence", { enumerable: true, get: function () { return typedFacts_1.stripExtractorCodeFence; } });
10
- Object.defineProperty(exports, "repairMalformedUnicodeEscapes", { enumerable: true, get: function () { return typedFacts_1.repairMalformedUnicodeEscapes; } });
11
- /**
12
- * C3 chronicle event extraction (HYP-20260921-chronicle-extraction, ADR-0003
13
- * §4 payload schema for the event lane). v1 schema only — when, event,
14
- * entities — frozen prompt + model. `when` REPRODUCES THE USER'S OWN WORDING
15
- * ("last Saturday", relative days, times); the row's valid_from is derived
16
- * from the note's frontmatter date and stays NULL when unknown — a relative
17
- * `when` is never resolved into an invented normalized date. Ledger KILLs
18
- * bound this module: N12/N14/N16b/N20/N25 (extractor-prompt variants are a
19
- * structural ceiling — no v2/v3/v4 prompt iteration), H5 (one-pass multi-head
20
- * — 48% row loss, so this extracts events only), and H6 (zero-yield prefilter
21
- * KILL — the hash-keyed skip is the only cost control). Parse failures are
22
- * EXPLICIT extractor-parse-errors, retried ≤3 times, never silently `[]`, and
23
- * never cached (pipeline resets the note hash so the next pass retries).
24
- */
25
- exports.EVENTS_MODEL = 'gemma4:31b';
26
- /** extraction_version = schema:model (ADR §4): hash-keyed cache key component. */
27
- exports.EVENTS_EXTRACTION_VERSION = `events-v1:${exports.EVENTS_MODEL}`;
28
- /** RENOR_CHRONICLE (default OFF). Opt in with 1/true/yes/on. No default flip. */
29
- function chronicleEnabled(env = process.env) {
30
- const raw = (env.RENOR_CHRONICLE ?? '').trim().toLowerCase();
31
- return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on';
32
- }
33
- const SYSTEM_PROMPT = `Extract KEY EVENTS the USER describes from the note. Return STRICT JSON only — no YAML, no markdown code fences, no commentary.
34
-
35
- Shape: {"events": [{"when": "<exactly the user's own wording>", "event": "<short phrase, 3-8 words>", "entities": ["<comma-separated entities>"]}]}
36
-
37
- Rules:
38
- - when: reproduce the user's own wording for the time ("last Saturday", "yesterday", "March 3rd", "this morning") — NEVER convert it to a normalized or invented date
39
- - event: short description of what happened (did / attended / started / cancelled / bought / met / visited / celebrated / completed / plans)
40
- - entities: people, places, things involved; omit if none
41
- - Only events the USER did or experienced — not the assistant, not hypotheticals, not third parties
42
- - Each distinct event ONCE. Extract 0-15 events.
43
- - Do NOT extract from frontmatter metadata.`;
44
- async function extractEvents(client, noteTitle, noteBody) {
45
- // Truncate body to keep the prompt bounded (same budget as the legacy extractor).
46
- const body = noteBody.substring(0, 3000);
47
- const response = await client.chat.completions.create({
48
- model: exports.EVENTS_MODEL,
49
- temperature: 0,
50
- response_format: { type: 'json_object' },
51
- messages: [
52
- { role: 'system', content: SYSTEM_PROMPT },
53
- { role: 'user', content: `Note: "${noteTitle}"\n\n${body}` },
54
- ],
55
- });
56
- const content = response.choices?.[0]?.message?.content;
57
- if (!content || !content.trim()) {
58
- throw new typedFacts_1.ExtractorParseError('extractor-parse-error: event extraction returned empty content');
59
- }
60
- // JSON-strict after fence-strip: a fenced or bare response parses; YAML,
61
- // truncated JSON, or commentary still fails hard as an explicit parse error.
62
- let parsed;
63
- try {
64
- parsed = JSON.parse((0, typedFacts_1.repairMalformedUnicodeEscapes)((0, typedFacts_1.stripExtractorCodeFence)(content)));
65
- }
66
- catch {
67
- throw new typedFacts_1.ExtractorParseError(`extractor-parse-error: event extraction returned non-JSON response (head: ${content.slice(0, 80)})`);
68
- }
69
- if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.events)) {
70
- throw new typedFacts_1.ExtractorParseError('extractor-parse-error: event extraction response is missing an events array');
71
- }
72
- const events = [];
73
- for (const raw of parsed.events) {
74
- const when = raw?.when === undefined || raw?.when === null ? '' : String(raw.when).trim();
75
- const event = raw?.event === undefined || raw?.event === null ? '' : String(raw.event).trim();
76
- if (!when || !event)
77
- continue; // malformed rows are dropped, not fatal
78
- const extracted = { when, event };
79
- const entities = raw?.entities;
80
- if (Array.isArray(entities) && entities.length > 0) {
81
- const cleaned = entities.map((e) => String(e).trim()).filter(Boolean);
82
- if (cleaned.length > 0)
83
- extracted.entities = cleaned;
84
- }
85
- else if (typeof entities === 'string' && entities.trim()) {
86
- // The research prompt asked for comma-separated entities; accept either shape.
87
- const cleaned = entities.split(',').map((e) => e.trim()).filter(Boolean);
88
- if (cleaned.length > 0)
89
- extracted.entities = cleaned;
90
- }
91
- events.push(extracted);
92
- }
93
- return events;
94
- }
95
- /**
96
- * Retry wrapper: parse failures are retried up to MAX_EXTRACT_ATTEMPTS total;
97
- * the final failure is rethrown explicitly (never cached, never silent []).
98
- * Non-parse errors (network, auth) propagate immediately.
99
- */
100
- async function extractEventsWithRetry(extract, noteTitle, noteBody) {
101
- let lastError;
102
- for (let attempt = 1; attempt <= typedFacts_1.MAX_EXTRACT_ATTEMPTS; attempt++) {
103
- try {
104
- return await extract(noteTitle, noteBody);
105
- }
106
- catch (err) {
107
- if (!(err instanceof typedFacts_1.ExtractorParseError))
108
- throw err;
109
- lastError = err;
110
- }
111
- }
112
- throw lastError;
113
- }
@@ -1,122 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.isExtractorSpecificProvider = isExtractorSpecificProvider;
4
- exports.resolveExtractorClientConfig = resolveExtractorClientConfig;
5
- exports.createExtractorClient = createExtractorClient;
6
- exports.openExtractorClient = openExtractorClient;
7
- const ollamaChat_1 = require("../evidence/ollamaChat");
8
- const keyPool_1 = require("../evidence/keyPool");
9
- /**
10
- * Whether the resolved provider is one of the extractor-specific providers (the
11
- * dedicated RENOR_EXTRACTOR_* slot or the Ollama Cloud pool) rather than the
12
- * generic OPENAI_API_KEY fallback.
13
- *
14
- * The legacy `LLM_EXTRACTION` fact path reads this to choose its model id: it
15
- * names `FACTS_MODEL` only for an extractor-specific provider, and on the generic
16
- * fallback it keeps the fact extractor's own default, which is the model `main`
17
- * sent there. The structured-row lanes are NOT decided here — their extractors
18
- * name `FACTS_MODEL` on every provider branch, including the fallback, which is
19
- * also what `main` did. Changing that is a separate decision, recorded as a
20
- * finding in this branch's review packet, not a property this helper can fix.
21
- */
22
- function isExtractorSpecificProvider(config) {
23
- return !!config && config.provider !== 'openai-fallback';
24
- }
25
- const DEFAULT_BASE_URL = 'https://ollama.com/v1';
26
- function nonEmpty(value) {
27
- const trimmed = (value ?? '').trim();
28
- return trimmed === '' ? undefined : trimmed;
29
- }
30
- function splitKeys(value) {
31
- return (value ?? '').split(',').map(key => key.trim()).filter(Boolean);
32
- }
33
- function resolveExtractorClientConfig(env = process.env, model) {
34
- const dedicatedKeys = [
35
- ...splitKeys(env.RENOR_EXTRACTOR_API_KEYS),
36
- ...splitKeys(env.RENOR_EXTRACTOR_API_KEY),
37
- ];
38
- if (dedicatedKeys.length > 0) {
39
- return {
40
- baseUrl: nonEmpty(env.RENOR_EXTRACTOR_BASE_URL) ?? DEFAULT_BASE_URL,
41
- keys: dedicatedKeys,
42
- model,
43
- provider: 'dedicated',
44
- keyEnvNames: [
45
- ...(splitKeys(env.RENOR_EXTRACTOR_API_KEYS).length > 0 ? ['RENOR_EXTRACTOR_API_KEYS'] : []),
46
- ...(splitKeys(env.RENOR_EXTRACTOR_API_KEY).length > 0 ? ['RENOR_EXTRACTOR_API_KEY'] : []),
47
- ],
48
- };
49
- }
50
- const ollama = (0, keyPool_1.collectOllamaKeys)(env);
51
- const ollamaKeys = ollama.keys;
52
- const usedOllamaNames = ollama.names;
53
- if (ollamaKeys.length > 0) {
54
- return {
55
- baseUrl: nonEmpty(env.RENOR_EXTRACTOR_BASE_URL) ?? DEFAULT_BASE_URL,
56
- keys: ollamaKeys,
57
- model,
58
- provider: 'ollama-cloud',
59
- keyEnvNames: usedOllamaNames,
60
- };
61
- }
62
- const fallbackBase = nonEmpty(env.OPENAI_BASE_URL) ?? nonEmpty(env.EMBEDDING_BASE_URL);
63
- const fallbackKey = nonEmpty(env.OPENAI_API_KEY);
64
- if (fallbackKey && fallbackBase) {
65
- return {
66
- baseUrl: fallbackBase,
67
- keys: splitKeys(env.OPENAI_API_KEY),
68
- model,
69
- provider: 'openai-fallback',
70
- keyEnvNames: ['OPENAI_API_KEY'],
71
- };
72
- }
73
- return null;
74
- }
75
- /**
76
- * OpenAI-shaped handle over the shared chat transport. Forwards
77
- * `response_format` when the extractor set it. Dropping that field made JSON
78
- * mode a silent no-op.
79
- */
80
- function createExtractorClient(config) {
81
- const chat = (0, ollamaChat_1.createOllamaChat)({
82
- baseUrl: config.baseUrl,
83
- keys: config.keys,
84
- timeoutMs: Number(process.env.RENOR_EXTRACTOR_TIMEOUT_MS ?? 180_000),
85
- maxRetries: 3,
86
- });
87
- return {
88
- chat: {
89
- completions: {
90
- create: async (request) => {
91
- const messages = (request.messages ?? []).filter(message => message
92
- && (message.role === 'user' || message.role === 'system')
93
- && typeof message.content === 'string');
94
- const response = await chat({
95
- model: request.model ?? config.model,
96
- messages: messages.map(message => ({
97
- role: message.role,
98
- content: message.content,
99
- })),
100
- temperature: 0,
101
- ...(request.response_format?.type === 'json_object'
102
- ? { response_format: { type: 'json_object' } }
103
- : {}),
104
- });
105
- return {
106
- choices: [{ message: { content: response.content } }],
107
- usage: {
108
- prompt_tokens: response.promptTokens,
109
- completion_tokens: response.completionTokens,
110
- },
111
- };
112
- },
113
- },
114
- },
115
- };
116
- }
117
- function openExtractorClient(model, env = process.env) {
118
- const config = resolveExtractorClientConfig(env, model);
119
- if (!config)
120
- return null;
121
- return { client: createExtractorClient(config), config };
122
- }
@@ -1,46 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.extractFacts = extractFacts;
4
- async function extractFacts(client, noteTitle, noteBody, model = 'gpt-4o-mini') {
5
- // Truncate body to ~3000 chars to stay within reasonable token budget
6
- const body = noteBody.substring(0, 3000);
7
- const response = await client.chat.completions.create({
8
- model,
9
- temperature: 0,
10
- response_format: { type: 'json_object' },
11
- messages: [
12
- {
13
- role: 'system',
14
- content: `Extract factual statements from the note. Return JSON: {"facts": [{"subject": "...", "predicate": "...", "object": "...", "confidence": 0.0-1.0}]}
15
-
16
- Rules:
17
- - subject: the entity the fact is about (use the note title if the note is about one main entity)
18
- - predicate: a short verb phrase (e.g. "works_at", "uses", "depends_on", "is_a", "located_in", "deadline", "created_by")
19
- - object: the target entity or value
20
- - confidence: 0.0-1.0 based on how explicitly the fact is stated
21
- - Extract 0-10 facts. Only extract facts clearly stated in the text.
22
- - Do NOT extract facts from frontmatter metadata — those are handled separately.
23
- - Keep predicates consistent and snake_case.`
24
- },
25
- {
26
- role: 'user',
27
- content: `Note: "${noteTitle}"\n\n${body}`
28
- }
29
- ]
30
- });
31
- const content = response.choices[0]?.message.content;
32
- if (!content) {
33
- throw new Error('Fact extraction returned empty content');
34
- }
35
- let parsed;
36
- try {
37
- parsed = JSON.parse(content);
38
- }
39
- catch {
40
- throw new Error('Fact extraction returned invalid JSON');
41
- }
42
- if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.facts)) {
43
- throw new Error('Fact extraction response is missing a facts array');
44
- }
45
- return parsed.facts.filter((f) => Boolean(f.subject && f.predicate && f.object && typeof f.confidence === 'number'));
46
- }
@@ -1,179 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.FACTS_EXTRACTION_VERSION = void 0;
4
- exports.factsWritePathEnabled = factsWritePathEnabled;
5
- exports.contentValidFrom = contentValidFrom;
6
- exports.extractFactCandidates = extractFactCandidates;
7
- exports.captureNoteFacts = captureNoteFacts;
8
- const rowStore_1 = require("../context/rowStore");
9
- const decision_model_1 = require("./decision-model");
10
- /**
11
- * C2 facts write-path (ADR-0003 §4/§6, component-matrix C2): save-time typed
12
- * numeric-fact capture from real markdown through the REAL index pipeline
13
- * (indexVault → extraction → structured-row store). Facts flow into the C0
14
- * rowStore, whose identity-keyed supersession extends the battle-tested
15
- * lifecycle bitemporal model (valid_from/valid_to) to extracted rows — the
16
- * markdown Memory Events Ledger stays `_mcp_`-lifecycle-only per
17
- * inventory-facts (its semantics do not apply to extracted note facts), so
18
- * extracted facts are NOT appended there.
19
- *
20
- * Flag gate: RENOR_FACTS_WRITE_PATH (default OFF). Flag off = zero decision
21
- * calls, zero new writes, byte-identical shipped behavior. The ask-time
22
- * evidence second pass is untouched (C2 is save-time; both compose).
23
- *
24
- * v1 schema only — no v2/v3/v4 prompt variants (five extractor KILLs). The
25
- * extractor is the local-rules path TODAY (no live model calls, quota=0); a
26
- * model-backed extractor plugs in through the DecisionModel seam.
27
- */
28
- exports.FACTS_EXTRACTION_VERSION = 'c2-facts-v1';
29
- /** RENOR_FACTS_WRITE_PATH (default OFF). Opt in with 1/true/yes/on. */
30
- function factsWritePathEnabled(env = process.env) {
31
- const raw = (env.RENOR_FACTS_WRITE_PATH ?? '').trim().toLowerCase();
32
- return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on';
33
- }
34
- /** Content dates: frontmatter `session_date`/`date`; unknown stays unknown (NULL — never `now`, never invented). */
35
- const CONTENT_DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:]+(\.\d+)?(Z|[+-]\d{2}:\d{2})?)?$/;
36
- function contentValidFrom(frontmatter) {
37
- for (const key of ['session_date', 'date']) {
38
- const raw = frontmatter[key];
39
- if (raw instanceof Date && !Number.isNaN(raw.getTime())) {
40
- return raw.toISOString().slice(0, 10); // YAML unquoted dates parse as Date; date-only UTC midnight
41
- }
42
- if (typeof raw === 'string' && CONTENT_DATE_RE.test(raw.trim()))
43
- return raw.trim();
44
- }
45
- return null;
46
- }
47
- /**
48
- * v1 numeric-fact candidates from gate-approved chunk text. Deterministic:
49
- * sentence-split, rule-matched value/unit/verb, note title as object (the
50
- * legacy extractor's same "note about one main entity" guidance). No model
51
- * call, no parse-failure mode — JSON/YAML parse errors cannot arise from a
52
- * local deterministic extractor by construction.
53
- */
54
- function extractFactCandidates(text, noteTitle) {
55
- if (!text || !text.trim())
56
- return [];
57
- const sentences = text.split(/(?<=[.!?])\s+|\n+/).map((s) => s.trim()).filter(Boolean);
58
- const out = [];
59
- for (const sentence of sentences) {
60
- if (decision_model_1.LocalRulesDecisionModel.captureGate(sentence) !== 'fact')
61
- continue;
62
- const valueMatch = decision_model_1.NUMERIC_FACT_RE.exec(sentence);
63
- if (!valueMatch)
64
- continue;
65
- const cueMatch = decision_model_1.STATIVE_CUE_RE.exec(sentence);
66
- out.push({
67
- value: valueMatch[1].replace(',', '.'),
68
- unit: valueMatch[2] ? valueMatch[2].toLowerCase() : undefined,
69
- verb: cueMatch ? cueMatch[1].toLowerCase() : undefined,
70
- object: noteTitle,
71
- event: sentence.length > 300 ? sentence.slice(0, 300) : sentence,
72
- });
73
- }
74
- return out;
75
- }
76
- /**
77
- * Capture one note's typed facts: per-chunk capture-gate decision → candidate
78
- * extraction → dedupe adjudication against the current row with the same
79
- * dedupe identity → write (structured row, or confirm-count bump on
80
- * `identical`; a cross-note `identical` additionally writes the confirming
81
- * note's own source-scoped row so the fact survives either note's deletion). Adjudicates against the lowest-id current row so repeated
82
- * runs are deterministic. Hash-keyed skip: rows already present for
83
- * (source_path, source_hash, extraction_version) → zero decisions (the
84
- * pipeline additionally skips unchanged notes via note_hash before this runs).
85
- */
86
- async function captureNoteFacts(db, args, model = (0, decision_model_1.defaultDecisionModel)()) {
87
- const hashSkip = db.prepare(`SELECT COUNT(*) AS c FROM structured_rows WHERE source_path = ? AND source_hash = ? AND extraction_version = ?`).get(args.notePath, args.sourceHash, exports.FACTS_EXTRACTION_VERSION);
88
- if (hashSkip.c > 0)
89
- return { rowsWritten: 0, confirmBumps: 0, decisions: 0 };
90
- const validFrom = contentValidFrom(args.frontmatter);
91
- const findCurrent = db.prepare(`SELECT id, source_path, payload_json FROM structured_rows
92
- WHERE row_kind = 'fact' AND scope = 'global' AND dedupe_key = ? AND valid_to IS NULL
93
- ORDER BY id LIMIT 1`);
94
- const bumpConfirm = db.prepare(`UPDATE structured_rows SET confirm_count = confirm_count + 1 WHERE row_kind = 'fact' AND scope = 'global' AND dedupe_key = ? AND valid_to IS NULL`);
95
- // Does THIS note already hold a current row for the identity? Guards the
96
- // cross-note confirm write against double-inserting when several chunks of
97
- // the same note adjudicate `identical` against the original note's row.
98
- const findOwnCurrent = db.prepare(`SELECT id FROM structured_rows
99
- WHERE row_kind = 'fact' AND scope = 'global' AND dedupe_key = ? AND source_path = ? AND valid_to IS NULL
100
- LIMIT 1`);
101
- let rowsWritten = 0;
102
- let confirmBumps = 0;
103
- let decisions = 0;
104
- for (const chunk of args.chunks) {
105
- const gate = await model.decide({ task: 'capture_gate', text: chunk.text });
106
- decisions++;
107
- if (gate.task !== 'capture_gate' || gate.decision !== 'fact')
108
- continue;
109
- for (const candidate of extractFactCandidates(chunk.text, args.noteTitle)) {
110
- const dedupeKey = (0, rowStore_1.dedupeKeyFor)({
111
- rowKind: 'fact',
112
- sourcePath: args.notePath,
113
- sourceChunkId: chunk.id,
114
- sourceHash: args.sourceHash,
115
- extractionVersion: exports.FACTS_EXTRACTION_VERSION,
116
- validFrom,
117
- payload: candidate,
118
- });
119
- const existing = findCurrent.get(dedupeKey);
120
- const existingCandidate = existing
121
- ? JSON.parse(existing.payload_json)
122
- : null;
123
- const adjudication = await model.decide({
124
- task: 'dedupe_adjudication',
125
- candidate,
126
- existing: existingCandidate,
127
- });
128
- decisions++;
129
- if (adjudication.task !== 'dedupe_adjudication')
130
- continue;
131
- if (adjudication.decision === 'identical') {
132
- bumpConfirm.run(dedupeKey);
133
- confirmBumps++;
134
- // Same-note `identical` (re-extraction/edit): bump only, no duplicate
135
- // row (original C2 behavior). Cross-note `identical` — this note is
136
- // NOT the source of the compared row — must ALSO write its own
137
- // source-scoped row: the dedupe key has no source, so a bump-only
138
- // write leaves the fact pinned solely to the original note's row,
139
- // whose FK cascade dies with that note while this note still asserts
140
- // the fact (unsurfaceable via the retrieval-hit join).
141
- if (!existing || existing.source_path === args.notePath)
142
- continue;
143
- const own = findOwnCurrent.get(dedupeKey, args.notePath);
144
- if (own)
145
- continue; // already captured from another chunk/run of this note
146
- (0, rowStore_1.insertStructuredRows)(db, [
147
- {
148
- rowKind: 'fact',
149
- sourcePath: args.notePath,
150
- sourceChunkId: chunk.id,
151
- sourceHash: args.sourceHash,
152
- extractionVersion: exports.FACTS_EXTRACTION_VERSION,
153
- validFrom,
154
- payload: candidate,
155
- },
156
- ], { skipSupersession: true });
157
- rowsWritten++;
158
- continue;
159
- }
160
- // 'updated' and 'distinct' both write; the rowStore's identity-keyed
161
- // supersession (never blanket newest-wins — N18 KILL) closes the
162
- // collided current row only when both content dates are known and the
163
- // new source is not older. NULL valid_from never supersedes.
164
- (0, rowStore_1.insertStructuredRows)(db, [
165
- {
166
- rowKind: 'fact',
167
- sourcePath: args.notePath,
168
- sourceChunkId: chunk.id,
169
- sourceHash: args.sourceHash,
170
- extractionVersion: exports.FACTS_EXTRACTION_VERSION,
171
- validFrom,
172
- payload: candidate,
173
- },
174
- ]);
175
- rowsWritten++;
176
- }
177
- }
178
- return { rowsWritten, confirmBumps, decisions };
179
- }