renor-engine 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +73 -0
- package/README.md +105 -2
- package/dist/BUILD.json +6 -0
- package/dist/api/apiAuth.js +43 -0
- package/dist/api/server.js +172 -0
- package/dist/buildIdentity.js +132 -0
- package/dist/cli-extractors.js +46 -0
- package/dist/cli.js +491 -0
- package/dist/config.js +65 -0
- package/dist/context/composer.js +320 -0
- package/dist/context/readHonesty.js +478 -0
- package/dist/context/rowStore.js +238 -0
- package/dist/conversation/formatter.js +81 -0
- package/dist/conversation/parser.js +189 -0
- package/dist/conversation/types.js +2 -0
- package/dist/conversation/writer.js +42 -0
- package/dist/db/client.js +147 -0
- package/dist/db/indexSalvage.js +81 -0
- package/dist/db/migrations.js +642 -0
- package/dist/db/readCache.js +109 -0
- package/dist/db/schema.js +5 -0
- package/dist/db/statements.js +89 -0
- package/dist/digests/digestCache.js +142 -0
- package/dist/digests/digests.js +597 -0
- package/dist/digests/worker.js +348 -0
- package/dist/embeddings/caching.js +337 -0
- package/dist/embeddings/coalescing.js +177 -0
- package/dist/embeddings/interactive.js +103 -0
- package/dist/embeddings/local.js +68 -0
- package/dist/embeddings/openai.js +105 -0
- package/dist/embeddings/rebuild.js +166 -0
- package/dist/embeddings/simdScan.js +258 -0
- package/dist/embeddings/space.js +21 -0
- package/dist/embeddings/types.js +14 -0
- package/dist/embeddings/vectorIndex.js +675 -0
- package/dist/embeddings/vertex.js +224 -0
- package/dist/eval/artifacts.js +42 -0
- package/dist/eval/cacheReport.js +139 -0
- package/dist/eval/comparison.js +182 -0
- package/dist/eval/context700-candidate.js +145 -0
- package/dist/eval/evalCache.js +525 -0
- package/dist/eval/hashEmbedding.js +35 -0
- package/dist/eval/longmemeval.js +56 -0
- package/dist/eval/metrics.js +181 -0
- package/dist/eval/renorbenchBaseline.js +486 -0
- package/dist/eval/renorbenchDocuments.js +134 -0
- package/dist/eval/renorbenchProduct.js +182 -0
- package/dist/eval/runLedger.js +279 -0
- package/dist/eval/runManifest.js +479 -0
- package/dist/eval/scoring.js +197 -0
- package/dist/evidence/attach.js +267 -0
- package/dist/evidence/background.js +147 -0
- package/dist/evidence/evidenceCache.js +182 -0
- package/dist/evidence/gate.js +37 -0
- package/dist/evidence/keyPool.js +85 -0
- package/dist/evidence/ollamaChat.js +84 -0
- package/dist/evidence/optin.js +165 -0
- package/dist/evidence/options.js +130 -0
- package/dist/evidence/secondPass.js +456 -0
- package/dist/extraction/classifyCapture.js +34 -0
- package/dist/extraction/decision-model.js +174 -0
- package/dist/extraction/eventExtractor.js +113 -0
- package/dist/extraction/extractionClient.js +122 -0
- package/dist/extraction/factExtractor.js +46 -0
- package/dist/extraction/factsWritePath.js +179 -0
- package/dist/extraction/preferenceExtractor.js +153 -0
- package/dist/extraction/secretGuard.js +176 -0
- package/dist/extraction/secretScanner.js +665 -0
- package/dist/extraction/typedFacts.js +224 -0
- package/dist/extraction/writePathAdjudication.js +110 -0
- package/dist/importing/apply.js +644 -0
- package/dist/importing/bundle-write.js +80 -0
- package/dist/importing/cli.js +407 -0
- package/dist/importing/existing.js +49 -0
- package/dist/importing/facts.js +123 -0
- package/dist/importing/forgetImported.js +79 -0
- package/dist/importing/hosted.js +467 -0
- package/dist/importing/index.js +29 -0
- package/dist/importing/journalLog.js +96 -0
- package/dist/importing/markdown.js +98 -0
- package/dist/importing/plan.js +206 -0
- package/dist/importing/secretScreen.js +113 -0
- package/dist/importing/serialize.js +188 -0
- package/dist/importing/source.js +463 -0
- package/dist/importing/types.js +6 -0
- package/dist/indexer/chunker.js +319 -0
- package/dist/indexer/dailyEntries.js +63 -0
- package/dist/indexer/hasher.js +10 -0
- package/dist/indexer/laneRuns.js +48 -0
- package/dist/indexer/o200k.js +481 -0
- package/dist/indexer/o200k.json +8 -0
- package/dist/indexer/outsideRoot.js +77 -0
- package/dist/indexer/parser.js +167 -0
- package/dist/indexer/pipeline.js +1168 -0
- package/dist/indexer/staleness.js +201 -0
- package/dist/indexer/tokenizer.js +77 -0
- package/dist/indexer/walker.js +184 -0
- package/dist/mcp/hive.js +88 -0
- package/dist/mcp/requestErrors.js +47 -0
- package/dist/mcp/server.js +396 -0
- package/dist/memory/agentLease.js +43 -0
- package/dist/memory/confinedWrite.js +315 -0
- package/dist/memory/controls.js +73 -0
- package/dist/memory/dailyNote.js +265 -0
- package/dist/memory/durableAppend.js +57 -0
- package/dist/memory/factConflicts.js +203 -0
- package/dist/memory/factTerms.js +321 -0
- package/dist/memory/hiveNameHeader.cjs +10 -0
- package/dist/memory/inventory.js +55 -0
- package/dist/memory/invisibleText.js +20 -0
- package/dist/memory/lifecycle.js +1756 -0
- package/dist/memory/limits.js +13 -0
- package/dist/memory/requests.js +183 -0
- package/dist/memory/sourceName.js +200 -0
- package/dist/memory/whatsNew.js +158 -0
- package/dist/memory/writer.js +175 -0
- package/dist/memory/writerConflicts.js +85 -0
- package/dist/memorymap/memoryMap.js +1568 -0
- package/dist/memorymap/replacedJournal.js +58 -0
- package/dist/memorymap/replacements.js +405 -0
- package/dist/mvp/activity.js +86 -0
- package/dist/mvp/agentMemory.js +1744 -0
- package/dist/mvp/auth.js +531 -0
- package/dist/mvp/cli.js +73 -0
- package/dist/mvp/client-store.js +462 -0
- package/dist/mvp/compileCache.js +249 -0
- package/dist/mvp/compileSeed.js +27 -0
- package/dist/mvp/config.js +288 -0
- package/dist/mvp/connectionStatus.js +80 -0
- package/dist/mvp/contextRelevance.js +413 -0
- package/dist/mvp/engine.js +1065 -0
- package/dist/mvp/exportAll.js +359 -0
- package/dist/mvp/factLanePack.js +77 -0
- package/dist/mvp/factMarkers.js +49 -0
- package/dist/mvp/factsLane.js +203 -0
- package/dist/mvp/hiddenSpans.js +585 -0
- package/dist/mvp/history.js +413 -0
- package/dist/mvp/hitDates.js +31 -0
- package/dist/mvp/hitFloor.js +36 -0
- package/dist/mvp/identity.js +18 -0
- package/dist/mvp/idle.js +113 -0
- package/dist/mvp/memoryDir.js +98 -0
- package/dist/mvp/posthogMcp.js +210 -0
- package/dist/mvp/readScope.js +104 -0
- package/dist/mvp/readUsefulness.js +222 -0
- package/dist/mvp/sanitize.js +48 -0
- package/dist/mvp/schemaRejections.js +91 -0
- package/dist/mvp/scope.js +127 -0
- package/dist/mvp/server.js +2089 -0
- package/dist/mvp/serverCard.js +29 -0
- package/dist/mvp/storedSize.js +139 -0
- package/dist/mvp/telemetry.js +441 -0
- package/dist/retrieval/adviceLines.js +195 -0
- package/dist/retrieval/adviceQuestion.js +44 -0
- package/dist/retrieval/conversationShape.js +44 -0
- package/dist/retrieval/dailyEntryTime.js +55 -0
- package/dist/retrieval/entityLookup.js +113 -0
- package/dist/retrieval/factsFirst.js +188 -0
- package/dist/retrieval/keyword.js +84 -0
- package/dist/retrieval/ledgerEventTime.js +51 -0
- package/dist/retrieval/ledgerLiveness.js +83 -0
- package/dist/retrieval/ledgerPresentation.js +161 -0
- package/dist/retrieval/noteIdentity.js +57 -0
- package/dist/retrieval/orchestrator.js +1086 -0
- package/dist/retrieval/queryHelpers.js +142 -0
- package/dist/retrieval/questionSpan.js +475 -0
- package/dist/retrieval/readerEvents.js +690 -0
- package/dist/retrieval/readerLayout.js +503 -0
- package/dist/retrieval/relativeTimeHints.js +113 -0
- package/dist/retrieval/semantic.js +67 -0
- package/dist/router/routerCache.js +106 -0
- package/dist/router/sessionRouter.js +726 -0
- package/dist/toolRefusal.js +120 -0
- package/dist/watcher/watcher.js +75 -0
- package/package.json +120 -4
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.EVENTS_EXTRACTION_VERSION = exports.EVENTS_MODEL = exports.repairMalformedUnicodeEscapes = exports.stripExtractorCodeFence = exports.ExtractorParseError = void 0;
|
|
4
|
+
exports.chronicleEnabled = chronicleEnabled;
|
|
5
|
+
exports.extractEvents = extractEvents;
|
|
6
|
+
exports.extractEventsWithRetry = extractEventsWithRetry;
|
|
7
|
+
const typedFacts_1 = require("./typedFacts");
|
|
8
|
+
Object.defineProperty(exports, "ExtractorParseError", { enumerable: true, get: function () { return typedFacts_1.ExtractorParseError; } });
|
|
9
|
+
Object.defineProperty(exports, "stripExtractorCodeFence", { enumerable: true, get: function () { return typedFacts_1.stripExtractorCodeFence; } });
|
|
10
|
+
Object.defineProperty(exports, "repairMalformedUnicodeEscapes", { enumerable: true, get: function () { return typedFacts_1.repairMalformedUnicodeEscapes; } });
|
|
11
|
+
/**
|
|
12
|
+
* C3 chronicle event extraction (HYP-20260921-chronicle-extraction, ADR-0003
|
|
13
|
+
* §4 payload schema for the event lane). v1 schema only — when, event,
|
|
14
|
+
* entities — frozen prompt + model. `when` REPRODUCES THE USER'S OWN WORDING
|
|
15
|
+
* ("last Saturday", relative days, times); the row's valid_from is derived
|
|
16
|
+
* from the note's frontmatter date and stays NULL when unknown — a relative
|
|
17
|
+
* `when` is never resolved into an invented normalized date. Ledger KILLs
|
|
18
|
+
* bound this module: N12/N14/N16b/N20/N25 (extractor-prompt variants are a
|
|
19
|
+
* structural ceiling — no v2/v3/v4 prompt iteration), H5 (one-pass multi-head
|
|
20
|
+
* — 48% row loss, so this extracts events only), and H6 (zero-yield prefilter
|
|
21
|
+
* KILL — the hash-keyed skip is the only cost control). Parse failures are
|
|
22
|
+
* EXPLICIT extractor-parse-errors, retried ≤3 times, never silently `[]`, and
|
|
23
|
+
* never cached (pipeline resets the note hash so the next pass retries).
|
|
24
|
+
*/
|
|
25
|
+
exports.EVENTS_MODEL = 'gemma4:31b';
|
|
26
|
+
/** extraction_version = schema:model (ADR §4): hash-keyed cache key component. */
|
|
27
|
+
exports.EVENTS_EXTRACTION_VERSION = `events-v1:${exports.EVENTS_MODEL}`;
|
|
28
|
+
/** RENOR_CHRONICLE (default OFF). Opt in with 1/true/yes/on. No default flip. */
|
|
29
|
+
function chronicleEnabled(env = process.env) {
|
|
30
|
+
const raw = (env.RENOR_CHRONICLE ?? '').trim().toLowerCase();
|
|
31
|
+
return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on';
|
|
32
|
+
}
|
|
33
|
+
const SYSTEM_PROMPT = `Extract KEY EVENTS the USER describes from the note. Return STRICT JSON only — no YAML, no markdown code fences, no commentary.
|
|
34
|
+
|
|
35
|
+
Shape: {"events": [{"when": "<exactly the user's own wording>", "event": "<short phrase, 3-8 words>", "entities": ["<comma-separated entities>"]}]}
|
|
36
|
+
|
|
37
|
+
Rules:
|
|
38
|
+
- when: reproduce the user's own wording for the time ("last Saturday", "yesterday", "March 3rd", "this morning") — NEVER convert it to a normalized or invented date
|
|
39
|
+
- event: short description of what happened (did / attended / started / cancelled / bought / met / visited / celebrated / completed / plans)
|
|
40
|
+
- entities: people, places, things involved; omit if none
|
|
41
|
+
- Only events the USER did or experienced — not the assistant, not hypotheticals, not third parties
|
|
42
|
+
- Each distinct event ONCE. Extract 0-15 events.
|
|
43
|
+
- Do NOT extract from frontmatter metadata.`;
|
|
44
|
+
async function extractEvents(client, noteTitle, noteBody) {
|
|
45
|
+
// Truncate body to keep the prompt bounded (same budget as the legacy extractor).
|
|
46
|
+
const body = noteBody.substring(0, 3000);
|
|
47
|
+
const response = await client.chat.completions.create({
|
|
48
|
+
model: exports.EVENTS_MODEL,
|
|
49
|
+
temperature: 0,
|
|
50
|
+
response_format: { type: 'json_object' },
|
|
51
|
+
messages: [
|
|
52
|
+
{ role: 'system', content: SYSTEM_PROMPT },
|
|
53
|
+
{ role: 'user', content: `Note: "${noteTitle}"\n\n${body}` },
|
|
54
|
+
],
|
|
55
|
+
});
|
|
56
|
+
const content = response.choices?.[0]?.message?.content;
|
|
57
|
+
if (!content || !content.trim()) {
|
|
58
|
+
throw new typedFacts_1.ExtractorParseError('extractor-parse-error: event extraction returned empty content');
|
|
59
|
+
}
|
|
60
|
+
// JSON-strict after fence-strip: a fenced or bare response parses; YAML,
|
|
61
|
+
// truncated JSON, or commentary still fails hard as an explicit parse error.
|
|
62
|
+
let parsed;
|
|
63
|
+
try {
|
|
64
|
+
parsed = JSON.parse((0, typedFacts_1.repairMalformedUnicodeEscapes)((0, typedFacts_1.stripExtractorCodeFence)(content)));
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
throw new typedFacts_1.ExtractorParseError(`extractor-parse-error: event extraction returned non-JSON response (head: ${content.slice(0, 80)})`);
|
|
68
|
+
}
|
|
69
|
+
if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.events)) {
|
|
70
|
+
throw new typedFacts_1.ExtractorParseError('extractor-parse-error: event extraction response is missing an events array');
|
|
71
|
+
}
|
|
72
|
+
const events = [];
|
|
73
|
+
for (const raw of parsed.events) {
|
|
74
|
+
const when = raw?.when === undefined || raw?.when === null ? '' : String(raw.when).trim();
|
|
75
|
+
const event = raw?.event === undefined || raw?.event === null ? '' : String(raw.event).trim();
|
|
76
|
+
if (!when || !event)
|
|
77
|
+
continue; // malformed rows are dropped, not fatal
|
|
78
|
+
const extracted = { when, event };
|
|
79
|
+
const entities = raw?.entities;
|
|
80
|
+
if (Array.isArray(entities) && entities.length > 0) {
|
|
81
|
+
const cleaned = entities.map((e) => String(e).trim()).filter(Boolean);
|
|
82
|
+
if (cleaned.length > 0)
|
|
83
|
+
extracted.entities = cleaned;
|
|
84
|
+
}
|
|
85
|
+
else if (typeof entities === 'string' && entities.trim()) {
|
|
86
|
+
// The research prompt asked for comma-separated entities; accept either shape.
|
|
87
|
+
const cleaned = entities.split(',').map((e) => e.trim()).filter(Boolean);
|
|
88
|
+
if (cleaned.length > 0)
|
|
89
|
+
extracted.entities = cleaned;
|
|
90
|
+
}
|
|
91
|
+
events.push(extracted);
|
|
92
|
+
}
|
|
93
|
+
return events;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Retry wrapper: parse failures are retried up to MAX_EXTRACT_ATTEMPTS total;
|
|
97
|
+
* the final failure is rethrown explicitly (never cached, never silent []).
|
|
98
|
+
* Non-parse errors (network, auth) propagate immediately.
|
|
99
|
+
*/
|
|
100
|
+
async function extractEventsWithRetry(extract, noteTitle, noteBody) {
|
|
101
|
+
let lastError;
|
|
102
|
+
for (let attempt = 1; attempt <= typedFacts_1.MAX_EXTRACT_ATTEMPTS; attempt++) {
|
|
103
|
+
try {
|
|
104
|
+
return await extract(noteTitle, noteBody);
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
if (!(err instanceof typedFacts_1.ExtractorParseError))
|
|
108
|
+
throw err;
|
|
109
|
+
lastError = err;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
throw lastError;
|
|
113
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.isExtractorSpecificProvider = isExtractorSpecificProvider;
|
|
4
|
+
exports.resolveExtractorClientConfig = resolveExtractorClientConfig;
|
|
5
|
+
exports.createExtractorClient = createExtractorClient;
|
|
6
|
+
exports.openExtractorClient = openExtractorClient;
|
|
7
|
+
const ollamaChat_1 = require("../evidence/ollamaChat");
|
|
8
|
+
const keyPool_1 = require("../evidence/keyPool");
|
|
9
|
+
/**
|
|
10
|
+
* Whether the resolved provider is one of the extractor-specific providers (the
|
|
11
|
+
* dedicated RENOR_EXTRACTOR_* slot or the Ollama Cloud pool) rather than the
|
|
12
|
+
* generic OPENAI_API_KEY fallback.
|
|
13
|
+
*
|
|
14
|
+
* The legacy `LLM_EXTRACTION` fact path reads this to choose its model id: it
|
|
15
|
+
* names `FACTS_MODEL` only for an extractor-specific provider, and on the generic
|
|
16
|
+
* fallback it keeps the fact extractor's own default, which is the model `main`
|
|
17
|
+
* sent there. The structured-row lanes are NOT decided here — their extractors
|
|
18
|
+
* name `FACTS_MODEL` on every provider branch, including the fallback, which is
|
|
19
|
+
* also what `main` did. Changing that is a separate decision, recorded as a
|
|
20
|
+
* finding in this branch's review packet, not a property this helper can fix.
|
|
21
|
+
*/
|
|
22
|
+
function isExtractorSpecificProvider(config) {
|
|
23
|
+
return !!config && config.provider !== 'openai-fallback';
|
|
24
|
+
}
|
|
25
|
+
const DEFAULT_BASE_URL = 'https://ollama.com/v1';
|
|
26
|
+
function nonEmpty(value) {
|
|
27
|
+
const trimmed = (value ?? '').trim();
|
|
28
|
+
return trimmed === '' ? undefined : trimmed;
|
|
29
|
+
}
|
|
30
|
+
function splitKeys(value) {
|
|
31
|
+
return (value ?? '').split(',').map(key => key.trim()).filter(Boolean);
|
|
32
|
+
}
|
|
33
|
+
function resolveExtractorClientConfig(env = process.env, model) {
|
|
34
|
+
const dedicatedKeys = [
|
|
35
|
+
...splitKeys(env.RENOR_EXTRACTOR_API_KEYS),
|
|
36
|
+
...splitKeys(env.RENOR_EXTRACTOR_API_KEY),
|
|
37
|
+
];
|
|
38
|
+
if (dedicatedKeys.length > 0) {
|
|
39
|
+
return {
|
|
40
|
+
baseUrl: nonEmpty(env.RENOR_EXTRACTOR_BASE_URL) ?? DEFAULT_BASE_URL,
|
|
41
|
+
keys: dedicatedKeys,
|
|
42
|
+
model,
|
|
43
|
+
provider: 'dedicated',
|
|
44
|
+
keyEnvNames: [
|
|
45
|
+
...(splitKeys(env.RENOR_EXTRACTOR_API_KEYS).length > 0 ? ['RENOR_EXTRACTOR_API_KEYS'] : []),
|
|
46
|
+
...(splitKeys(env.RENOR_EXTRACTOR_API_KEY).length > 0 ? ['RENOR_EXTRACTOR_API_KEY'] : []),
|
|
47
|
+
],
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
const ollama = (0, keyPool_1.collectOllamaKeys)(env);
|
|
51
|
+
const ollamaKeys = ollama.keys;
|
|
52
|
+
const usedOllamaNames = ollama.names;
|
|
53
|
+
if (ollamaKeys.length > 0) {
|
|
54
|
+
return {
|
|
55
|
+
baseUrl: nonEmpty(env.RENOR_EXTRACTOR_BASE_URL) ?? DEFAULT_BASE_URL,
|
|
56
|
+
keys: ollamaKeys,
|
|
57
|
+
model,
|
|
58
|
+
provider: 'ollama-cloud',
|
|
59
|
+
keyEnvNames: usedOllamaNames,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
const fallbackBase = nonEmpty(env.OPENAI_BASE_URL) ?? nonEmpty(env.EMBEDDING_BASE_URL);
|
|
63
|
+
const fallbackKey = nonEmpty(env.OPENAI_API_KEY);
|
|
64
|
+
if (fallbackKey && fallbackBase) {
|
|
65
|
+
return {
|
|
66
|
+
baseUrl: fallbackBase,
|
|
67
|
+
keys: splitKeys(env.OPENAI_API_KEY),
|
|
68
|
+
model,
|
|
69
|
+
provider: 'openai-fallback',
|
|
70
|
+
keyEnvNames: ['OPENAI_API_KEY'],
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* OpenAI-shaped handle over the shared chat transport. Forwards
|
|
77
|
+
* `response_format` when the extractor set it. Dropping that field made JSON
|
|
78
|
+
* mode a silent no-op.
|
|
79
|
+
*/
|
|
80
|
+
function createExtractorClient(config) {
|
|
81
|
+
const chat = (0, ollamaChat_1.createOllamaChat)({
|
|
82
|
+
baseUrl: config.baseUrl,
|
|
83
|
+
keys: config.keys,
|
|
84
|
+
timeoutMs: Number(process.env.RENOR_EXTRACTOR_TIMEOUT_MS ?? 180_000),
|
|
85
|
+
maxRetries: 3,
|
|
86
|
+
});
|
|
87
|
+
return {
|
|
88
|
+
chat: {
|
|
89
|
+
completions: {
|
|
90
|
+
create: async (request) => {
|
|
91
|
+
const messages = (request.messages ?? []).filter(message => message
|
|
92
|
+
&& (message.role === 'user' || message.role === 'system')
|
|
93
|
+
&& typeof message.content === 'string');
|
|
94
|
+
const response = await chat({
|
|
95
|
+
model: request.model ?? config.model,
|
|
96
|
+
messages: messages.map(message => ({
|
|
97
|
+
role: message.role,
|
|
98
|
+
content: message.content,
|
|
99
|
+
})),
|
|
100
|
+
temperature: 0,
|
|
101
|
+
...(request.response_format?.type === 'json_object'
|
|
102
|
+
? { response_format: { type: 'json_object' } }
|
|
103
|
+
: {}),
|
|
104
|
+
});
|
|
105
|
+
return {
|
|
106
|
+
choices: [{ message: { content: response.content } }],
|
|
107
|
+
usage: {
|
|
108
|
+
prompt_tokens: response.promptTokens,
|
|
109
|
+
completion_tokens: response.completionTokens,
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
},
|
|
113
|
+
},
|
|
114
|
+
},
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
function openExtractorClient(model, env = process.env) {
|
|
118
|
+
const config = resolveExtractorClientConfig(env, model);
|
|
119
|
+
if (!config)
|
|
120
|
+
return null;
|
|
121
|
+
return { client: createExtractorClient(config), config };
|
|
122
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.extractFacts = extractFacts;
|
|
4
|
+
async function extractFacts(client, noteTitle, noteBody, model = 'gpt-4o-mini') {
|
|
5
|
+
// Truncate body to ~3000 chars to stay within reasonable token budget
|
|
6
|
+
const body = noteBody.substring(0, 3000);
|
|
7
|
+
const response = await client.chat.completions.create({
|
|
8
|
+
model,
|
|
9
|
+
temperature: 0,
|
|
10
|
+
response_format: { type: 'json_object' },
|
|
11
|
+
messages: [
|
|
12
|
+
{
|
|
13
|
+
role: 'system',
|
|
14
|
+
content: `Extract factual statements from the note. Return JSON: {"facts": [{"subject": "...", "predicate": "...", "object": "...", "confidence": 0.0-1.0}]}
|
|
15
|
+
|
|
16
|
+
Rules:
|
|
17
|
+
- subject: the entity the fact is about (use the note title if the note is about one main entity)
|
|
18
|
+
- predicate: a short verb phrase (e.g. "works_at", "uses", "depends_on", "is_a", "located_in", "deadline", "created_by")
|
|
19
|
+
- object: the target entity or value
|
|
20
|
+
- confidence: 0.0-1.0 based on how explicitly the fact is stated
|
|
21
|
+
- Extract 0-10 facts. Only extract facts clearly stated in the text.
|
|
22
|
+
- Do NOT extract facts from frontmatter metadata — those are handled separately.
|
|
23
|
+
- Keep predicates consistent and snake_case.`
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
role: 'user',
|
|
27
|
+
content: `Note: "${noteTitle}"\n\n${body}`
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
});
|
|
31
|
+
const content = response.choices[0]?.message.content;
|
|
32
|
+
if (!content) {
|
|
33
|
+
throw new Error('Fact extraction returned empty content');
|
|
34
|
+
}
|
|
35
|
+
let parsed;
|
|
36
|
+
try {
|
|
37
|
+
parsed = JSON.parse(content);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
throw new Error('Fact extraction returned invalid JSON');
|
|
41
|
+
}
|
|
42
|
+
if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.facts)) {
|
|
43
|
+
throw new Error('Fact extraction response is missing a facts array');
|
|
44
|
+
}
|
|
45
|
+
return parsed.facts.filter((f) => f.subject && f.predicate && f.object && typeof f.confidence === 'number');
|
|
46
|
+
}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FACTS_EXTRACTION_VERSION = void 0;
|
|
4
|
+
exports.factsWritePathEnabled = factsWritePathEnabled;
|
|
5
|
+
exports.contentValidFrom = contentValidFrom;
|
|
6
|
+
exports.extractFactCandidates = extractFactCandidates;
|
|
7
|
+
exports.captureNoteFacts = captureNoteFacts;
|
|
8
|
+
const rowStore_1 = require("../context/rowStore");
|
|
9
|
+
const decision_model_1 = require("./decision-model");
|
|
10
|
+
/**
|
|
11
|
+
* C2 facts write-path (ADR-0003 §4/§6, component-matrix C2): save-time typed
|
|
12
|
+
* numeric-fact capture from real markdown through the REAL index pipeline
|
|
13
|
+
* (indexVault → extraction → structured-row store). Facts flow into the C0
|
|
14
|
+
* rowStore, whose identity-keyed supersession extends the battle-tested
|
|
15
|
+
* lifecycle bitemporal model (valid_from/valid_to) to extracted rows — the
|
|
16
|
+
* markdown Memory Events Ledger stays `_mcp_`-lifecycle-only per
|
|
17
|
+
* inventory-facts (its semantics do not apply to extracted note facts), so
|
|
18
|
+
* extracted facts are NOT appended there.
|
|
19
|
+
*
|
|
20
|
+
* Flag gate: RENOR_FACTS_WRITE_PATH (default OFF). Flag off = zero decision
|
|
21
|
+
* calls, zero new writes, byte-identical shipped behavior. The ask-time
|
|
22
|
+
* evidence second pass is untouched (C2 is save-time; both compose).
|
|
23
|
+
*
|
|
24
|
+
* v1 schema only — no v2/v3/v4 prompt variants (five extractor KILLs). The
|
|
25
|
+
* extractor is the local-rules path TODAY (no live model calls, quota=0); a
|
|
26
|
+
* model-backed extractor plugs in through the DecisionModel seam.
|
|
27
|
+
*/
|
|
28
|
+
exports.FACTS_EXTRACTION_VERSION = 'c2-facts-v1';
|
|
29
|
+
/** RENOR_FACTS_WRITE_PATH (default OFF). Opt in with 1/true/yes/on. */
|
|
30
|
+
function factsWritePathEnabled(env = process.env) {
|
|
31
|
+
const raw = (env.RENOR_FACTS_WRITE_PATH ?? '').trim().toLowerCase();
|
|
32
|
+
return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on';
|
|
33
|
+
}
|
|
34
|
+
/** Content dates: frontmatter `session_date`/`date`; unknown stays unknown (NULL — never `now`, never invented). */
|
|
35
|
+
const CONTENT_DATE_RE = /^\d{4}-\d{2}-\d{2}(T[\d:]+(\.\d+)?(Z|[+-]\d{2}:\d{2})?)?$/;
|
|
36
|
+
function contentValidFrom(frontmatter) {
|
|
37
|
+
for (const key of ['session_date', 'date']) {
|
|
38
|
+
const raw = frontmatter[key];
|
|
39
|
+
if (raw instanceof Date && !Number.isNaN(raw.getTime())) {
|
|
40
|
+
return raw.toISOString().slice(0, 10); // YAML unquoted dates parse as Date; date-only UTC midnight
|
|
41
|
+
}
|
|
42
|
+
if (typeof raw === 'string' && CONTENT_DATE_RE.test(raw.trim()))
|
|
43
|
+
return raw.trim();
|
|
44
|
+
}
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* v1 numeric-fact candidates from gate-approved chunk text. Deterministic:
|
|
49
|
+
* sentence-split, rule-matched value/unit/verb, note title as object (the
|
|
50
|
+
* legacy extractor's same "note about one main entity" guidance). No model
|
|
51
|
+
* call, no parse-failure mode — JSON/YAML parse errors cannot arise from a
|
|
52
|
+
* local deterministic extractor by construction.
|
|
53
|
+
*/
|
|
54
|
+
function extractFactCandidates(text, noteTitle) {
|
|
55
|
+
if (!text || !text.trim())
|
|
56
|
+
return [];
|
|
57
|
+
const sentences = text.split(/(?<=[.!?])\s+|\n+/).map((s) => s.trim()).filter(Boolean);
|
|
58
|
+
const out = [];
|
|
59
|
+
for (const sentence of sentences) {
|
|
60
|
+
if (decision_model_1.LocalRulesDecisionModel.captureGate(sentence) !== 'fact')
|
|
61
|
+
continue;
|
|
62
|
+
const valueMatch = decision_model_1.NUMERIC_FACT_RE.exec(sentence);
|
|
63
|
+
if (!valueMatch)
|
|
64
|
+
continue;
|
|
65
|
+
const cueMatch = decision_model_1.STATIVE_CUE_RE.exec(sentence);
|
|
66
|
+
out.push({
|
|
67
|
+
value: valueMatch[1].replace(',', '.'),
|
|
68
|
+
unit: valueMatch[2] ? valueMatch[2].toLowerCase() : undefined,
|
|
69
|
+
verb: cueMatch ? cueMatch[1].toLowerCase() : undefined,
|
|
70
|
+
object: noteTitle,
|
|
71
|
+
event: sentence.length > 300 ? sentence.slice(0, 300) : sentence,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Capture one note's typed facts: per-chunk capture-gate decision → candidate
|
|
78
|
+
* extraction → dedupe adjudication against the current row with the same
|
|
79
|
+
* dedupe identity → write (structured row, or confirm-count bump on
|
|
80
|
+
* `identical`; a cross-note `identical` additionally writes the confirming
|
|
81
|
+
* note's own source-scoped row so the fact survives either note's deletion). Adjudicates against the lowest-id current row so repeated
|
|
82
|
+
* runs are deterministic. Hash-keyed skip: rows already present for
|
|
83
|
+
* (source_path, source_hash, extraction_version) → zero decisions (the
|
|
84
|
+
* pipeline additionally skips unchanged notes via note_hash before this runs).
|
|
85
|
+
*/
|
|
86
|
+
async function captureNoteFacts(db, args, model = (0, decision_model_1.defaultDecisionModel)()) {
|
|
87
|
+
const hashSkip = db.prepare(`SELECT COUNT(*) AS c FROM structured_rows WHERE source_path = ? AND source_hash = ? AND extraction_version = ?`).get(args.notePath, args.sourceHash, exports.FACTS_EXTRACTION_VERSION);
|
|
88
|
+
if (hashSkip.c > 0)
|
|
89
|
+
return { rowsWritten: 0, confirmBumps: 0, decisions: 0 };
|
|
90
|
+
const validFrom = contentValidFrom(args.frontmatter);
|
|
91
|
+
const findCurrent = db.prepare(`SELECT id, source_path, payload_json FROM structured_rows
|
|
92
|
+
WHERE row_kind = 'fact' AND scope = 'global' AND dedupe_key = ? AND valid_to IS NULL
|
|
93
|
+
ORDER BY id LIMIT 1`);
|
|
94
|
+
const bumpConfirm = db.prepare(`UPDATE structured_rows SET confirm_count = confirm_count + 1 WHERE row_kind = 'fact' AND scope = 'global' AND dedupe_key = ? AND valid_to IS NULL`);
|
|
95
|
+
// Does THIS note already hold a current row for the identity? Guards the
|
|
96
|
+
// cross-note confirm write against double-inserting when several chunks of
|
|
97
|
+
// the same note adjudicate `identical` against the original note's row.
|
|
98
|
+
const findOwnCurrent = db.prepare(`SELECT id FROM structured_rows
|
|
99
|
+
WHERE row_kind = 'fact' AND scope = 'global' AND dedupe_key = ? AND source_path = ? AND valid_to IS NULL
|
|
100
|
+
LIMIT 1`);
|
|
101
|
+
let rowsWritten = 0;
|
|
102
|
+
let confirmBumps = 0;
|
|
103
|
+
let decisions = 0;
|
|
104
|
+
for (const chunk of args.chunks) {
|
|
105
|
+
const gate = await model.decide({ task: 'capture_gate', text: chunk.text });
|
|
106
|
+
decisions++;
|
|
107
|
+
if (gate.task !== 'capture_gate' || gate.decision !== 'fact')
|
|
108
|
+
continue;
|
|
109
|
+
for (const candidate of extractFactCandidates(chunk.text, args.noteTitle)) {
|
|
110
|
+
const dedupeKey = (0, rowStore_1.dedupeKeyFor)({
|
|
111
|
+
rowKind: 'fact',
|
|
112
|
+
sourcePath: args.notePath,
|
|
113
|
+
sourceChunkId: chunk.id,
|
|
114
|
+
sourceHash: args.sourceHash,
|
|
115
|
+
extractionVersion: exports.FACTS_EXTRACTION_VERSION,
|
|
116
|
+
validFrom,
|
|
117
|
+
payload: candidate,
|
|
118
|
+
});
|
|
119
|
+
const existing = findCurrent.get(dedupeKey);
|
|
120
|
+
const existingCandidate = existing
|
|
121
|
+
? JSON.parse(existing.payload_json)
|
|
122
|
+
: null;
|
|
123
|
+
const adjudication = await model.decide({
|
|
124
|
+
task: 'dedupe_adjudication',
|
|
125
|
+
candidate,
|
|
126
|
+
existing: existingCandidate,
|
|
127
|
+
});
|
|
128
|
+
decisions++;
|
|
129
|
+
if (adjudication.task !== 'dedupe_adjudication')
|
|
130
|
+
continue;
|
|
131
|
+
if (adjudication.decision === 'identical') {
|
|
132
|
+
bumpConfirm.run(dedupeKey);
|
|
133
|
+
confirmBumps++;
|
|
134
|
+
// Same-note `identical` (re-extraction/edit): bump only, no duplicate
|
|
135
|
+
// row (original C2 behavior). Cross-note `identical` — this note is
|
|
136
|
+
// NOT the source of the compared row — must ALSO write its own
|
|
137
|
+
// source-scoped row: the dedupe key has no source, so a bump-only
|
|
138
|
+
// write leaves the fact pinned solely to the original note's row,
|
|
139
|
+
// whose FK cascade dies with that note while this note still asserts
|
|
140
|
+
// the fact (unsurfaceable via the retrieval-hit join).
|
|
141
|
+
if (!existing || existing.source_path === args.notePath)
|
|
142
|
+
continue;
|
|
143
|
+
const own = findOwnCurrent.get(dedupeKey, args.notePath);
|
|
144
|
+
if (own)
|
|
145
|
+
continue; // already captured from another chunk/run of this note
|
|
146
|
+
(0, rowStore_1.insertStructuredRows)(db, [
|
|
147
|
+
{
|
|
148
|
+
rowKind: 'fact',
|
|
149
|
+
sourcePath: args.notePath,
|
|
150
|
+
sourceChunkId: chunk.id,
|
|
151
|
+
sourceHash: args.sourceHash,
|
|
152
|
+
extractionVersion: exports.FACTS_EXTRACTION_VERSION,
|
|
153
|
+
validFrom,
|
|
154
|
+
payload: candidate,
|
|
155
|
+
},
|
|
156
|
+
], { skipSupersession: true });
|
|
157
|
+
rowsWritten++;
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
// 'updated' and 'distinct' both write; the rowStore's identity-keyed
|
|
161
|
+
// supersession (never blanket newest-wins — N18 KILL) closes the
|
|
162
|
+
// collided current row only when both content dates are known and the
|
|
163
|
+
// new source is not older. NULL valid_from never supersedes.
|
|
164
|
+
(0, rowStore_1.insertStructuredRows)(db, [
|
|
165
|
+
{
|
|
166
|
+
rowKind: 'fact',
|
|
167
|
+
sourcePath: args.notePath,
|
|
168
|
+
sourceChunkId: chunk.id,
|
|
169
|
+
sourceHash: args.sourceHash,
|
|
170
|
+
extractionVersion: exports.FACTS_EXTRACTION_VERSION,
|
|
171
|
+
validFrom,
|
|
172
|
+
payload: candidate,
|
|
173
|
+
},
|
|
174
|
+
]);
|
|
175
|
+
rowsWritten++;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return { rowsWritten, confirmBumps, decisions };
|
|
179
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PREFS_EXTRACTION_VERSION = exports.PREFS_MODEL = exports.repairMalformedUnicodeEscapes = exports.stripExtractorCodeFence = exports.ExtractorParseError = void 0;
|
|
4
|
+
exports.prefsEnabled = prefsEnabled;
|
|
5
|
+
exports.looksLikeOneOff = looksLikeOneOff;
|
|
6
|
+
exports.extractPreferences = extractPreferences;
|
|
7
|
+
exports.extractPreferencesWithRetry = extractPreferencesWithRetry;
|
|
8
|
+
const typedFacts_1 = require("./typedFacts");
|
|
9
|
+
Object.defineProperty(exports, "ExtractorParseError", { enumerable: true, get: function () { return typedFacts_1.ExtractorParseError; } });
|
|
10
|
+
Object.defineProperty(exports, "stripExtractorCodeFence", { enumerable: true, get: function () { return typedFacts_1.stripExtractorCodeFence; } });
|
|
11
|
+
Object.defineProperty(exports, "repairMalformedUnicodeEscapes", { enumerable: true, get: function () { return typedFacts_1.repairMalformedUnicodeEscapes; } });
|
|
12
|
+
/**
|
|
13
|
+
* C4 standing-preference extraction (HYP-20260921-prefs-extraction, ADR-0003
|
|
14
|
+
* §4 payload schema for the preference lane). v1 schema only — statement,
|
|
15
|
+
* kind — frozen prompt + model. `kind` is an UNVALIDATED RENDER-ONLY enum
|
|
16
|
+
* (N12 KILL: taxonomy reliability is extractor-limited — never gate behavior
|
|
17
|
+
* on it); it is NOT part of the dedupe key (norm(statement) is).
|
|
18
|
+
*
|
|
19
|
+
* Standing-vs-one-off is NOT prompt-only: the extractor prompt instructs the
|
|
20
|
+
* model to reject one-off events, and a code-level structural guard
|
|
21
|
+
* (`looksLikeOneOff`) drops extracted rows whose statement carries past-tense
|
|
22
|
+
* one-off markers ("yesterday", "last <day>", "-ed <unit> ago", …). The
|
|
23
|
+
* synthetic fixture enforces this both ways: "I always run Tuesdays" survives;
|
|
24
|
+
* "I ran 5km yesterday" is rejected even when the model (stub) emits it.
|
|
25
|
+
*
|
|
26
|
+
* Ledger KILLs bound this module: N12/N14/N16b/N20/N25 (no prompt iteration
|
|
27
|
+
* beyond v1), H5 (one-pass multi-head KILL — this extracts preferences only),
|
|
28
|
+
* H6 (yield-prefilter KILL, skip ceiling 43.0% — the hash-keyed skip is the
|
|
29
|
+
* only cost control), N15 (cap-80 KILL — selection is retrieval-scoped in the
|
|
30
|
+
* composer, never a bigger cap). Parse failures are EXPLICIT
|
|
31
|
+
* extractor-parse-errors, retried ≤3 times, never silently `[]`, and never
|
|
32
|
+
* cached (pipeline resets the note hash so the next pass retries).
|
|
33
|
+
*/
|
|
34
|
+
exports.PREFS_MODEL = 'gemma4:31b';
|
|
35
|
+
/** extraction_version = schema:model (ADR §4): hash-keyed cache key component. */
|
|
36
|
+
exports.PREFS_EXTRACTION_VERSION = `prefs-v1:${exports.PREFS_MODEL}`;
|
|
37
|
+
/** RENOR_PREFS (default OFF). Opt in with 1/true/yes/on. No default flip. */
|
|
38
|
+
function prefsEnabled(env = process.env) {
|
|
39
|
+
const raw = (env.RENOR_PREFS ?? '').trim().toLowerCase();
|
|
40
|
+
return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on';
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Structural guard for the standing-vs-one-off fixture (must not rely on the
|
|
44
|
+
* prompt alone): a statement that describes a completed past-tense one-off
|
|
45
|
+
* action is NOT a standing preference, even if the extractor emitted it.
|
|
46
|
+
* Dataset-free — anchored on explicit temporal one-off markers only, never on
|
|
47
|
+
* topic words (a standing rule about running Tuesdays must survive).
|
|
48
|
+
*/
|
|
49
|
+
function looksLikeOneOff(statement) {
|
|
50
|
+
const s = statement.toLowerCase();
|
|
51
|
+
const markers = [
|
|
52
|
+
/\byesterday\b/,
|
|
53
|
+
/\blast (night|week|month|year|monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b/,
|
|
54
|
+
/\bthis (morning|afternoon|evening)\b/,
|
|
55
|
+
/\b(earlier|ago)\b/,
|
|
56
|
+
/\bjust (ran|did|finished|completed|bought|met|visited|attended|started|watched)\b/,
|
|
57
|
+
/\b(i|we|he|she|they) (ran|walked|went|did|finished|completed|bought|met|visited|attended|watched|ate|drank|had) (yesterday|this morning|last )/,
|
|
58
|
+
/\b(had|have) (just|already) /,
|
|
59
|
+
];
|
|
60
|
+
return markers.some((re) => re.test(s));
|
|
61
|
+
}
|
|
62
|
+
const SYSTEM_PROMPT = `Extract STANDING PREFERENCES, CONSTRAINTS and durable PROFILE FACTS the USER asserts about themselves. Return STRICT JSON only — no YAML, no markdown code fences, no commentary.
|
|
63
|
+
|
|
64
|
+
Shape: {"prefs": [{"statement": "<first-person-compatible standing rule>", "kind": "like" | "dislike" | "policy" | "attribute"}]}
|
|
65
|
+
|
|
66
|
+
Rules:
|
|
67
|
+
- statement: durable rule or attribute as the user would say it (likes/dislikes, aversions, dietary restrictions, allergies, style/communication/tool preferences, standing policies like "I never...", "I always...", "I prefer... over...", durable personal attributes like job, family, pets, home, skills)
|
|
68
|
+
- kind: one of like, dislike, policy, attribute (best guess; never leave empty)
|
|
69
|
+
- STANDING only: what is persistently true or desired — NOT one-off events ("I ran 5km yesterday" is an event, "I always run Tuesdays" is a preference), NOT questions, NOT assistant suggestions, NOT hypotheticals
|
|
70
|
+
- Only preferences the USER asserted — first-person-compatible
|
|
71
|
+
- Each distinct preference ONCE. Extract 0-15 preferences.
|
|
72
|
+
- Do NOT extract from frontmatter metadata.`;
|
|
73
|
+
async function extractPreferences(client, noteTitle, noteBody) {
|
|
74
|
+
// Truncate body to keep the prompt bounded (same budget as the other lanes).
|
|
75
|
+
const body = noteBody.substring(0, 3000);
|
|
76
|
+
const response = await client.chat.completions.create({
|
|
77
|
+
model: exports.PREFS_MODEL,
|
|
78
|
+
temperature: 0,
|
|
79
|
+
response_format: { type: 'json_object' },
|
|
80
|
+
messages: [
|
|
81
|
+
{ role: 'system', content: SYSTEM_PROMPT },
|
|
82
|
+
{ role: 'user', content: `Note: "${noteTitle}"\n\n${body}` },
|
|
83
|
+
],
|
|
84
|
+
});
|
|
85
|
+
const content = response.choices?.[0]?.message?.content;
|
|
86
|
+
if (!content || !content.trim()) {
|
|
87
|
+
throw new typedFacts_1.ExtractorParseError('extractor-parse-error: preference extraction returned empty content');
|
|
88
|
+
}
|
|
89
|
+
// JSON-strict after fence-strip: a fenced or bare response parses; YAML,
|
|
90
|
+
// truncated JSON, or commentary still fails hard as an explicit parse error.
|
|
91
|
+
let parsed;
|
|
92
|
+
try {
|
|
93
|
+
parsed = JSON.parse((0, typedFacts_1.repairMalformedUnicodeEscapes)((0, typedFacts_1.stripExtractorCodeFence)(content)));
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
throw new typedFacts_1.ExtractorParseError(`extractor-parse-error: preference extraction returned non-JSON response (head: ${content.slice(0, 80)})`);
|
|
97
|
+
}
|
|
98
|
+
if (Array.isArray(parsed)) {
|
|
99
|
+
// Schema-drift tolerance: some gemma4 responses emit the bare prefs array
|
|
100
|
+
// without the {"prefs": ...} wrapper. Valid JSON either way; strict on shape.
|
|
101
|
+
parsed.prefs = parsed;
|
|
102
|
+
}
|
|
103
|
+
else if (parsed && typeof parsed === 'object' && !Array.isArray(parsed.prefs)) {
|
|
104
|
+
// Wrapper-key drift: gemma4 sometimes renames the wrapper key (e.g.
|
|
105
|
+
// "merchants") or emits a single object. Rows without a non-empty
|
|
106
|
+
// `statement` are dropped below, so mis-shaped rows can never become false
|
|
107
|
+
// preferences — empty is the worst case.
|
|
108
|
+
const firstArray = Object.values(parsed).find((v) => Array.isArray(v) && v.length > 0 && typeof v[0] === 'object' && v[0] !== null);
|
|
109
|
+
if (firstArray)
|
|
110
|
+
parsed.prefs = firstArray;
|
|
111
|
+
else
|
|
112
|
+
parsed.prefs = [parsed];
|
|
113
|
+
}
|
|
114
|
+
if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.prefs)) {
|
|
115
|
+
throw new typedFacts_1.ExtractorParseError('extractor-parse-error: preference extraction response is missing a prefs array');
|
|
116
|
+
}
|
|
117
|
+
const prefs = [];
|
|
118
|
+
for (const raw of parsed.prefs) {
|
|
119
|
+
const statement = raw?.statement === undefined || raw?.statement === null ? '' : String(raw.statement).trim();
|
|
120
|
+
if (!statement)
|
|
121
|
+
continue; // malformed rows are dropped, not fatal
|
|
122
|
+
// Structural standing-vs-one-off guard: one-off past-tense events are
|
|
123
|
+
// rejected HERE, not just in the prompt (synthetic-fixture contract).
|
|
124
|
+
if (looksLikeOneOff(statement))
|
|
125
|
+
continue;
|
|
126
|
+
const pref = { statement };
|
|
127
|
+
const kind = raw?.kind;
|
|
128
|
+
if (kind !== undefined && kind !== null && String(kind).trim()) {
|
|
129
|
+
pref.kind = String(kind).trim(); // unvalidated render-only enum (N12)
|
|
130
|
+
}
|
|
131
|
+
prefs.push(pref);
|
|
132
|
+
}
|
|
133
|
+
return prefs;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Retry wrapper: parse failures are retried up to MAX_EXTRACT_ATTEMPTS total;
|
|
137
|
+
* the final failure is rethrown explicitly (never cached, never silent []).
|
|
138
|
+
* Non-parse errors (network, auth) propagate immediately.
|
|
139
|
+
*/
|
|
140
|
+
async function extractPreferencesWithRetry(extract, noteTitle, noteBody) {
|
|
141
|
+
let lastError;
|
|
142
|
+
for (let attempt = 1; attempt <= typedFacts_1.MAX_EXTRACT_ATTEMPTS; attempt++) {
|
|
143
|
+
try {
|
|
144
|
+
return await extract(noteTitle, noteBody);
|
|
145
|
+
}
|
|
146
|
+
catch (err) {
|
|
147
|
+
if (!(err instanceof typedFacts_1.ExtractorParseError))
|
|
148
|
+
throw err;
|
|
149
|
+
lastError = err;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
throw lastError;
|
|
153
|
+
}
|