renor-engine 0.0.0-stage → 1.0.0

package/dist/extraction/typedFacts.js

@@ -0,0 +1,224 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExtractorParseError = exports.MAX_EXTRACT_ATTEMPTS = exports.FACTS_EXTRACTION_VERSION = exports.FACTS_MODEL = void 0;
+exports.repairMalformedUnicodeEscapes = repairMalformedUnicodeEscapes;
+exports.stripExtractorCodeFence = stripExtractorCodeFence;
+exports.typedFactsEnabled = typedFactsEnabled;
+exports.contentDateFromFrontmatter = contentDateFromFrontmatter;
+exports.extractNumericFacts = extractNumericFacts;
+exports.extractNumericFactsWithRetry = extractNumericFactsWithRetry;
+/**
+ * C2 typed numeric-fact extraction (HYP-20260921-facts-product-join, ADR-0003
+ * §4 payload schema for the fact lane). v1 schema only — value, unit, verb,
+ * object, event — frozen prompt + model. Ledger KILLs bound this module:
+ * N12/N14/N16b/N20/N25 (extractor-prompt variants are a structural ceiling —
+ * no v2/v3/v4 prompt iteration) and H5 (one-pass multi-head — 48% row loss,
+ * so this extracts facts only). Parse failures are EXPLICIT
+ * extractor-parse-errors, retried ≤3 times, never silently `[]`, and never
+ * cached (pipeline resets the note hash so the next pass retries).
+ */
+exports.FACTS_MODEL = 'gemma4:31b';
+/** extraction_version = schema:model (ADR §4): hash-keyed cache key component. */
+exports.FACTS_EXTRACTION_VERSION = `facts-v1:${exports.FACTS_MODEL}`;
+/** ≤3 attempts total on parse failure; a final failure is thrown, never swallowed. */
+exports.MAX_EXTRACT_ATTEMPTS = 3;
+class ExtractorParseError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ExtractorParseError';
+    }
+}
+exports.ExtractorParseError = ExtractorParseError;
+/**
+ * Strip a leading/trailing markdown code fence from an LLM extraction response
+ * before JSON.parse. The frozen gemma4:31b extractor emits ```json fences on a
+ * fraction of responses despite strict-JSON prompt instructions (measured:
+ * ~10–25% of notes fail as raw JSON on real LongMemEval session corpora).
+ * JSON-strictness is preserved: everything that is not a fenced or bare JSON
+ * object still fails hard as an explicit ExtractorParseError.
+ */
+/**
+ * Repair a specific, measured malformation in gemma4:31b's extraction JSON: the
+ * model renders a mixed-number literal (e.g. "1 1/2" as a unicode fraction) as a
+ * TRUNCATED `\uXXXX` escape — `"1 \u00frac{1}{2}"`, i.e. `\u00f` followed by
+ * `rac`. `\u00f` is not a valid escape because `r` is not a hex digit, so
+ * JSON.parse throws "Bad Unicode escape".
+ *
+ * Measured on the failing LongMemEval note `09b651aa_3` (a long recipe session
+ * with several mixed-number amounts): 5 of 6 identical temperature-0 calls
+ * returned this malformation with finish_reason=stop and a well-formed tail —
+ * it is NOT truncation and NOT a transient error, and the existing 3-attempt
+ * retry therefore fails ~58% of the time on such a note. One unrecovered note
+ * aborts the whole product run (the index precondition is fail-closed), which is
+ * why structured-rows arms died mid-flight.
+ *
+ * The repair is deliberately minimal and lossless for every other input: it
+ * rewrites only a `\u` escape whose four hex digits are NOT all hex. The intent
+ * is unambiguous (the model tried to emit a hex escape and ran past it), so the
+ * malformed escape is replaced with the replacement character rather than being
+ * silently dropped or guessed. Valid escapes, including astral pairs, are
+ * untouched, so this cannot alter a well-formed response.
+ *
+ * It repairs EVERY malformed escape in the response, and that is the whole point
+ * of the global flag below. The measured note had SEVERAL mixed-number amounts
+ * in one response, and a single response can carry several of these escapes; a
+ * non-global replace left all but the first in place, so JSON.parse still threw
+ * "Bad Unicode escape" and the note still aborted the run — the repair appeared
+ * to work on the one-escape fixture in the test while the real input kept
+ * failing. Found by the independent review of PR #156.
+ *
+ * This is a TRANSPORT repair, not a parse-strictness relaxation: after it runs,
+ * a response that is not valid JSON still fails hard as ExtractorParseError.
+ */
+function repairMalformedUnicodeEscapes(content) {
+    // A malformed escape is a backslash that INTRODUCES an escape — i.e. is not
+    // itself escaped — followed by `u`, at most three hex digits, then a non-hex
+    // character. Two things this must leave alone, and the second one is why the
+    // run of backslashes is examined rather than just the last two characters:
+    //
+    //   - a VALID `\uXXXX` escape: at four hex digits the lookahead can never
+    //     match, so it is skipped;
+    //   - a LITERAL `u` after an ESCAPED backslash, e.g. the Windows path
+    //     `"C:\\user"`. `\\` is ONE escaped backslash and the `u` is content, so
+    //     the run of backslashes before that `u` is even; only an ODD run ends on
+    //     an escape introducer. Matching the final two characters regardless
+    //     rewrote the second backslash of the pair and corrupted real content:
+    //     `C:\user` became `C:\ufffdser`. Silently changing valid user text is a
+    //     worse failure than the malformation this repair exists to fix, and the
+    //     independent review of PR #156 caught it.
+    return content.replace(/(\\+)u([0-9a-fA-F]{0,3})(?![0-9a-fA-F])/g, (match, backslashes, hex) => {
+        // Only an ODD run of backslashes ends on an escape introducer; an even run
+        // is entirely escaped backslashes, so the `u` after it is ordinary content
+        // and the match is left exactly as it was.
+        if (backslashes.length % 2 === 0)
+            return match;
+        const introducer = backslashes.slice(0, -1);
+        if (hex.length === 0) {
+            // TWO SHAPES, TWO REPAIRS. With NO hex digit the model emitted a bare
+            // backslash before an ordinary character (a Windows path, a regex, a
+            // price like `\u`), and nothing was consumed as escape payload, so the
+            // text is recoverable exactly: escape the backslash. Substituting
+            // U+FFFD here DELETES the character that followed it, which is how
+            // `C:\user` became `C:\ufffdser`.
+            return `${introducer}\\\\u`;
+        }
+        // One to three hex digits: the model tried to emit a `\uXXXX` escape and
+        // ran past it (the measured gemma4:31b truncated-fraction malformation).
+        // Those characters really were escape payload, so the unambiguous repair
+        // is the replacement character.
+        return `${introducer}\\ufffd`;
+    });
+}
+function stripExtractorCodeFence(content) {
+    const s = content.trim();
+    const m = /^```[a-zA-Z]*\s*\n([\s\S]*?)\n?```$/.exec(s);
+    return m ? m[1].trim() : s;
+}
+/** RENOR_TYPED_FACTS (default OFF). Opt in with 1/true/yes/on. No default flip. */
+function typedFactsEnabled(env = process.env) {
+    const raw = (env.RENOR_TYPED_FACTS ?? '').trim().toLowerCase();
+    return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on';
+}
+/**
+ * Content date for valid_from (ADR §4): frontmatter `session_date`, else
+ * `date`, when parsable. NULL = unknown, never `now()`, never invented —
+ * an unparsable date string stays NULL (unknown must remain distinguishable).
+ * gray-matter parses bare YAML dates into JS Date objects; dates normalize to
+ * day precision (content dates are dates, not timestamps).
+ */
+function contentDateFromFrontmatter(frontmatter) {
+    for (const key of ['session_date', 'date']) {
+        const raw = frontmatter[key];
+        if (raw === undefined || raw === null)
+            continue;
+        if (raw instanceof Date) {
+            if (Number.isNaN(raw.getTime()))
+                continue;
+            return raw.toISOString().slice(0, 10);
+        }
+        const s = String(raw).trim();
+        if (!s)
+            continue;
+        const isoDay = s.match(/^(\d{4}-\d{2}-\d{2})/);
+        if (isoDay)
+            return isoDay[1];
+        const t = Date.parse(s);
+        if (!Number.isNaN(t))
+            return new Date(t).toISOString().slice(0, 10);
+    }
+    return null;
+}
+const SYSTEM_PROMPT = `Extract numeric facts from the note. Return STRICT JSON only — no YAML, no markdown code fences, no commentary.
+
+Shape: {"facts": [{"value": "<number exactly as written>", "unit": "<unit or omit>", "verb": "<short verb phrase or omit>", "object": "<what the number is about>", "event": "<optional event label or omit>"}]}
+
+Rules:
+- value: the numeric quantity as written (e.g. "3:45", "42", "R$ 1.200", "15%")
+- unit: measurement unit when stated (km, kg, hours, BRL, %, ...)
+- verb: short snake_case verb phrase (e.g. "ran", "weighs", "costs", "deadline")
+- object: the thing the number is about, in plain text
+- Only numeric facts clearly stated in the note body. Extract 0-15 facts.
+- Do NOT extract from frontmatter metadata.`;
+async function extractNumericFacts(client, noteTitle, noteBody) {
+    // Truncate body to keep the prompt bounded (same budget as the legacy extractor).
+    const body = noteBody.substring(0, 3000);
+    const response = await client.chat.completions.create({
+        model: exports.FACTS_MODEL,
+        temperature: 0,
+        response_format: { type: 'json_object' },
+        messages: [
+            { role: 'system', content: SYSTEM_PROMPT },
+            { role: 'user', content: `Note: "${noteTitle}"\n\n${body}` },
+        ],
+    });
+    const content = response.choices?.[0]?.message?.content;
+    if (!content || !content.trim()) {
+        throw new ExtractorParseError('extractor-parse-error: fact extraction returned empty content');
+    }
+    // JSON-strict after fence-strip: a fenced or bare response parses; YAML,
+    // truncated JSON, or commentary still fails hard as an explicit parse error.
+    let parsed;
+    try {
+        parsed = JSON.parse(repairMalformedUnicodeEscapes(stripExtractorCodeFence(content)));
+    }
+    catch {
+        throw new ExtractorParseError(`extractor-parse-error: fact extraction returned non-JSON response (head: ${content.slice(0, 80)})`);
+    }
+    if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.facts)) {
+        throw new ExtractorParseError('extractor-parse-error: fact extraction response is missing a facts array');
+    }
+    const facts = [];
+    for (const raw of parsed.facts) {
+        const value = raw?.value === undefined || raw?.value === null ? '' : String(raw.value).trim();
+        const object = raw?.object === undefined || raw?.object === null ? '' : String(raw.object).trim();
+        if (!value || !object)
+            continue; // malformed rows are dropped, not fatal
+        const fact = { value, object };
+        for (const key of ['unit', 'verb', 'event']) {
+            const v = raw[key];
+            if (v !== undefined && v !== null && String(v).trim())
+                fact[key] = String(v).trim();
+        }
+        facts.push(fact);
+    }
+    return facts;
+}
+/**
+ * Retry wrapper: parse failures are retried up to MAX_EXTRACT_ATTEMPTS total;
+ * the final failure is rethrown explicitly (never cached, never silent []).
+ * Non-parse errors (network, auth) propagate immediately.
+ */
+async function extractNumericFactsWithRetry(extract, noteTitle, noteBody) {
+    let lastError;
+    for (let attempt = 1; attempt <= exports.MAX_EXTRACT_ATTEMPTS; attempt++) {
+        try {
+            return await extract(noteTitle, noteBody);
+        }
+        catch (err) {
+            if (!(err instanceof ExtractorParseError))
+                throw err;
+            lastError = err;
+        }
+    }
+    throw lastError;
+}

package/dist/extraction/writePathAdjudication.js

@@ -0,0 +1,110 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chunkForCandidate = chunkForCandidate;
+exports.adjudicateAndWriteRows = adjudicateAndWriteRows;
+const rowStore_1 = require("../context/rowStore");
+const decision_model_1 = require("./decision-model");
+/**
+ * Attribute a candidate to the chunk whose text contains its key phrase
+ * (fact: value; event: event phrase; preference: statement). Plain,
+ * deterministic substring match — provenance-correct when it matches, and the
+ * caller falls back to the fleet's first-chunk attribution when it doesn't.
+ */
+function chunkForCandidate(chunks, payload) {
+    const phrase = 'statement' in payload
+        ? payload.statement
+        : 'when' in payload
+            ? payload.event
+            : 'value' in payload
+                ? payload.value
+                : undefined;
+    if (!phrase)
+        return null;
+    const needle = phrase.toLowerCase();
+    for (const chunk of chunks) {
+        if (chunk.text.toLowerCase().includes(needle)) {
+            return { chunkId: chunk.id, text: chunk.text };
+        }
+    }
+    return null;
+}
+/**
+ * Decide, then write — the harmonized single write path for one note's lane
+ * candidates. Rows are written AT MOST ONCE per candidate: identical facts
+ * bump the existing row's confirm_count (plus the cross-note coexist write),
+ * everything else goes through `insertStructuredRows` (identity-keyed
+ * supersession + migration v11's partial unique index closing the
+ * same-identity insert race).
+ */
+async function adjudicateAndWriteRows(db, args, model = (0, decision_model_1.defaultDecisionModel)()) {
+    const findCurrent = db.prepare(`SELECT id, source_path, payload_json FROM structured_rows
+     WHERE row_kind = ? AND scope = 'global' AND dedupe_key = ? AND valid_to IS NULL
+     ORDER BY id LIMIT 1`);
+    const bumpConfirm = db.prepare(`UPDATE structured_rows SET confirm_count = confirm_count + 1 WHERE row_kind = ? AND scope = 'global' AND dedupe_key = ? AND valid_to IS NULL`);
+    // Does THIS note already hold a current row for the identity? Guards the
+    // cross-note confirm write against double-inserting when several candidates
+    // of the same note adjudicate `identical` against the original note's row.
+    const findOwnCurrent = db.prepare(`SELECT id FROM structured_rows
+     WHERE row_kind = ? AND scope = 'global' AND dedupe_key = ? AND source_path = ? AND valid_to IS NULL
+     LIMIT 1`);
+    let rowsWritten = 0;
+    let confirmBumps = 0;
+    let decisions = 0;
+    let blocked = 0;
+    for (const candidate of args.candidates) {
+        // 1. Capture gate / sensitivity guardrail — BEFORE any write.
+        const gate = await model.decide({ task: 'capture_gate', text: candidate.sourceText });
+        decisions++;
+        if (gate.task !== 'capture_gate')
+            continue;
+        if (gate.decision === 'sensitive') {
+            blocked++;
+            continue;
+        }
+        const row = {
+            rowKind: args.rowKind,
+            sourcePath: args.notePath,
+            sourceChunkId: candidate.sourceChunkId,
+            sourceHash: args.sourceHash,
+            extractionVersion: args.extractionVersion,
+            validFrom: args.validFrom,
+            payload: candidate.payload,
+        };
+        const dedupeKey = (0, rowStore_1.dedupeKeyFor)(row);
+        const existing = findCurrent.get(args.rowKind, dedupeKey);
+        const adjudication = await model.decide({
+            task: 'dedupe_adjudication',
+            rowKind: args.rowKind,
+            candidate: candidate.payload,
+            existing: existing ? JSON.parse(existing.payload_json) : null,
+        });
+        decisions++;
+        if (adjudication.task !== 'dedupe_adjudication')
+            continue;
+        if (adjudication.decision === 'identical' && args.rowKind === 'fact') {
+            // Facts: an identical restatement corroborates — bump confirm_count,
+            // never double-write the same identity from the same adjudication.
+            bumpConfirm.run(args.rowKind, dedupeKey);
+            confirmBumps++;
+            if (!existing || existing.source_path === args.notePath)
+                continue;
+            // Cross-note `identical` must ALSO write this note's own source-scoped
+            // row (review FIX 1): the dedupe key has no source, so a bump-only
+            // write leaves the fact pinned solely to the original note's row, whose
+            // FK cascade dies with that note while this note still asserts it.
+            const own = findOwnCurrent.get(args.rowKind, dedupeKey, args.notePath);
+            if (own)
+                continue; // already captured from another chunk/run of this note
+            (0, rowStore_1.insertStructuredRows)(db, [{ ...row, sourceChunkId: candidate.sourceChunkId }], { skipSupersession: true });
+            rowsWritten++;
+            continue;
+        }
+        // 'updated'/'distinct' — and events/preferences restating the same
+        // identity — write through the rowStore: identity-keyed supersession
+        // (never blanket newest-wins; NULL dates never supersede) + the v11
+        // partial unique index closing the same-identity insert race.
+        (0, rowStore_1.insertStructuredRows)(db, [row]);
+        rowsWritten++;
+    }
+    return { rowsWritten, confirmBumps, decisions, blocked };
+}