renor-engine 0.0.0-stage → 1.0.0

package/dist/indexer/chunker.js

@@ -0,0 +1,319 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chunkNote = chunkNote;
+exports.chunkNoteCut = chunkNoteCut;
+exports.chunkBodyFrom = chunkBodyFrom;
+const crypto_1 = require("crypto");
+const tokenizer_1 = require("./tokenizer");
+const limits_1 = require("../memory/limits");
+/** Token count for chunking. Special-token syntax (`<|im_start|>`, `<|endoftext|>`, …)
+ * can legitimately appear inside note CONTENT (e.g. imported ShareGPT histories);
+ * gpt-tokenizer rejects such text as chat protocol by default and would crash
+ * indexing. We count them as plain text instead of failing closed — that is what
+ * encodeTokens does, which is also why the tokenizer is loaded on first use
+ * rather than at import (see ./tokenizer).
+ *
+ * Counts are remembered by text. A note that is appended to is chunked whole again each time,
+ * and all of it but the tail is text already counted; counting was most of that pass. The memory
+ * kept is capped. A text of up to COUNTED_TEXT_MAX characters is kept by its text; a longer one, up to
+ * COUNTED_LONG_MAX, by its SHA-256, so a daily note's entries (up to a statement's 4,000 characters each) are
+ * counted once, not on every save, and the cache holds a digest for each, not the entry. Longer texts are counted
+ * every time.
+ *
+ * The key kept is never a slice of the note. A chunk's text is a slice, and a slice of a long string keeps the
+ * whole string alive: kept as it was, each text remembered held the note it was cut from (3.4 MB for a day's
+ * note), the cap counted only the slice, and a daily note's engine grew by the note's size with every save. */
+const COUNTED_WEIGHT_LIMIT = 8_000_000;
+const COUNTED_TEXT_MAX = 2_000;
+const COUNTED_LONG_MAX = 100_000;
+/** What a text kept by its digest counts against COUNTED_WEIGHT_LIMIT: the digest and the map's entry. */
+const COUNTED_DIGEST_WEIGHT = 128;
+const counted = new Map();
+const countedLong = new Map();
+let countedWeight = 0;
+// A round trip through JSON builds a new string with the same code units, a lone surrogate included.
+const owned = (s) => JSON.parse(JSON.stringify(s));
+const tokens = (s) => {
+    if (s.length > COUNTED_LONG_MAX)
+        return (0, tokenizer_1.encodeTokens)(s);
+    const long = s.length > COUNTED_TEXT_MAX;
+    const memo = long ? countedLong : counted;
+    const key = long ? (0, crypto_1.createHash)('sha256').update(s, 'utf16le').digest('base64') : s;
+    const known = memo.get(key);
+    if (known !== undefined)
+        return known;
+    const n = (0, tokenizer_1.encodeTokens)(s);
+    const weight = long ? COUNTED_DIGEST_WEIGHT : s.length;
+    if (countedWeight + weight > COUNTED_WEIGHT_LIMIT) {
+        counted.clear();
+        countedLong.clear();
+        countedWeight = 0;
+    }
+    memo.set(long ? key : owned(s), n);
+    countedWeight += weight;
+    return n;
+};
+function chunkNote(note, options = {}) {
+    return chunkNoteCut(note, options).chunks;
+}
+/** chunkNote, and where its last section starts (see NoteCut). */
+function chunkNoteCut(note, options = {}) {
+    const lines = note.body.split('\n');
+    const entries = entryRanges(note, lines);
+    if (entries)
+        return { chunks: chunkEntries(note, lines, entries, options) };
+    const cut = splitByHeadings(lines, 0, []);
+    const { chunks, lastFirst } = chunkSections(note.path, cut.sections, options);
+    return chunks.length > 0
+        ? { chunks, last: cut.last, lastFirst }
+        : {
+            chunks: [
+                {
+                    notePath: note.path,
+                    headingPath: '',
+                    text: note.body.trim(),
+                    startLine: 0,
+                    endLine: lines.length,
+                    tokenCount: tokens(note.body),
+                },
+            ],
+        };
+}
+/**
+ * The chunks chunkNote cuts from body line `from` on, in a note whose body lines before `from` are unchanged and
+ * whose last section, before, started at `from` with the heading stack `stack` (see LastSection): `lines` are the
+ * body's lines from `from` to its end. Every number in the result is a line of the whole body.
+ */
+function chunkBodyFrom(notePath, lines, from, options = {}) {
+    const cut = splitByHeadings(lines, from.line, from.stack);
+    const { chunks, lastFirst } = chunkSections(notePath, cut.sections, options);
+    return { chunks, last: cut.last, lastFirst };
+}
+function chunkSections(notePath, sections, options) {
+    const max = options.maxTokens ?? 400;
+    const overlap = options.overlapTokens ?? 0;
+    const chunks = [];
+    let lastFirst = 0;
+    for (const sec of sections) {
+        lastFirst = chunks.length;
+        const text = sec.lines.join('\n').trim();
+        if (!text)
+            continue;
+        if (tokens(text) <= max) {
+            chunks.push({
+                notePath,
+                headingPath: sec.headingPath,
+                text,
+                startLine: sec.startLine,
+                endLine: sec.endLine,
+                tokenCount: tokens(text),
+            });
+        }
+        else {
+            chunks.push(...splitByParagraphs(text, sec.headingPath, notePath, sec.startLine, max, overlap));
+        }
+    }
+    return { chunks, lastFirst };
+}
+/**
+ * The sections of `lines`, numbered from line `offset` of the body, read with the heading stack `initial` in force
+ * before the first line; and where the last of them starts (see LastSection).
+ */
+function splitByHeadings(lines, offset, initial) {
+    const sections = [];
+    let stack = initial;
+    let before = initial; // the stack in force before the heading that started the section being read
+    let buf = [];
+    let start = 0;
+    for (let i = 0; i < lines.length; i++) {
+        const h1 = lines[i].match(/^#\s+(.+)$/);
+        const h2 = !h1 ? lines[i].match(/^##\s+(.+)$/) : null;
+        const h3 = !h1 && !h2 ? lines[i].match(/^###\s+(.+)$/) : null;
+        if (h1 || h2 || h3) {
+            if (buf.length)
+                sections.push({ headingPath: stack.join(' > '), lines: buf, startLine: offset + start, endLine: offset + i });
+            before = stack;
+            if (h1)
+                stack = [h1[1].trim()];
+            else if (h2)
+                stack = [stack[0] ?? '', h2[1].trim()].filter(Boolean);
+            else if (h3)
+                stack = [stack[0] ?? '', stack[1] ?? '', h3[1].trim()].filter(Boolean);
+            buf = [lines[i]];
+            start = i;
+        }
+        else {
+            buf.push(lines[i]);
+        }
+    }
+    if (buf.length)
+        sections.push({ headingPath: stack.join(' > '), lines: buf, startLine: offset + start, endLine: offset + lines.length });
+    return { sections, last: { line: offset + start, stack: before } };
+}
+function splitByParagraphs(text, headingPath, notePath, base, max, overlapTokens = 0) {
+    const paras = text.split(/\n\n+/).flatMap(p => (p.length > ONE_CHUNK_CHARS
+        ? splitOversized(p, max).map((piece, i) => ({ text: piece, sep: i === 0 ? '\n\n' : '' }))
+        : [{ text: p, sep: '\n\n' }]));
+    const joined = (buf) => buf.map((q, i) => (i === 0 ? q.text : q.sep + q.text)).join('');
+    const chunks = [];
+    let buf = [];
+    let bt = 0;
+    let offset = base;
+    for (const para of paras) {
+        const p = para.text;
+        const pt = tokens(p);
+        if (bt + pt > max && buf.length > 0) {
+            const ct = joined(buf);
+            const lc = ct.split('\n').length;
+            chunks.push({ notePath, headingPath, text: ct, startLine: offset, endLine: offset + lc, tokenCount: tokens(ct) });
+            // The rest of a cut paragraph goes on from where this chunk stops, on the same line or the next, not a line below.
+            offset += para.sep === '' ? lc - 1 : lc;
+            // Implement overlap: carry over last paragraph(s) that fit within overlapTokens
+            buf = [];
+            bt = 0;
+            if (overlapTokens > 0) {
+                let overlapBuf = [];
+                let overlapBt = 0;
+                // Start from the end of the previous buffer and work backwards to find paragraphs that fit
+                const prevParas = ct.split('\n\n');
+                for (let i = prevParas.length - 1; i >= 0; i--) {
+                    const prevPt = tokens(prevParas[i]);
+                    if (overlapBt + prevPt <= overlapTokens) {
+                        overlapBuf.unshift({ text: prevParas[i], sep: '\n\n' });
+                        overlapBt += prevPt;
+                    }
+                    else {
+                        break;
+                    }
+                }
+                buf = overlapBuf;
+                bt = overlapBt;
+                // Adjust offset backwards to account for overlap content
+                if (overlapBuf.length > 0) {
+                    const overlapLines = joined(overlapBuf).split('\n').length;
+                    offset -= overlapLines;
+                }
+            }
+        }
+        buf.push(para);
+        bt += pt;
+    }
+    if (buf.length) {
+        const ct = joined(buf);
+        chunks.push({ notePath, headingPath, text: ct, startLine: offset, endLine: offset + ct.split('\n').length, tokenCount: tokens(ct) });
+    }
+    return chunks;
+}
+/** A line of only spaces, tabs and carriage returns is layout: the memory map's rule for where an entry ends. */
+const BLANK_LINE_RE = /^[ \t\r]*$/;
+/** How an entry's first line begins. The body is trimmed, so a last entry of nothing but spaces has lost the space after its source. */
+const ENTRY_START_RE = /^- \[\d{2}:\d{2}:\d{2}\] \(/;
+/** Does this text hold anything besides headings? A heading with nothing under it is a title, not a memory. */
+const hasBody = (text) => text.split('\n').some(l => l.trim() !== '' && !/^#{1,6}\s/.test(l));
+/**
+ * A daily note's entries as [start, end) line ranges of its body, or null when the note is cut by headings and
+ * paragraphs instead (it is not a daily note, or has no entry). An entry runs from the line that starts it to the
+ * next entry, less the blank lines that lay out the file; everything in it is the statement's, whatever it looks like.
+ */
+function entryRanges(note, lines) {
+    const starts = note.entryStarts;
+    if (!starts || starts.length === 0)
+        return null;
+    // The starts were found on the stored text. If the body no longer agrees (a start that is not an entry's first
+    // line, or out of order), cut by headings rather than cut a statement in the wrong place.
+    let previous = -1;
+    for (const start of starts) {
+        if (!Number.isInteger(start) || start <= previous || start >= lines.length || !ENTRY_START_RE.test(lines[start]))
+            return null;
+        previous = start;
+    }
+    return starts.map((start, k) => {
+        let end = k + 1 < starts.length ? starts[k + 1] : lines.length;
+        while (end > start + 1 && BLANK_LINE_RE.test(lines[end - 1]))
+            end--;
+        return [start, end];
+    });
+}
+/**
+ * One chunk per entry of a daily note, so that recalling one memory returns that memory and not the day it was said
+ * on. An entry is cut whole whatever its words contain (headings, blank lines, `---`, a line that looks like another
+ * entry); one longer than a chunk is cut into paragraphs, but never together with another entry. The chunk is the
+ * entry as the memory map shows it, its time and source included, under the heading the note opens with
+ * (`2026-10-03`). What stands above the first entry is cut as any note is, minus a heading with nothing under it,
+ * since every entry already carries that heading in its path.
+ */
+function chunkEntries(note, lines, entries, options) {
+    const max = options.maxTokens ?? 400;
+    const overlap = options.overlapTokens ?? 0;
+    const chunks = [];
+    const above = lines.slice(0, entries[0][0]);
+    let headingPath = '';
+    if (above.some(l => l.trim() !== '')) {
+        const { sections } = splitByHeadings(above, 0, []);
+        headingPath = sections[sections.length - 1].headingPath;
+        for (const chunk of chunkNote({ ...note, body: above.join('\n'), entryStarts: undefined }, options)) {
+            if (hasBody(chunk.text))
+                chunks.push(chunk);
+        }
+    }
+    for (const [start, end] of entries) {
+        const text = lines.slice(start, end).join('\n');
+        const tokenCount = tokens(text);
+        if (tokenCount <= max) {
+            chunks.push({ notePath: note.path, headingPath, text, startLine: start, endLine: end, tokenCount });
+        }
+        else {
+            chunks.push(...splitByParagraphs(text, headingPath, note.path, start, max, overlap));
+        }
+    }
+    return chunks;
+}
+/**
+ * A paragraph this short stays one chunk however many tokens it holds, as every paragraph did. A statement the engine
+ * saved is up to MAX_STATEMENT_LENGTH characters, stored with each character that needs it as a six-character \uXXXX
+ * escape, and it is one daily entry, one chunk and recalled as exactly its words (tests/mvp/statement-ledger.test.ts).
+ * A longer paragraph came from a pasted log, an import or a hostile file, and is cut at the chunk limit.
+ */
+const ONE_CHUNK_CHARS = limits_1.MAX_STATEMENT_LENGTH * 8;
+const PIECE_CHARS = 256;
+/** The end of a piece that starts at `from` and wants to end at `to`: after the last space or newline in its back half, never inside a surrogate pair. */
+function pieceEnd(p, from, to) {
+    if (to >= p.length)
+        return p.length;
+    const floor = from + ((to - from) >> 1);
+    for (let cut = to; cut > floor; cut--) {
+        if (p[cut] === ' ' || p[cut] === '\n')
+            return cut + 1;
+    }
+    const last = p.charCodeAt(to - 1);
+    return last >= 0xd800 && last < 0xdc00 && to - from > 1 ? to - 1 : to;
+}
+/**
+ * A paragraph over `max` tokens (one long line, a pasted log, a minified blob), cut at whitespace into pieces of at most
+ * `max` tokens. Only short pieces are counted, never the whole, and a piece that is still over `max` (CJK, emoji) is halved,
+ * so the cost is what the paragraph holds. The pieces are chunked like paragraphs.
+ */
+function splitOversized(p, max) {
+    const out = [];
+    let buf = '';
+    let bt = 0;
+    for (let i = 0; i < p.length;) {
+        let j = pieceEnd(p, i, i + PIECE_CHARS);
+        let pt = tokens(p.slice(i, j));
+        while (pt > max && j - i > 2) {
+            j = pieceEnd(p, i, i + ((j - i) >> 1));
+            pt = tokens(p.slice(i, j));
+        }
+        if (bt + pt > max && buf) {
+            out.push(buf);
+            buf = '';
+            bt = 0;
+        }
+        buf += p.slice(i, j);
+        bt += pt;
+        i = j;
+    }
+    if (buf)
+        out.push(buf);
+    return out;
+}

package/dist/indexer/dailyEntries.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DAILY_CHUNK_LAYOUT = void 0;
+exports.noteContentHash = noteContentHash;
+exports.dailyEntryStarts = dailyEntryStarts;
+const hasher_1 = require("./hasher");
+const dailyNote_1 = require("../memory/dailyNote");
+/**
+ * How a daily note is cut into chunks, as a version: `entries-1` is one chunk per entry (see chunker.ts). A
+ * note's chunks are only as current as the layout they were cut in, so the layout is part of what the index
+ * remembers about a note (`noteContentHash`). Change how daily notes are cut and change this, and every daily note
+ * the index holds is cut again by the next ordinary index pass.
+ */
+exports.DAILY_CHUNK_LAYOUT = 'entries-1';
+/**
+ * What the index keeps to know a note is unchanged: the hash of its bytes and, for a daily note, of the layout its
+ * chunks are cut in. An index built before the layout existed holds a note under the hash of its bytes alone, so the
+ * next pass sees that note as changed, cuts it again and stores the new hash, once; nothing else about the note,
+ * its file or any other note moves.
+ */
+function noteContentHash(filePath, content) {
+    return (0, hasher_1.hashContent)((0, dailyNote_1.isDailyNotePath)(filePath) ? `${exports.DAILY_CHUNK_LAYOUT}\u0000${content}` : content);
+}
+/** How many line breaks (`\n` alone, as every reader of this format splits) lie before `end` in `text`. */
+function newlinesBefore(text, end) {
+    let n = 0;
+    for (let at = text.indexOf('\n'); at !== -1 && at < end; at = text.indexOf('\n', at + 1))
+        n++;
+    return n;
+}
+/**
+ * Where the entries of a daily note start, as indices into the lines of its parsed `body`; undefined when it has
+ * none (a file of free text under a `Daily/` folder, such as an imported journal, is cut like any other note).
+ *
+ * An entry starts at a line that begins at column 0 like `- [08:30:00] (claude) ...`. That is judged on the text as
+ * stored, not as read: a statement's later line that looks like an entry is kept behind a backslash in the file so
+ * it cannot forge one (dailyNote.ts), and once the words are restored it looks like an entry again. Cutting the
+ * restored text on that shape would split a statement in two; the stored text says which lines really start one.
+ * This is the same test the memory map uses, so a chunk is one of its entries.
+ *
+ * The restore changes lines in place, so line `i` of the stored text is line `i` of the decoded text; `rawBody` is
+ * what follows the frontmatter there and `body` is that, trimmed, which fixes the line the body begins on.
+ */
+function dailyEntryStarts(stored, decoded, rawBody, body) {
+    if (!decoded.endsWith(rawBody))
+        return undefined;
+    const bodyAt = decoded.length - rawBody.length + (rawBody.length - rawBody.trimStart().length);
+    const first = newlinesBefore(decoded, bodyAt);
+    const bodyLines = body.split('\n');
+    const storedLines = stored.split('\n');
+    // Fail closed: if the restore ever changes the number of lines, the stored text no longer says which line is
+    // which, so say nothing about entries and let the note be cut by headings as before. (The chunker checks each
+    // start against the body once more.)
+    if (storedLines.length !== newlinesBefore(decoded, decoded.length) + 1)
+        return undefined;
+    const starts = [];
+    for (let i = 0; i < bodyLines.length; i++) {
+        const line = storedLines[first + i] ?? '';
+        if (line.startsWith('- [') && dailyNote_1.ENTRY_LINE_RE.test(line))
+            starts.push(i);
+    }
+    return starts.length > 0 ? starts : undefined;
+}

package/dist/indexer/hasher.js

@@ -0,0 +1,10 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hashContent = hashContent;
+const crypto_1 = __importDefault(require("crypto"));
+function hashContent(content) {
+    return crypto_1.default.createHash('sha256').update(content, 'utf8').digest('hex');
+}

package/dist/indexer/laneRuns.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.laneZeroYieldMarker = laneZeroYieldMarker;
+exports.recordLaneRun = recordLaneRun;
+function laneZeroYieldMarker(db, sourcePath, rowKind, sourceHash, extractionVersion) {
+    // The run fact alone is not enough: a concurrent writer for the SAME note
+    // (the watcher indexing while a CLI index runs) can have inserted a row since
+    // that run finished, and a live row must always outrank a marker. The row
+    // half is therefore re-checked live, in this same statement, so the decision
+    // is one atomic read of both halves. Extra extraction in that race is the
+    // safe direction; serving an empty lane is not.
+    const row = db.prepare(`SELECT max_rows_written FROM lane_runs
+     WHERE source_path = ? AND row_kind = ? AND source_hash = ? AND extraction_version = ?
+       AND max_rows_written = 0
+       AND NOT EXISTS (
+         SELECT 1 FROM structured_rows r
+         WHERE r.source_path = lane_runs.source_path AND r.row_kind = lane_runs.row_kind
+           AND r.source_hash = lane_runs.source_hash AND r.extraction_version = lane_runs.extraction_version
+           AND r.valid_to IS NULL
+       )
+     LIMIT 1`).get(sourcePath, rowKind, sourceHash, extractionVersion);
+    if (!row)
+        return undefined;
+    return { laneZeroYield: true };
+}
+/**
+ * Record a completed lane run. Called after the lane's extractor and its
+ * adjudicated writes returned without throwing — one row per (note, lane,
+ * content-version), refreshed in place on a repeat run.
+ *
+ * One statement, deliberately NOT wrapped in a transaction: the only value
+ * written is `rowsWritten`, an argument of this call, so there is no
+ * read-modify-write window to protect. (The MAX in the upsert is evaluated by
+ * SQLite inside the statement.) Wrapping it in a default (deferred)
+ * better-sqlite3 transaction would add a WAL snapshot-upgrade window
+ * (SQLITE_BUSY_SNAPSHOT) against the very cross-process writer this marker is
+ * defensive about — a failure introduced in the race instead of closed.
+ */
+function recordLaneRun(db, sourcePath, rowKind, sourceHash, extractionVersion, rowsWritten) {
+    // Monotone on purpose: max(stored, this run). A zero can never erase a
+    // positive record for the same key, whichever run finishes last.
+    db.prepare(`INSERT INTO lane_runs
+       (source_path, row_kind, source_hash, extraction_version, max_rows_written, completed_at)
+     VALUES (?, ?, ?, ?, ?, ?)
+     ON CONFLICT(source_path, row_kind, source_hash, extraction_version) DO UPDATE SET
+       max_rows_written = max(lane_runs.max_rows_written, excluded.max_rows_written),
+       completed_at = excluded.completed_at`).run(sourcePath, rowKind, sourceHash, extractionVersion, rowsWritten, new Date().toISOString());
+}