renor-engine 0.0.0-stage → 1.0.0

package/dist/indexer/outsideRoot.js

@@ -0,0 +1,77 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeRowsOutsideRoot = removeRowsOutsideRoot;
+exports.reindexPending = reindexPending;
+exports.clearReindexPending = clearReindexPending;
+const path_1 = __importDefault(require("path"));
+const sqliteVec = require('sqlite-vec');
+/**
+ * Remove the index's rows for files outside the folder it is opened on.
+ *
+ * The index names each note by its absolute path. A workspace folder moved or restored from a backup into another
+ * directory therefore leaves every row of the old location behind, and a search serves each note twice: once from
+ * the new folder, and once from the old path, which no longer exists. This removes the rows whose path is absolute and
+ * not under `memoryDir`: the note, and what the note made (its chunks, entities, structured rows and lane runs go with
+ * it by cascade), plus the facts, relations and write events that name it, and the vectors of its chunks.
+ *
+ * Rows that name no file are kept: the engine's lifecycle facts (source `_mcp_`), the ledger's events and the agents'
+ * requests and cursors are not made from a note's path, so the folder's move does not touch them. Rows for notes under
+ * the folder are kept as they are, so an unmoved folder removes nothing and its index is not changed.
+ *
+ * The index is derived from the Markdown, so what is removed is indexed again by the next pass over the folder, at the
+ * paths it has now (`engineIndex`). The removal and the mark that the folder is owed a pass (`reindexPending`) are one
+ * transaction: a process that ends after it finds the mark on its next start, however long the pass would have taken.
+ * The canonical notes and the ledger are never written.
+ */
+function removeRowsOutsideRoot(db, memoryDir) {
+    const root = path_1.default.resolve(memoryDir);
+    const params = { root, under: root.endsWith(path_1.default.sep) ? root : root + path_1.default.sep };
+    // Absolute, and neither the folder itself nor below it. `length(@under)` counts characters as `substr` does.
+    const outside = (column) => `(substr(${column}, 1, 1) = '/' AND ${column} <> @root AND substr(${column}, 1, length(@under)) <> @under)`;
+    return db.transaction(() => {
+        const removed = { notes: 0, facts: 0, relations: 0, events: 0, vectors: 0 };
+        // Facts first: fact_terms is keyed by fact id and has no cascade, so its rows go with the facts that name the note.
+        const factIds = `SELECT id FROM facts WHERE ${outside('source_path')}`;
+        db.prepare(`DELETE FROM fact_terms WHERE fact_id IN (${factIds})`).run(params);
+        removed.facts = db.prepare(`DELETE FROM facts WHERE ${outside('source_path')}`).run(params).changes;
+        removed.relations = db.prepare(`DELETE FROM relations WHERE ${outside('source_note')}`).run(params).changes;
+        removed.events = db.prepare(`DELETE FROM write_events WHERE ${outside('note_path')}`).run(params).changes;
+        // Notes last: chunks, entities, structured rows and lane runs cascade from them, and chunks_fts and the note
+        // identity index follow their triggers.
+        removed.notes = db.prepare(`DELETE FROM notes WHERE ${outside('path')}`).run(params).changes;
+        // Vectors are keyed by chunk id with no cascade: those of the chunks just removed are orphans now. Only a removed
+        // note can leave one, so an index with nothing to remove is not read for them.
+        const vectorTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'chunk_embeddings'").get();
+        if (removed.notes > 0 && vectorTable !== undefined) {
+            // The table is a vec0 virtual table: this connection can only read it once the extension is loaded on it.
+            sqliteVec.load(db);
+            const orphans = db.prepare('SELECT rowid FROM chunk_embeddings WHERE rowid NOT IN (SELECT id FROM chunks)').pluck().all();
+            if (orphans.length > 0) {
+                db.prepare('DELETE FROM chunk_embeddings WHERE rowid IN (SELECT value FROM json_each(?))').run(JSON.stringify(orphans));
+            }
+            removed.vectors = orphans.length;
+        }
+        const total = removed.notes + removed.facts + removed.relations + removed.events + removed.vectors;
+        if (total > 0) {
+            db.prepare(`INSERT INTO index_state (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value`)
+                .run(REINDEX_PENDING, params.root);
+        }
+        return { ...removed, total };
+    })();
+}
+const REINDEX_PENDING = 'reindex_pending';
+/**
+ * True while the folder is owed an index pass: a removal made it, and no pass has since finished without a failure.
+ * A start reads this rather than the count of rows it removed, so a start that is cut short does not leave an index
+ * that nothing indexes again.
+ */
+function reindexPending(db) {
+    return db.prepare('SELECT 1 FROM index_state WHERE key = ?').get(REINDEX_PENDING) !== undefined;
+}
+/** Clears the mark once the folder has been indexed at its current paths with no failure. */
+function clearReindexPending(db) {
+    db.prepare('DELETE FROM index_state WHERE key = ?').run(REINDEX_PENDING);
+}

package/dist/indexer/parser.js

@@ -0,0 +1,167 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.assertFrontmatterBounded = assertFrontmatterBounded;
+exports.parseNote = parseNote;
+exports.parseNoteDetailed = parseNoteDetailed;
+exports.wikilinkTargets = wikilinkTargets;
+exports.wikilinksSettled = wikilinksSettled;
+const gray_matter_1 = __importDefault(require("gray-matter"));
+const path_1 = __importDefault(require("path"));
+const dailyNote_1 = require("../memory/dailyNote");
+const dailyEntries_1 = require("./dailyEntries");
+/**
+ * The first index at or after `p` where `pattern` matches (the text's length when none does), memoised: every p
+ * up to that index has the same answer, so a run of failing `[[` starts shares one scan. Without it each start
+ * scans to the end of the text and a pathological note (`[[[[...`) costs O(n^2).
+ */
+function nextMatch(text, pattern) {
+    let from = -1;
+    let at = -1;
+    return p => {
+        if (from <= p && p <= at)
+            return at;
+        pattern.lastIndex = p;
+        const m = pattern.exec(text);
+        from = p;
+        at = m === null ? text.length : m.index;
+        return at;
+    };
+}
+/**
+ * The wikilinks in `text` from `from`, matched as `[[target]]`, `[[target|alias]]` or `[[target#heading]]`: the
+ * target is the run up to the first `]`, `|` or `#` (it may span lines), closed by `]]`, or by a `|` or `#` and an
+ * alias that ends at the first `]` followed by another `]`. Matches do not overlap, and a `[[` that does not close
+ * is retried one character on. The same matches as the regex /\[\[([^\]|#]+)(?:[|#][^\]]*)?]]/g, in linear time.
+ */
+function wikilinkSpans(text, from) {
+    const spans = [];
+    const nextDelimiter = nextMatch(text, /[\]|#]/g);
+    const nextBracket = nextMatch(text, /\]/g);
+    let open = text.indexOf('[[', from);
+    while (open !== -1) {
+        const start = open + 2;
+        const delimiter = nextDelimiter(start);
+        let end = -1;
+        if (delimiter > start && delimiter < text.length) {
+            if (text[delimiter] === ']') {
+                if (text[delimiter + 1] === ']')
+                    end = delimiter + 2;
+            }
+            else {
+                const close = nextBracket(delimiter + 1);
+                if (text[close + 1] === ']')
+                    end = close + 2;
+            }
+        }
+        if (end === -1) {
+            open = text.indexOf('[[', open + 1);
+        }
+        else {
+            spans.push({ end, target: text.slice(start, delimiter) });
+            open = text.indexOf('[[', end);
+        }
+    }
+    return spans;
+}
+/**
+ * Frontmatter is stored as JSON. A YAML alias shares one parsed value, so a small note can serialize to
+ * far more bytes than it holds: a 2,500-character scalar aliased 2,000 times is a 15 KB note and 5 MB of
+ * JSON. Total the serialized size while walking the expanded tree and refuse past a bound proportional to
+ * the note, before anything is written.
+ */
+const FRONTMATTER_FLOOR_BYTES = 64 * 1024;
+const FRONTMATTER_EXPANSION = 4;
+const FRONTMATTER_DEPTH_LIMIT = 64;
+function assertFrontmatterBounded(value, sourceLength) {
+    const limit = FRONTMATTER_FLOOR_BYTES + FRONTMATTER_EXPANSION * sourceLength;
+    let size = 0;
+    const walk = (v, depth) => {
+        if (depth > FRONTMATTER_DEPTH_LIMIT)
+            throw new Error('Markdown frontmatter is too large or deeply aliased');
+        // Size each value the way JSON.stringify will write it, including toJSON (dates, binary).
+        if (v && typeof v.toJSON === 'function')
+            v = v.toJSON();
+        if (Array.isArray(v)) {
+            size += 2 + Math.max(v.length - 1, 0); // brackets and commas
+            for (const child of v)
+                walk(child, depth + 1);
+        }
+        else if (v && typeof v === 'object') {
+            const entries = Object.entries(v);
+            size += 2 + Math.max(entries.length - 1, 0);
+            for (const [key, child] of entries) {
+                size += JSON.stringify(key).length + 1;
+                walk(child, depth + 1);
+            }
+        }
+        else {
+            size += JSON.stringify(v)?.length ?? 0;
+        }
+        if (size > limit)
+            throw new Error('Markdown frontmatter is too large or deeply aliased');
+    };
+    walk(value, 0);
+}
+/**
+ * gray-matter caches every string it parses, and caches before it parses: a parse that throws leaves empty
+ * `data` in the cache and the next call for the same text returns it as if it had parsed. Any options object
+ * bypasses that cache. It also ships a JavaScript engine that evaluates `---js` frontmatter as code; a note
+ * is data, so only the YAML and JSON engines are usable.
+ */
+const MATTER_OPTIONS = {
+    engines: {
+        javascript: () => { throw new Error('Unsupported frontmatter language: javascript'); },
+    },
+};
+function parseNote(filePath, content) {
+    return parseNoteDetailed(filePath, content).note;
+}
+/**
+ * parseNote, with what the indexer needs to index an append to the note by its appended part (pipeline.ts): the
+ * text after the frontmatter the body was trimmed from (a suffix of `content`), and whether the title came from a
+ * heading in it rather than from the file name.
+ */
+function parseNoteDetailed(filePath, content) {
+    // The one door every reader of a note's text goes through (the index behind search, recall and get_context,
+    // the memory map's note view, the evidence pass). A daily note keeps a line of the user's words that would
+    // start like an entry behind a backslash; the words are restored here, so none of them sees the escape.
+    const text = (0, dailyNote_1.isDailyNotePath)(filePath) ? (0, dailyNote_1.decodeDailyNoteText)(content) : content;
+    const { data: frontmatter, content: rawBody } = (0, gray_matter_1.default)(text, MATTER_OPTIONS);
+    assertFrontmatterBounded(frontmatter, content.length);
+    const heading = rawBody.match(/^#\s+(.+)$/m);
+    const title = heading?.[1].trim() ?? path_1.default.basename(filePath, '.md');
+    const wikilinks = wikilinkTargets(rawBody);
+    const body = rawBody.trim();
+    const entryStarts = (0, dailyNote_1.isDailyNotePath)(filePath) ? (0, dailyEntries_1.dailyEntryStarts)(content, text, rawBody, body) : undefined;
+    return {
+        note: {
+            path: filePath,
+            title,
+            frontmatter,
+            body,
+            wikilinks,
+            ...(entryStarts ? { entryStarts } : {}),
+        },
+        rawBody,
+        titleFromHeading: heading !== null,
+    };
+}
+/** The wikilink targets in `text`, each once, in the order they first appear. */
+function wikilinkTargets(text, from = 0) {
+    return [...new Set(wikilinkSpans(text, from).map(s => s.target.trim()))].filter(Boolean);
+}
+/**
+ * Whether text appended to `text` can only add wikilinks after the ones it holds, scanning as `wikilinkTargets`
+ * does from `from`. A link's target may span lines, so an opening `[[` with no `]` after it could still be closed
+ * by what is appended, and become a link that starts in `text`; any other opening is settled by a `]` in `text`.
+ */
+function wikilinksSettled(text, from = 0) {
+    let settledTo = from;
+    for (const span of wikilinkSpans(text, from))
+        settledTo = span.end;
+    const open = text.lastIndexOf('[[');
+    return open < settledTo || text.indexOf(']', open + 2) !== -1;
+}