renor-engine 0.0.0-stage → 1.0.0

package/dist/importing/journalLog.js

@@ -0,0 +1,96 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JournalLog = void 0;
+exports.journalLogName = journalLogName;
+exports.readJournalLog = readJournalLog;
+exports.removeJournalLog = removeJournalLog;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const toolRefusal_1 = require("../toolRefusal");
+/** The log's file name for a batch, beside its manifest. */
+function journalLogName(batchId) {
+    return `${batchId}.jsonl`;
+}
+/** Opened on the first append, so a batch that writes no note creates no file. Each line is fsynced before the next. */
+class JournalLog {
+    file;
+    fd = null;
+    constructor(file) {
+        this.file = file;
+    }
+    append(entry) {
+        if (this.fd === null)
+            this.fd = openLog(this.file);
+        const bytes = Buffer.from(`${JSON.stringify(entry)}\n`, 'utf8');
+        for (let done = 0; done < bytes.length;)
+            done += fs_1.default.writeSync(this.fd, bytes, done, bytes.length - done);
+        fs_1.default.fsyncSync(this.fd);
+    }
+    /** Idempotent. The bytes were fsynced when they were appended, so a failed close loses nothing. */
+    close() {
+        if (this.fd === null)
+            return;
+        const fd = this.fd;
+        this.fd = null;
+        try {
+            fs_1.default.closeSync(fd);
+        }
+        catch { /* every appended line is already on disk */ }
+    }
+}
+exports.JournalLog = JournalLog;
+function openLog(file) {
+    const isNew = !fs_1.default.existsSync(file);
+    // O_NOFOLLOW: a log reached through a symbolic link is refused, as a manifest is.
+    const flags = fs_1.default.constants.O_WRONLY | fs_1.default.constants.O_APPEND | fs_1.default.constants.O_CREAT | fs_1.default.constants.O_NOFOLLOW;
+    const fd = fs_1.default.openSync(file, flags, 0o600);
+    if (isNew) {
+        // The new name is durable before its first line is acknowledged.
+        const dirFd = fs_1.default.openSync(path_1.default.dirname(file), 'r');
+        try {
+            fs_1.default.fsyncSync(dirFd);
+        }
+        finally {
+            fs_1.default.closeSync(dirFd);
+        }
+    }
+    return fd;
+}
+/** The log's entries in the order they were appended, or undefined when the batch has no log. */
+function readJournalLog(file) {
+    const stat = fs_1.default.lstatSync(file, { throwIfNoEntry: false });
+    if (!stat)
+        return undefined;
+    if (!stat.isFile())
+        throw (0, toolRefusal_1.refusal)('unsafe_path', 'Unsafe import journal');
+    const lines = fs_1.default.readFileSync(file, 'utf8').split('\n');
+    // The text after the last newline: empty when the log ends whole, otherwise the cut line a crash left.
+    lines.pop();
+    return lines.map((line, i) => parseEntry(line, i + 1));
+}
+function parseEntry(line, lineNo) {
+    let value;
+    try {
+        value = JSON.parse(line);
+    }
+    catch {
+        value = undefined;
+    }
+    if (!value || (value.list !== 'created' && value.list !== 'reasserted') || typeof value.relPath !== 'string' || typeof value.hash !== 'string') {
+        throw new Error(`an import journal line (${lineNo}) cannot be read; undo_import stopped before changing anything`);
+    }
+    return { list: value.list, relPath: value.relPath, hash: value.hash };
+}
+/**
+ * Removes a log that no longer records an applying batch. Best effort: a finished batch's manifest holds the same notes,
+ * so a log that survives a crash is ignored by every reader of that batch.
+ */
+function removeJournalLog(file) {
+    try {
+        fs_1.default.unlinkSync(file);
+    }
+    catch { /* absent, or left for the next manifest rewrite to remove */ }
+}

package/dist/importing/markdown.js

@@ -0,0 +1,98 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readMarkdownSource = readMarkdownSource;
+exports.parseMarkdownImportNote = parseMarkdownImportNote;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const walker_1 = require("../indexer/walker");
+const parser_1 = require("../indexer/parser");
+const chunker_1 = require("../indexer/chunker");
+const source_1 = require("./source");
+const controls_1 = require("../memory/controls");
+const types_1 = require("./types");
+/** Plain vault input: use the shipped walker/parser/chunker, without a source index. */
+async function readMarkdownSource(folder, filter) {
+    const root = fs_1.default.realpathSync(folder);
+    if (!fs_1.default.statSync(root).isDirectory())
+        throw new Error('Markdown source must be a folder');
+    const nonUtf8Names = [];
+    const files = (await (0, walker_1.walkVault)(root, { onNonUtf8Markdown: name => nonUtf8Names.push(name) })).sort((a, b) => (0, source_1.toRelative)(root, a).localeCompare((0, source_1.toRelative)(root, b)));
+    const selected = files.filter(file => (0, source_1.isSelected)((0, source_1.toRelative)(root, file), filter));
+    const limited = filter.limit === undefined ? selected : selected.slice(0, filter.limit);
+    const notes = limited.map(file => {
+        const relPath = (0, source_1.toRelative)(root, file);
+        // Refuse a file replaced with a symlink after walking, and do not follow a
+        // substituted ancestor outside the chosen vault.
+        const resolved = fs_1.default.realpathSync(file);
+        const relative = path_1.default.relative(root, resolved);
+        if (relative === '..' || relative.startsWith(`..${path_1.default.sep}`) || path_1.default.isAbsolute(relative)) {
+            throw new Error(`Markdown source escapes the chosen folder: ${relPath}`);
+        }
+        const fd = fs_1.default.openSync(file, fs_1.default.constants.O_RDONLY | fs_1.default.constants.O_NOFOLLOW);
+        try {
+            const stat = fs_1.default.fstatSync(fd);
+            if (!stat.isFile())
+                throw new Error(`Markdown source is not a regular file: ${relPath}`);
+            const bytes = fs_1.default.readFileSync(fd);
+            try {
+                new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+            }
+            catch {
+                throw new Error(`Markdown source is not UTF-8: ${relPath}`);
+            }
+            const content = bytes.toString('utf8');
+            if (content.includes('\0'))
+                throw new Error(`Markdown source contains binary data: ${relPath}`);
+            return parseMarkdownImportNote(file, relPath, content, stat.mtime.toISOString());
+        }
+        finally {
+            fs_1.default.closeSync(fd);
+        }
+    });
+    return {
+        format: types_1.BUNDLE_FORMAT, sourceLabel: path_1.default.basename(root), notes, facts: [],
+        // A name the selection excludes anyway is not reported; every other one could not be imported.
+        nonUtf8Names: nonUtf8Names.filter(name => (0, source_1.isSelected)(name, filter)).sort(),
+        stats: {
+            notesAvailable: files.length, notesSelected: notes.length,
+            // Everything the selection left out, the --limit cut included, as the other adapters count it.
+            notesFilteredOut: files.length - limited.length, notesWithOriginalBytes: notes.length,
+            chunksSelected: notes.reduce((sum, note) => sum + note.chunks.length, 0),
+            factsAvailable: 0, factsSelected: 0,
+        },
+    };
+}
+/** Shared adapter for local files and bounded hosted Markdown uploads. */
+function parseMarkdownImportNote(sourcePath, relPath, content, modifiedAt) {
+    // A control character is refused as it is in a note or a statement (memory/controls.ts): the note is
+    // written into the vault and indexed, so what this funnel admits the workspace holds afterwards. One
+    // rule for a folder import and a hosted upload both.
+    (0, controls_1.assertNoUnstorableControls)(`imported note ${relPath}`, content);
+    // gray-matter ships a JavaScript engine that evaluates frontmatter. Remote
+    // Markdown is data: admit only default/YAML or JSON before calling the parser.
+    const opening = content.replace(/^\uFEFF/, '').match(/^---([^\r\n]*)/);
+    if (opening && !['', 'yaml', 'json'].includes(opening[1].trim())) {
+        throw new Error(`Unsupported Markdown frontmatter language: ${relPath}. Use YAML between --- lines, or remove the block.`);
+    }
+    let parsed;
+    try {
+        parsed = (0, parser_1.parseNote)(sourcePath, content);
+    }
+    catch {
+        // Do not expose private YAML text in an error.
+        throw new Error(`Cannot parse Markdown frontmatter: ${relPath}. Fix the YAML between the --- lines or remove that block, then send again.`);
+    }
+    return {
+        sourcePath, relPath, title: parsed.title,
+        kind: typeof parsed.frontmatter.kind === 'string' ? parsed.frontmatter.kind : null,
+        modifiedAt, frontmatter: parsed.frontmatter, originalContent: content,
+        // Spans are body-relative; the renderer locates chunks in original bytes.
+        chunks: (0, chunker_1.chunkNote)(parsed).map(chunk => ({
+            headingPath: chunk.headingPath, text: chunk.text, tokenCount: chunk.tokenCount,
+        })),
+        entities: [],
+    };
+}

package/dist/importing/plan.js

@@ -0,0 +1,206 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IMPORT_ROOT = void 0;
+exports.streamKey = streamKey;
+exports.assertImportNamespace = assertImportNamespace;
+exports.buildImportPlan = buildImportPlan;
+exports.contentHash = contentHash;
+const hasher_1 = require("../indexer/hasher");
+const queryHelpers_1 = require("../retrieval/queryHelpers");
+const orchestrator_1 = require("../retrieval/orchestrator");
+const facts_1 = require("./facts");
+const scope_1 = require("../mvp/scope");
+const serialize_1 = require("./serialize");
+const secretGuard_1 = require("../extraction/secretGuard");
+const secretScreen_1 = require("./secretScreen");
+const sourceName_1 = require("../memory/sourceName");
+/**
+ * The ledger source of a carried fact. A bundle file can name any source, and so can the `--event-source` flag, so without
+ * this an imported fact could read as the agent that saved it (its author is what get_context shows and marks); an import
+ * always says it is one.
+ */
+const importedLedgerSource = (carried) => ((0, sourceName_1.isImportSource)(carried) ? carried : `${sourceName_1.IMPORT_SOURCE_PREFIX}${carried}`);
+/**
+ * Normalized stream identity — the SAME rule the lifecycle uses to decide a stream.
+ *
+ * `identityKey` applies the lifecycle's Unicode subject fold (NFC + case, explicit
+ * aliases unified) so a stream the lifecycle considers one subject is skipped rather
+ * than carried — carrying it either failed the whole import (older value, replay
+ * monotonicity) or silently superseded the destination's owner-stated value (newer
+ * value). The predicate is kept **exact**, matching `currentFact`'s `predicate = ?`:
+ * lowercasing it here collapsed `Primary Goal` and `primary goal`, two streams the
+ * lifecycle holds separately, so a real stream was silently dropped. The scope is
+ * normalized exactly as the ledger/replay normalizes it, so raw variants cannot
+ * collide later, and keyed by its slug: `scope:x`, `project:x` and `x` are one
+ * stream, whose current value a write closes whatever spelling it was saved
+ * with (scopeSpellings). Nothing here rewrites the bytes that are emitted or hashed.
+ */
+function streamKey(subject, predicate, scope) {
+    const normalized = (0, facts_1.tryNormalizeScope)(scope);
+    const key = normalized ? (0, scope_1.scopeSpellings)(normalized)[0] : scope.trim().toLowerCase();
+    return `${(0, queryHelpers_1.identityKey)(subject)}\u0000${predicate}\u0000${key}`;
+}
+/** Every import writes under this folder: a read marks a memory there as imported (mvp/agentMemory.ts, IMPORTED_PATH). */
+exports.IMPORT_ROOT = 'Imported';
+/**
+ * Refuses a destination outside Imported/. The import mark is read off a note's path, so an import written elsewhere read
+ * as the workspace's own notes, and one written where the engine keeps its daily log (`Memory`, or the root) turned an
+ * imported file's entry lines into saves by whatever source the file named.
+ */
+function assertImportNamespace(namespace) {
+    if ((0, serialize_1.destinationRelPath)(namespace, 'x').split('/')[0] !== exports.IMPORT_ROOT) {
+        throw new Error(`the import destination must be under ${exports.IMPORT_ROOT}/ (got ${JSON.stringify(namespace)}): a read marks an imported memory by that folder`);
+    }
+}
+function buildImportPlan(bundle, options) {
+    assertImportNamespace(options.namespace);
+    const existingStreams = options.existingStreams ?? new Set();
+    const destinationNow = options.now ?? new Date().toISOString();
+    const notes = [];
+    const seenPaths = new Set();
+    let duplicateNotePath = 0;
+    let emptyNoteBody = 0;
+    let notesFromOriginalBytes = 0;
+    for (const note of bundle.notes) {
+        const relPath = (0, serialize_1.destinationRelPath)(options.namespace, note.relPath);
+        if (seenPaths.has(relPath)) {
+            duplicateNotePath += 1;
+            continue;
+        }
+        seenPaths.add(relPath);
+        const rendered = (0, serialize_1.renderNoteMarkdown)(note);
+        if (!rendered.content.trim()) {
+            emptyNoteBody += 1;
+            continue;
+        }
+        if (rendered.fromOriginalBytes)
+            notesFromOriginalBytes += 1;
+        notes.push({
+            relPath,
+            content: rendered.content,
+            contentHash: (0, hasher_1.hashContent)(rendered.content),
+            fromOriginalBytes: rendered.fromOriginalBytes,
+            chunkLines: rendered.chunkLines,
+            chunksNotLocated: rendered.chunksNotLocated,
+            chunksSpanInherited: rendered.chunksSpanInherited,
+            note,
+        });
+    }
+    const skipEvents = new Map();
+    const bump = (reason) => skipEvents.set(reason, (skipEvents.get(reason) ?? 0) + 1);
+    const byStream = new Map();
+    for (const fact of bundle.facts) {
+        const key = streamKey(fact.subject, fact.predicate, fact.scope);
+        const list = byStream.get(key);
+        if (list)
+            list.push(fact);
+        else
+            byStream.set(key, [fact]);
+    }
+    let multiValuedFactsDropped = 0;
+    const events = [];
+    const emittedStreams = new Set();
+    for (const [key, facts] of byStream) {
+        // The lifecycle layer holds at most ONE current value per (subject, predicate, scope).
+        // The engine deliberately allows note-derived facts to be multi-valued, so a stream
+        // with several distinct current values cannot become one current fact. Picking a
+        // winner would misrepresent which value is current — and row order is not a
+        // decision — so the whole stream is reported and dropped.
+        const distinctObjects = new Set(facts.map(f => f.object));
+        if (distinctObjects.size > 1) {
+            multiValuedFactsDropped += facts.length;
+            bump('multi-valued stream: the lifecycle holds one current value per subject+predicate');
+            continue;
+        }
+        if (existingStreams.has(key)) {
+            bump('destination already represents this subject+predicate');
+            continue;
+        }
+        if (emittedStreams.has(key)) {
+            bump('duplicate stream inside the bundle');
+            continue;
+        }
+        const fact = facts[0];
+        const validation = (0, facts_1.validateCarriedFact)({
+            subject: fact.subject,
+            predicate: fact.predicate,
+            object: fact.object,
+            scope: fact.scope,
+            validFrom: fact.validFrom,
+        });
+        if (!validation.ok || !validation.scope || !validation.at) {
+            bump(validation.reason ?? 'invalid fact');
+            continue;
+        }
+        // Bound the bundle's claimed time by the destination's clock BEFORE the id is derived, so the planned event, the
+        // stored valid_from and every read of it are bounded. Same rule and tolerance as every other content-derived clock.
+        const at = (0, orchestrator_1.clampSessionDate)(validation.at, destinationNow) ?? destinationNow;
+        // The event id is derived from every field replay compares, so a re-run against an
+        // unchanged source re-appends nothing and a changed source yields a new event
+        // instead of mutating one that replay already recorded.
+        const id = (0, facts_1.deterministicEventId)({
+            label: options.label,
+            subject: fact.subject,
+            predicate: fact.predicate,
+            scope: validation.scope,
+            object: fact.object,
+            at,
+        });
+        events.push({
+            id,
+            at,
+            kind: 'store',
+            subject: fact.subject,
+            predicate: fact.predicate,
+            object: fact.object,
+            scope: validation.scope,
+            source: importedLedgerSource(fact.ledgerSource ?? options.eventSource),
+        });
+        emittedStreams.add(key);
+    }
+    // RENOR_SECRET_GUARD (default ON): a note or fact that looks like a secret is left out of the plan, with the
+    // reason, so the dry run shows it and every other note is still imported. With the guard off none of this runs.
+    let guardSkips = null;
+    if ((0, secretGuard_1.secretGuardEnabled)()) {
+        const screened = (0, secretScreen_1.leaveOutFlagged)(notes, events);
+        // In place and without spreading: a bundle can hold more notes than a call can take as arguments.
+        notes.length = 0;
+        for (const kept of screened.notes)
+            notes.push(kept);
+        events.length = 0;
+        for (const kept of screened.events)
+            events.push(kept);
+        notesFromOriginalBytes = notes.filter(n => n.fromOriginalBytes).length;
+        guardSkips = screened.skips;
+    }
+    // Deterministic append order, so two runs produce an identical ledger.
+    events.sort((a, b) => streamKey(a.subject, a.predicate, a.scope).localeCompare(streamKey(b.subject, b.predicate, b.scope)));
+    const skippedFacts = [...skipEvents.entries()]
+        .map(([reason, count]) => ({ reason, count }))
+        .sort((a, b) => b.count - a.count);
+    return {
+        sourceLabel: bundle.sourceLabel,
+        namespace: options.namespace,
+        notes,
+        events,
+        skipped: { duplicateNotePath, emptyNoteBody, events: skippedFacts, ...(guardSkips ? { secretGuard: guardSkips } : {}) },
+        counts: {
+            notesInBundle: bundle.notes.length,
+            chunksInBundle: bundle.stats.chunksSelected,
+            factsInBundle: bundle.facts.length,
+            selectedNotes: notes.length,
+            selectedChunks: notes.reduce((sum, n) => sum + n.note.chunks.length, 0),
+            notesFromOriginalBytes,
+            notesReconstructed: notes.length - notesFromOriginalBytes,
+            chunksNotLocated: notes.reduce((sum, n) => sum + n.chunksNotLocated, 0),
+            chunksSpanInherited: notes.reduce((sum, n) => sum + n.chunksSpanInherited, 0),
+            eventsToAppend: events.length,
+            multiValuedFactsDropped,
+            factsSkipped: skippedFacts.reduce((sum, s) => sum + s.count, 0),
+        },
+    };
+}
+/** The engine's own note change-detection hash, re-exported for callers. */
+function contentHash(content) {
+    return (0, hasher_1.hashContent)(content);
+}

package/dist/importing/secretScreen.js

@@ -0,0 +1,113 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ImportRefusedError = void 0;
+exports.screenNote = screenNote;
+exports.screenEvent = screenEvent;
+exports.screenImportPlan = screenImportPlan;
+exports.refuseFlaggedPlan = refuseFlaggedPlan;
+exports.leaveOutFlagged = leaveOutFlagged;
+const secretGuard_1 = require("../extraction/secretGuard");
+/** Every string `applyImportPlan` would store for this note. */
+function noteTexts(planned) {
+    const note = planned.note;
+    const texts = [
+        { field: 'path', text: planned.relPath },
+        { field: 'content', text: planned.content },
+        { field: 'title', text: note.title },
+        { field: 'kind', text: note.kind ?? '' },
+        { field: 'frontmatter', text: JSON.stringify(note.frontmatter ?? {}) },
+    ];
+    for (const chunk of note.chunks) {
+        texts.push({ field: 'chunk', text: chunk.text }, { field: 'chunk heading', text: chunk.headingPath });
+    }
+    for (const entity of note.entities) {
+        texts.push({ field: 'entity', text: entity.canonicalName }, { field: 'entity', text: entity.type });
+        for (const alias of entity.aliases ?? [])
+            texts.push({ field: 'entity', text: alias });
+    }
+    return texts;
+}
+/** Every string `applyImportPlan` would append to the ledger for this fact. */
+function eventTexts(event) {
+    return [
+        { field: 'subject, predicate and object', text: `${event.subject} ${event.predicate}: ${event.object}` },
+        { field: 'source', text: event.source },
+    ];
+}
+function screenNote(planned) {
+    const hit = (0, secretGuard_1.findSensitive)(noteTexts(planned));
+    if (hit === null)
+        return null;
+    // The path was scanned first: if it is what was flagged, it must not be printed.
+    return { ref: hit.field === 'path' ? null : planned.relPath, field: hit.field, reason: hit.reason };
+}
+function screenEvent(event) {
+    const hit = (0, secretGuard_1.findSensitive)(eventTexts(event));
+    return hit === null ? null : { ref: event.id, field: hit.field, reason: hit.reason };
+}
+/** What a plan still holds that the guard would refuse. Pure: it changes nothing. */
+function screenImportPlan(plan) {
+    const notes = [];
+    for (const planned of plan.notes) {
+        const skip = screenNote(planned);
+        if (skip)
+            notes.push(skip);
+    }
+    const events = [];
+    for (const event of plan.events) {
+        const skip = screenEvent(event);
+        if (skip)
+            events.push(skip);
+    }
+    return { notes, events };
+}
+const describeSkip = (skip) => `${skip.ref ?? '(path withheld)'} (${skip.field}: ${skip.reason})`;
+/** The apply-time refusal. Counts and names only: the message carries no text from the plan. */
+class ImportRefusedError extends Error {
+    skips;
+    constructor(skips) {
+        const all = [...skips.notes, ...skips.events];
+        const shown = all.slice(0, 5).map(describeSkip).join('; ');
+        super(`import refused: ${skips.notes.length} note(s) and ${skips.events.length} fact(s) in this plan appear to contain a credential `
+            + 'or personal identifier, and the secret guard (RENOR_SECRET_GUARD) is on, so nothing was written. '
+            + 'Build the plan again (it leaves them out) or switch the guard off. '
+            + `First: ${shown}${all.length > 5 ? `; and ${all.length - 5} more` : ''}`);
+        this.name = 'ImportRefusedError';
+        this.skips = skips;
+    }
+}
+exports.ImportRefusedError = ImportRefusedError;
+/** Throws ImportRefusedError, and writes nothing, when the plan still holds anything the guard would refuse. */
+function refuseFlaggedPlan(plan) {
+    const skips = screenImportPlan(plan);
+    if (skips.notes.length === 0 && skips.events.length === 0)
+        return;
+    for (const skip of [...skips.notes, ...skips.events])
+        (0, secretGuard_1.logSkip)(skip.field, skip.reason);
+    throw new ImportRefusedError(skips);
+}
+/** The plan-time half: split `notes` and `events` into what stays and what is left out, in order. */
+function leaveOutFlagged(notes, events) {
+    const skips = { notes: [], events: [] };
+    const keptNotes = [];
+    for (const planned of notes) {
+        const skip = screenNote(planned);
+        if (skip) {
+            skips.notes.push(skip);
+            (0, secretGuard_1.logSkip)(skip.field, skip.reason);
+        }
+        else
+            keptNotes.push(planned);
+    }
+    const keptEvents = [];
+    for (const event of events) {
+        const skip = screenEvent(event);
+        if (skip) {
+            skips.events.push(skip);
+            (0, secretGuard_1.logSkip)(skip.field, skip.reason);
+        }
+        else
+            keptEvents.push(event);
+    }
+    return { notes: keptNotes, events: keptEvents, skips };
+}