renor-engine 0.0.0-stage → 1.0.0

package/dist/memory/writer.js

@@ -0,0 +1,175 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.withDailyFileLock = withDailyFileLock;
+exports.entryClock = entryClock;
+exports.appendDailyMemory = appendDailyMemory;
+exports.appendDailyEntry = appendDailyEntry;
+const path_1 = __importDefault(require("path"));
+const limits_1 = require("./limits");
+const invisibleText_1 = require("./invisibleText");
+const dailyNote_1 = require("./dailyNote");
+const sourceName_1 = require("./sourceName");
+const secretGuard_1 = require("../extraction/secretGuard");
+const toolRefusal_1 = require("../toolRefusal");
+const controls_1 = require("./controls");
+const confinedWrite_1 = require("./confinedWrite");
+/**
+ * A note entry is the user's own words, so it is kept whole: nothing is trimmed, stripped or cut. What
+ * cannot be kept is refused instead, which makes a save that succeeds a save that stored every
+ * character, and a refused save one that wrote nothing.
+ */
+function checkedNoteText(text) {
+    if (typeof text !== 'string')
+        throw (0, toolRefusal_1.refusal)('invalid_argument', 'the note must be a string');
+    if (text.length > limits_1.MAX_STATEMENT_LENGTH) {
+        throw (0, toolRefusal_1.refusal)('field_too_long', `the note exceeds the maximum length of ${limits_1.MAX_STATEMENT_LENGTH} (got ${text.length}); nothing was written`);
+    }
+    // An unpaired surrogate is not text: UTF-8 cannot hold it, so writing it would store U+FFFD instead.
+    if (/\p{Cs}/u.test(text)) {
+        throw (0, toolRefusal_1.refusal)('invalid_text', 'the note contains an unpaired UTF-16 surrogate, which cannot be stored as text; nothing was written');
+    }
+    // A control character is not text either, and the filesystem and SQLite both accept one, so a note
+    // holding NUL would be written, indexed and served as if it were ordinary words. Refused instead
+    // (memory/controls.ts), so a save that succeeds is a save that stored nothing but the words.
+    (0, controls_1.assertNoUnstorableControls)('the note', text);
+    if ((0, invisibleText_1.isBlankOrInvisible)(text))
+        throw (0, toolRefusal_1.refusal)('invalid_argument', 'the note is blank (only whitespace or invisible characters); nothing was written');
+    return text;
+}
+/**
+ * The forms of a source the secret guard judges: the one the header stores (headerSource: letters, digits and hyphens,
+ * the other characters written as hyphens) and the same name with every other character deleted, which is how the header
+ * stored it before. Spacing a key apart does not stop it being a key: the stored form joins the pieces with hyphens, and
+ * the deleted form joins them whole.
+ */
+function sourceFormsForGuard(source) {
+    if (typeof source !== 'string')
+        return [];
+    return [(0, sourceName_1.headerSource)(source), (0, sourceName_1.cleanSource)(source).replace(/[^\p{L}\p{N}-]/gu, '')];
+}
+function isValidCalendarDate(date) {
+    const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(date);
+    if (!match)
+        return false;
+    const year = Number(match[1]);
+    const month = Number(match[2]);
+    const day = Number(match[3]);
+    const parsed = new Date(Date.UTC(year, month - 1, day));
+    return parsed.getUTCFullYear() === year
+        && parsed.getUTCMonth() === month - 1
+        && parsed.getUTCDate() === day;
+}
+/**
+ * In-process mutual exclusion per daily file. Every writer of a daily file (append here, and the
+ * memory map's retire/restore, which rewrites one) runs inside this, so a rewrite can never drop
+ * an append that landed between its read and its write. One tenant runs one engine process.
+ */
+const dailyFileLocks = new Map();
+function withDailyFileLock(filePath, fn) {
+    const key = path_1.default.resolve(filePath);
+    const prev = dailyFileLocks.get(key) ?? Promise.resolve();
+    const run = prev.then(fn, fn);
+    const settled = run.then(() => undefined, () => undefined);
+    dailyFileLocks.set(key, settled);
+    void settled.then(() => {
+        if (dailyFileLocks.get(key) === settled)
+            dailyFileLocks.delete(key);
+    });
+    return run;
+}
+/** A fixed offset as a client may give it: `-03:00`, `-0300`, or `Z`. */
+const FIXED_OFFSET_RE = /^(?:Z|([+-])(\d{2}):?(\d{2}))$/;
+const invalidTimeZone = (timeZone) => new Error(`timezone ${JSON.stringify(timeZone.slice(0, 64))} is not a time zone: give an IANA name such as "America/Sao_Paulo" or a UTC offset such as "-03:00"; nothing was written`);
+/**
+ * The day and time to stamp an entry with, read once from `now`. In UTC unless the client names the user's time zone,
+ * an IANA name ("America/Sao_Paulo") or a fixed offset ("-03:00"): the entry is then filed under the user's own day at
+ * their own time, so a note said at 21:00 in São Paulo belongs to that evening, not to the next UTC day. A name that is
+ * neither is refused rather than read as UTC, so nothing is filed under a day the user did not mean.
+ */
+function entryClock(now, timeZone) {
+    if (timeZone === undefined) {
+        const iso = now.toISOString();
+        return { date: iso.slice(0, 10), time: iso.slice(11, 19) };
+    }
+    let minutes;
+    const fixed = FIXED_OFFSET_RE.exec(timeZone.trim());
+    if (fixed) {
+        minutes = fixed[1] ? (fixed[1] === '-' ? -1 : 1) * (Number(fixed[2]) * 60 + Number(fixed[3])) : 0;
+        if (Number(fixed[3] ?? 0) > 59 || Math.abs(minutes) > 14 * 60)
+            throw invalidTimeZone(timeZone);
+    }
+    else {
+        let named;
+        try {
+            named = new Intl.DateTimeFormat('en-US', { timeZone: timeZone.trim(), timeZoneName: 'longOffset' })
+                .formatToParts(now).find(p => p.type === 'timeZoneName')?.value;
+        }
+        catch {
+            throw invalidTimeZone(timeZone);
+        }
+        // `GMT-03:00`, or `GMT` alone at UTC itself.
+        const m = named === undefined ? null : /^GMT(?:([+-])(\d{2}):(\d{2}))?$/.exec(named);
+        if (!m)
+            throw invalidTimeZone(timeZone);
+        minutes = m[1] ? (m[1] === '-' ? -1 : 1) * (Number(m[2]) * 60 + Number(m[3])) : 0;
+    }
+    const iso = new Date(now.getTime() + minutes * 60_000).toISOString();
+    if (minutes === 0)
+        return { date: iso.slice(0, 10), time: iso.slice(11, 19) };
+    const abs = Math.abs(minutes);
+    const offset = `${minutes < 0 ? '-' : '+'}${String(Math.floor(abs / 60)).padStart(2, '0')}:${String(abs % 60).padStart(2, '0')}`;
+    return { date: iso.slice(0, 10), time: iso.slice(11, 19), offset };
+}
+/**
+ * Appends one entry to the daily note of `date`. `clock` stamps it (its time, and its offset when not UTC); without
+ * one it is stamped with the current time in UTC. Returns the file's path.
+ */
+async function appendDailyMemory(vaultPath, date, entry, memoryDir = 'Memory/Daily', clock) {
+    return (await appendDailyEntry(vaultPath, date, entry, memoryDir, clock)).filePath;
+}
+/**
+ * `appendDailyMemory`, saying what it wrote: the file, and the entry exactly as it went in (without the line break
+ * that separates it from the one above), which is what the memory map's id for it is made from.
+ */
+async function appendDailyEntry(vaultPath, date, entry, memoryDir = 'Memory/Daily', clock) {
+    if (!isValidCalendarDate(date)) {
+        throw new Error('date must be a valid YYYY-MM-DD calendar date');
+    }
+    // The header must read back as an entry, or the entry would be folded into the one above it.
+    if (clock && (!/^\d{2}:\d{2}:\d{2}$/.test(clock.time) || (clock.offset !== undefined && !/^[+-]\d{2}:\d{2}$/.test(clock.offset)))) {
+        throw new Error('entry time must be HH:MM:SS, with an offset of the form +HH:MM; nothing was written');
+    }
+    const text = checkedNoteText(entry.text);
+    // RENOR_SECRET_GUARD (default ON): refuse a note that looks like it carries a secret before anything is
+    // written, the directory included. The source is scanned as it will be stored, and joined: removing characters
+    // can join the pieces of a key that were spaced apart (sourceFormsForGuard).
+    if ((0, secretGuard_1.secretGuardEnabled)()) {
+        await (0, secretGuard_1.assertStorable)([
+            { field: 'note text', text },
+            ...sourceFormsForGuard(entry.source).map(form => ({ field: 'note source', text: form })),
+        ]);
+    }
+    const dir = path_1.default.join(vaultPath, memoryDir);
+    const filePath = path_1.default.join(dir, `${date}.md`);
+    const resolvedPath = path_1.default.resolve(filePath);
+    const resolvedBase = path_1.default.resolve(dir);
+    if (!resolvedPath.startsWith(resolvedBase)) {
+        throw new Error('Invalid date: path traversal detected');
+    }
+    // The source's name, in the alphabet a header keeps: letters, digits and hyphens only (sourceName.ts)
+    const sanitizedSource = (0, sourceName_1.headerSource)(entry.source);
+    const time = clock?.time ?? new Date().toISOString().substring(11, 19);
+    // The user's words go in as said, except that a later line which would start like an entry is kept behind a
+    // backslash so it cannot forge one, and the entry's header then says so (see dailyNote.ts). Both are
+    // reversible, and every reader undoes them. A file outside the daily-note layout is written as said.
+    const written = (0, dailyNote_1.formatEntry)(time, sanitizedSource, text, (0, dailyNote_1.isDailyNotePath)(filePath), clock?.offset);
+    const line = `\n${written}`;
+    // Appended in place, or created with its frontmatter, through confinedWrite: a link, a hard link or a FIFO planted in the
+    // vault is refused, never followed, and a failed save leaves the note as it was (see appendConfined).
+    const relFile = path_1.default.relative(path_1.default.resolve(vaultPath), resolvedPath);
+    await withDailyFileLock(filePath, () => (0, confinedWrite_1.appendConfined)(vaultPath, relFile, line, (0, dailyNote_1.engineDailyHeader)(date)));
+    return { filePath, entry: written };
+}

package/dist/memory/writerConflicts.js

@@ -0,0 +1,85 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UNNAMED_WRITER = void 0;
+exports.writerOf = writerOf;
+exports.displacementsAfter = displacementsAfter;
+exports.hasOpenDisplacementBy = hasOpenDisplacementBy;
+exports.writerConflictLine = writerConflictLine;
+exports.writerConflictLines = writerConflictLines;
+const statements_1 = require("../db/statements");
+/**
+ * Writer conflicts: when a writer replaces a value another writer saved, both values stay visible, marked, until the
+ * writer that lost the value saves the same subject and predicate again.
+ *
+ * The writer of a fact is the identity the gateway (or the OAuth client) authenticated for the connection that saved it
+ * (`agentOf` in mvp/server.ts). It is never the `source` the caller typed, which says nothing a caller cannot choose. A
+ * fact with no writer recorded (written before this table, or by a path that names none) was saved by its source, so its
+ * writer is its source, or `unnamed` when it has none.
+ *
+ * Each fact a write leaves current carries the displacements still open on it, in `fact_writers` keyed by its event:
+ *   - a save by the same writer keeps every open displacement (the other writer has not confirmed yet);
+ *   - a save by another writer clears the displacements that writer owns, whichever value it saves: that is its
+ *     confirmation. A save by a third writer clears nothing it does not own, so the other displacements stay open;
+ *   - it opens a displacement for the value it replaces (the value, its writer and its date), unless that value is the
+ *     one that displaced the writer's own displacement (it answered its counter-claim) or the two values are the same text.
+ * Replay derives the same rows from the ledger. A live index has no rows for facts saved before migration 26, so those
+ * have no displacements on it; a rebuild from the ledger derives rows for them, so a rebuilt index can show a line the
+ * live one does not. A fact with no row has no displacements.
+ */
+/** The writer a write has when it has no authenticated identity and no source. */
+exports.UNNAMED_WRITER = 'unnamed';
+/** The writer a fact event recorded, and the displacements open on it. */
+function writerOf(db, eventId, source) {
+    const row = eventId === null ? undefined : (0, statements_1.prepareOnce)(db, 'SELECT writer, displaced FROM fact_writers WHERE event_id = ?').get(eventId);
+    if (row)
+        return { writer: row.writer, displaced: JSON.parse(row.displaced) };
+    return { writer: source || exports.UNNAMED_WRITER, displaced: [] };
+}
+/** Two values agree when they are the same text once composed and trimmed. */
+const sameValue = (a, b) => a.normalize('NFC').trim() === b.normalize('NFC').trim();
+/**
+ * The displacements a save leaves open on the fact it creates: `write` is that save (its event, writer and value) and
+ * `closed` the fact it replaces, null when it replaces none.
+ */
+function displacementsAfter(db, closed, write) {
+    if (closed === null)
+        return [];
+    const before = writerOf(db, closed.eventId, closed.source);
+    if (before.writer === write.writer)
+        return before.displaced;
+    const rest = before.displaced.filter(d => d.writer !== write.writer);
+    const confirms = before.displaced.some(d => d.writer === write.writer && d.byEvent === closed.eventId);
+    if (confirms || sameValue(closed.object, write.object))
+        return rest;
+    return [...rest, { writer: before.writer, object: closed.object, validFrom: closed.validFrom, byEvent: write.eventId }];
+}
+/** Whether `writer` has a displacement open on the fact `eventId`, which its next save of the subject and predicate confirms. */
+function hasOpenDisplacementBy(db, eventId, writer) {
+    return writerOf(db, eventId, null).displaced.some(d => d.writer === writer);
+}
+/**
+ * One line a read shows beside a current value that replaced another writer's value: the values it displaced, each with
+ * its writer and date, and what clears them. Plain words, so an agent reads it as it reads the rest of the reply.
+ */
+function writerConflictLine(open) {
+    const values = open.map(d => `${JSON.stringify(d.object)} by ${d.writer}, ${d.validFrom.slice(0, 10)}`).join('; ');
+    return `CONFLICT: ${values}. Replaced by a different writer; it stays a conflict until that writer saves this subject and predicate again.`;
+}
+/**
+ * The conflict line of each current fact (by event id) that has displacements open, read in one query. A fact with none
+ * has no entry, so a reply about it is what it was before.
+ */
+function writerConflictLines(db, eventIds) {
+    const ids = [...new Set(eventIds.filter((id) => typeof id === 'string'))];
+    const lines = new Map();
+    if (ids.length === 0)
+        return lines;
+    const rows = (0, statements_1.prepareOnce)(db, 'SELECT event_id, displaced FROM fact_writers WHERE event_id IN (SELECT value FROM json_each(?))')
+        .all(JSON.stringify(ids));
+    for (const row of rows) {
+        const open = JSON.parse(row.displaced);
+        if (open.length > 0)
+            lines.set(row.event_id, writerConflictLine(open));
+    }
+    return lines;
+}