strom-research 1.0.0

package/dist/core/integrity.js

@@ -0,0 +1,219 @@
+// `strom verify` — was anything changed outside strom?
+//
+// Two levels, both built on the same facts:
+//
+// fast (before every write, in the commit gate, in orientation)
+//   1. HEAD carries a valid commit seal → everything committed is trusted,
+//      because strom only commits behind its own gate.
+//   2. Only what changed since HEAD is examined: new operations must continue
+//      the chain and be signed, every changed record must match the hash its
+//      operation recorded. Cost: two git calls when nothing changed.
+//
+// full (`strom verify`, `strom check`)
+//   Every operation of every log and every record file, plus the commit seal.
+import fs from "node:fs";
+import path from "node:path";
+import * as git from "./git.js";
+import { RECORD_TYPES } from "./model.js";
+import { fileSha, fingerprint, verifyCommitSeal, verifyOp } from "./seal.js";
+import { StromError } from "./errors.js";
+export function snapshot(tree) {
+    if (!git.hasHead(tree.root))
+        return { changes: [], head: new Map() };
+    const changes = git.changes(tree.root, ["data", "strom.json"]);
+    return { changes, head: git.showFiles(tree.root, "HEAD", changes.map((c) => c.path)) };
+}
+const RECORD_DIRS = new Set(Object.values(RECORD_TYPES).map((d) => `data/${d.dir}/`));
+function isRecordPath(p) {
+    const dir = p.slice(0, p.lastIndexOf("/") + 1);
+    return RECORD_DIRS.has(dir) && p.endsWith(".json");
+}
+function parseOps(text, file, findings, startLine = 0) {
+    const out = [];
+    text.split("\n").forEach((line, i) => {
+        if (!line.trim())
+            return;
+        try {
+            out.push(JSON.parse(line));
+        }
+        catch {
+            findings.push({ level: "error", code: "ops-corrupt", file, message: `line ${startLine + i + 1} is not valid JSON` });
+        }
+    });
+    return out;
+}
+/** Check chain and signatures of a run of operations. */
+function checkOps(tree, ops, prevSig, file, findings) {
+    const key = tree.key;
+    const fp = key ? fingerprint(key) : undefined;
+    let prev = prevSig;
+    let adopted = false;
+    for (const op of ops) {
+        if (op.prev !== prev)
+            findings.push({ level: "error", code: "chain-broken", file, message: `operation ${op.op} at ${op.at}: chain broken (log rewritten or edited)` });
+        prev = op.sig ?? "";
+        if (op.op === "seal.adopt")
+            adopted = true;
+        if (!key)
+            continue;
+        if (op.k === fp) {
+            if (!verifyOp(op, key))
+                findings.push({ level: "error", code: "forged-op", file, message: `operation ${op.op} at ${op.at} has an invalid signature (written outside strom)` });
+        }
+        else if (adopted) {
+            findings.push({ level: "error", code: "forged-op", file, message: `operation ${op.op} at ${op.at} is signed with an unknown key after the tree was adopted` });
+        }
+    }
+}
+function headSeal(tree, findings) {
+    const head = git.headInfo(tree.root);
+    if (!head)
+        return;
+    const key = tree.key;
+    if (!key) {
+        findings.push({
+            level: "warn",
+            code: "foreign-seal",
+            message: "this tree was sealed on another computer — reading works, writing needs the user's consent",
+            hint: "the user runs in their terminal: strom seal adopt",
+        });
+        return;
+    }
+    if (!verifyCommitSeal(key, head.tree, head.message))
+        findings.push({
+            level: "error",
+            code: "foreign-commit",
+            message: "the last commit was not made by strom (git used directly)",
+            hint: "strom repair   (return to the last sealed state)",
+        });
+}
+function checkRecordFile(tree, rel, expected, untracked, findings, tampered) {
+    const id = path.basename(rel, ".json");
+    const abs = path.join(tree.root, rel);
+    if (!fs.existsSync(abs)) {
+        tampered.add(rel);
+        findings.push({ level: "error", code: "manual-delete", id, file: rel, message: "record was deleted outside strom", hint: "strom repair" });
+        return;
+    }
+    if (expected === undefined) {
+        // No operation wrote this content: a new file is unsealed, a changed one was edited by hand.
+        tampered.add(rel);
+        findings.push(untracked
+            ? { level: "error", code: "unsealed", id, file: rel, message: "record was created outside strom", hint: "strom repair" }
+            : { level: "error", code: "manual-edit", id, file: rel, message: "record was edited outside strom", hint: "strom repair" });
+        return;
+    }
+    if (fileSha(abs) !== expected) {
+        tampered.add(rel);
+        findings.push({ level: "error", code: "manual-edit", id, file: rel, message: "record was edited outside strom", hint: "strom repair" });
+    }
+}
+/** Fast check: HEAD seal + everything changed since HEAD. */
+export function verifyFast(tree, snap = snapshot(tree)) {
+    const findings = [];
+    const tampered = new Set();
+    if (!git.hasHead(tree.root))
+        return { findings, tampered: [] };
+    headSeal(tree, findings);
+    const { changes, head: committed } = snap;
+    if (changes.length === 0)
+        return { findings, tampered: [] };
+    const opsChanged = changes.filter((c) => c.path.startsWith("data/ops/") && c.path.endsWith(".jsonl"));
+    const latest = new Map();
+    for (const c of opsChanged) {
+        const before = committed.get(c.path) ?? "";
+        const abs = path.join(tree.root, c.path);
+        if (!fs.existsSync(abs)) {
+            findings.push({ level: "error", code: "ops-deleted", file: c.path, message: "operation log was deleted" });
+            continue;
+        }
+        const now = fs.readFileSync(abs, "utf8");
+        if (!now.startsWith(before)) {
+            findings.push({ level: "error", code: "ops-rewritten", file: c.path, message: "operation log was rewritten (it is append-only)" });
+            continue;
+        }
+        const prior = parseOps(before, c.path, findings);
+        const added = parseOps(now.slice(before.length), c.path, findings, prior.length);
+        checkOps(tree, added, prior.at(-1)?.sig ?? "", c.path, findings);
+        for (const op of added)
+            for (const f of op.files ?? [])
+                latest.set(f.path, f.sha);
+    }
+    for (const c of changes) {
+        if (c.path.startsWith("data/ops/") || c.path === "data/_counters.json")
+            continue;
+        if (c.path === "strom.json") {
+            // strom changes its config through a logged operation (strom config set --for-tree).
+            if (latest.get(c.path) !== undefined && latest.get(c.path) === fileSha(path.join(tree.root, c.path)))
+                continue;
+            findings.push({ level: "error", code: "config-edited", file: c.path, message: "strom.json was changed outside strom", hint: "strom repair" });
+            tampered.add(c.path);
+            continue;
+        }
+        if (!isRecordPath(c.path)) {
+            findings.push({ level: "error", code: "unsealed", file: c.path, message: "unknown file in data/ (created outside strom)", hint: "strom repair" });
+            tampered.add(c.path);
+            continue;
+        }
+        checkRecordFile(tree, c.path, latest.get(c.path), c.code === "??", findings, tampered);
+    }
+    return { findings, tampered: [...tampered] };
+}
+/** Full check: every operation and every record. */
+export function verifyFull(tree) {
+    const findings = [];
+    const tampered = new Set();
+    headSeal(tree, findings);
+    const latest = new Map();
+    const opsDir = path.join(tree.dataDir, "ops");
+    const all = [];
+    if (fs.existsSync(opsDir))
+        for (const f of fs.readdirSync(opsDir).sort()) {
+            if (!f.endsWith(".jsonl"))
+                continue;
+            const rel = `data/ops/${f}`;
+            const ops = parseOps(fs.readFileSync(path.join(opsDir, f), "utf8"), rel, findings);
+            checkOps(tree, ops, "", rel, findings);
+            all.push(...ops);
+        }
+    // Logs are per session: the last word on a file is the newest operation across all of them.
+    all.sort((a, b) => (a.at < b.at ? -1 : a.at > b.at ? 1 : 0));
+    for (const op of all)
+        for (const sf of op.files ?? [])
+            latest.set(sf.path, sf.sha);
+    const present = new Set();
+    for (const def of Object.values(RECORD_TYPES)) {
+        const dir = path.join(tree.dataDir, def.dir);
+        if (!fs.existsSync(dir))
+            continue;
+        for (const f of fs.readdirSync(dir)) {
+            if (!f.endsWith(".json"))
+                continue;
+            const rel = `data/${def.dir}/${f}`;
+            present.add(rel);
+            checkRecordFile(tree, rel, latest.get(rel), true, findings, tampered);
+        }
+    }
+    for (const rel of latest.keys())
+        if (!present.has(rel) && rel !== "strom.json")
+            checkRecordFile(tree, rel, latest.get(rel), false, findings, tampered);
+    const cfg = latest.get("strom.json");
+    if (cfg && fileSha(path.join(tree.root, "strom.json")) !== cfg) {
+        tampered.add("strom.json");
+        findings.push({ level: "error", code: "config-edited", file: "strom.json", message: "strom.json was changed outside strom", hint: "strom repair" });
+    }
+    return { findings, tampered: [...tampered] };
+}
+/** Refuse to write on top of data that was changed outside strom. */
+export function assertIntact(tree) {
+    const { findings } = verifyFast(tree);
+    const errors = findings.filter((f) => f.level === "error");
+    if (errors.length === 0) {
+        tree.requireKey();
+        return;
+    }
+    throw new StromError(`data was changed outside strom (${errors.length} problem(s)) — refusing to write`, {
+        hint: "strom verify   (details)\nstrom repair   (restore the last sealed state)",
+        details: errors,
+    });
+}

package/dist/core/json.js

@@ -0,0 +1,87 @@
+// Canonical JSON and atomic file writes.
+//
+// Every record is written in one canonical form (fixed key order, 2-space
+// indent, trailing newline) so that git diffs only ever show real changes.
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+/** Keys that come first, in this order; everything else is alphabetical. */
+const LEADING_KEYS = ["id", "type", "name", "names", "title", "status", "kind"];
+function keyRank(key) {
+    const i = LEADING_KEYS.indexOf(key);
+    return i === -1 ? LEADING_KEYS.length : i;
+}
+function compareKeys(a, b) {
+    const ra = keyRank(a);
+    const rb = keyRank(b);
+    if (ra !== rb)
+        return ra - rb;
+    return a < b ? -1 : a > b ? 1 : 0;
+}
+/** Deep copy with canonical key order; drops `undefined` values. */
+export function canonicalize(value) {
+    if (Array.isArray(value))
+        return value.map(canonicalize);
+    if (value !== null && typeof value === "object") {
+        const src = value;
+        const out = {};
+        for (const key of Object.keys(src).sort(compareKeys)) {
+            const v = src[key];
+            if (v === undefined)
+                continue;
+            out[key] = canonicalize(v);
+        }
+        return out;
+    }
+    return value;
+}
+export function stringifyCanonical(value) {
+    return JSON.stringify(canonicalize(value), null, 2) + "\n";
+}
+/** Write via a temp file in the same directory and rename: never a half file. */
+export function writeFileAtomic(file, content) {
+    const dir = path.dirname(file);
+    fs.mkdirSync(dir, { recursive: true });
+    const tmp = path.join(dir, `.${path.basename(file)}.${process.pid}.${crypto.randomBytes(4).toString("hex")}.tmp`);
+    fs.writeFileSync(tmp, content, "utf8");
+    try {
+        fs.renameSync(tmp, file);
+    }
+    catch (err) {
+        fs.rmSync(tmp, { force: true });
+        throw err;
+    }
+}
+export function writeJson(file, value) {
+    writeFileAtomic(file, stringifyCanonical(value));
+}
+export function readJson(file) {
+    const text = fs.readFileSync(file, "utf8");
+    try {
+        return JSON.parse(text);
+    }
+    catch (err) {
+        throw new Error(`invalid JSON in ${file}: ${err.message}`);
+    }
+}
+export function readJsonIfExists(file) {
+    if (!fs.existsSync(file))
+        return undefined;
+    return readJson(file);
+}
+/** Append one JSON line (append-only logs). */
+export function appendJsonLine(file, value) {
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.appendFileSync(file, JSON.stringify(canonicalize(value)) + "\n", "utf8");
+}
+export function readJsonLines(file) {
+    if (!fs.existsSync(file))
+        return [];
+    const out = [];
+    for (const line of fs.readFileSync(file, "utf8").split("\n")) {
+        if (line.trim() === "")
+            continue;
+        out.push(JSON.parse(line));
+    }
+    return out;
+}

package/dist/core/lang.js

@@ -0,0 +1,70 @@
+// Research language: the language the agent uses with the user and for
+// research texts (notes, tasks, stories). CLI output itself is English.
+/** BCP 47 primary tag, e.g. "cs", "en", "de". */
+export function normalizeLang(value) {
+    if (!value)
+        return undefined;
+    const v = value.trim();
+    if (v === "" || v === "C" || v === "POSIX")
+        return undefined;
+    // "cs_CZ.UTF-8" -> "cs", "de-AT" -> "de"
+    const m = /^([A-Za-z]{2,3})(?:[-_.@]|$)/.exec(v);
+    return m?.[1]?.toLowerCase();
+}
+/** Detect the user's language from the environment and the OS locale. */
+export function detectLang(env) {
+    for (const key of ["LC_ALL", "LC_MESSAGES", "LANG", "LANGUAGE"]) {
+        const lang = normalizeLang(env[key]?.split(":")[0]);
+        if (lang)
+            return lang;
+    }
+    try {
+        const lang = normalizeLang(Intl.DateTimeFormat().resolvedOptions().locale);
+        if (lang)
+            return lang;
+    }
+    catch {
+        // Intl unavailable: fall through
+    }
+    return "en";
+}
+const NAMES = {
+    cs: "Czech",
+    sk: "Slovak",
+    de: "German",
+    pl: "Polish",
+    en: "English",
+    hu: "Hungarian",
+    fr: "French",
+    it: "Italian",
+    es: "Spanish",
+    uk: "Ukrainian",
+    ru: "Russian",
+    sl: "Slovenian",
+    hr: "Croatian",
+    nl: "Dutch",
+    sv: "Swedish",
+    da: "Danish",
+    nb: "Norwegian",
+    fi: "Finnish",
+    pt: "Portuguese",
+    lt: "Lithuanian",
+    lv: "Latvian",
+};
+/** The name of a language — in English, or in another language for a person who reads it ("čeština"). */
+export function langName(code, inLang) {
+    if (inLang && inLang !== "en") {
+        try {
+            const name = new Intl.DisplayNames([inLang], { type: "language", fallback: "none" }).of(code);
+            if (name)
+                return name;
+        }
+        catch {
+            // an unknown code: the English name, or the code
+        }
+    }
+    return NAMES[code] ?? code;
+}
+export function isValidLang(value) {
+    return /^[a-z]{2,3}$/.test(value);
+}

package/dist/core/live.js

@@ -0,0 +1,244 @@
+// The live bridge: the Strom app follows a research while it goes on. strom
+// runs a small web server on this computer only (127.0.0.1, a random port, a
+// secret token in every address) that only reads: the app, opened with
+// ?live=<address>, takes the tree's GEDCOM from it and hears what changes —
+// who is at work on what, what was recorded, what waits for the user. Nothing
+// leaves the computer. The bridge ends by itself when nobody has asked it
+// anything for a while (LIVE_IDLE_MS), or with strom live stop.
+//
+//   GET <token>/status     the tree, its researches, who is at work, what waits
+//   GET <token>/tree.ged   the tree for the Strom app, as it is now
+//   GET <token>/events     server-sent events: hello, change, working
+//
+// Only pages of the Strom app may read it (CORS: https://stromapp.info, and a
+// local copy on localhost for its development), and a browser asks first
+// whether a public page may talk to this computer (Private Network Access).
+import fs from "node:fs";
+import http from "node:http";
+import path from "node:path";
+import crypto from "node:crypto";
+import { spawn, spawnSync } from "node:child_process";
+import { Tree, VERSION } from "./tree.js";
+import { exportGedcom } from "../gedcom/export.js";
+import { liveWorkers } from "./workers.js";
+import { openSessions } from "./session.js";
+import { gitProgram } from "./git.js";
+import { stromLauncher } from "./self.js";
+/** Where a running bridge of a tree is noted. */
+export function liveFile(root) {
+    return path.join(root, ".strom", "live.json");
+}
+function alive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (e) {
+        return e.code === "EPERM";
+    }
+}
+/** The bridge of this tree when it runs. */
+export function liveRunning(root) {
+    try {
+        const info = JSON.parse(fs.readFileSync(liveFile(root), "utf8"));
+        if (info.pid && alive(info.pid))
+            return info;
+    }
+    catch {
+        // none
+    }
+    return undefined;
+}
+function sleep(ms) {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+/** Start the bridge of a tree in the background (or find it running); undefined when it did not come up. */
+export function startLive(root, env) {
+    const running = liveRunning(root);
+    if (running)
+        return running;
+    fs.rmSync(liveFile(root), { force: true });
+    const { command, args } = stromLauncher();
+    const child = spawn(command, [...args, "live", "serve"], {
+        cwd: root,
+        env: { ...env, STROM_TREE: root },
+        detached: true,
+        stdio: "ignore",
+        windowsHide: true,
+    });
+    child.unref();
+    for (let i = 0; i < 100; i++) {
+        const info = liveRunning(root);
+        if (info)
+            return info;
+        sleep(50);
+    }
+    return undefined;
+}
+/** Stop the bridge of a tree; false when none ran. */
+export function stopLive(root) {
+    const info = liveRunning(root);
+    if (!info)
+        return false;
+    try {
+        process.kill(info.pid);
+    }
+    catch {
+        // already gone
+    }
+    fs.rmSync(liveFile(root), { force: true });
+    return true;
+}
+/** The pages that may read the bridge: the Strom app, and a local copy of it for its development. */
+function allowedOrigin(origin) {
+    if (!origin)
+        return undefined;
+    if (origin === "https://stromapp.info")
+        return origin;
+    return /^http:\/\/(localhost|127\.0\.0\.1)(:\d+)?$/.test(origin) ? origin : undefined;
+}
+function head(root) {
+    const r = spawnSync(gitProgram() ?? "git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8", windowsHide: true });
+    return r.status === 0 ? r.stdout.trim() : "";
+}
+function subjects(root, from, to) {
+    const r = spawnSync(gitProgram() ?? "git", ["log", "--format=%s", from ? `${from}..${to}` : "-1", to], { cwd: root, encoding: "utf8", windowsHide: true });
+    return r.status === 0 ? r.stdout.trim().split("\n").filter(Boolean).reverse() : [];
+}
+/** Who is at work now, on which task (an agent strom started, and its open session). */
+function working(root, tree) {
+    const open = openSessions(tree);
+    return liveWorkers(root).map((w) => {
+        const s = open.find((x) => x.worker === w.id);
+        const task = s?.task ? tree.get(s.task) : undefined;
+        return { who: w.label, since: w.since, ...(s ? { session: s.id, ...(s.task ? { task: task ? `${task.id} ${task.what}` : s.task } : {}) } : {}) };
+    });
+}
+/** What the app shows beside the tree. */
+function status(root, env) {
+    const tree = Tree.open(root, env);
+    const waiting = tree.list("task").filter((t) => t.state === "waiting");
+    return {
+        strom: VERSION,
+        tree: { id: tree.config.id, name: tree.config.name, lang: tree.lang },
+        head: head(root),
+        persons: tree.count("person"),
+        families: tree.count("family"),
+        researches: tree.list("research").map((r) => ({ id: r.id, name: r.name, state: r.state })),
+        working: working(root, tree),
+        open: openSessions(tree).map((s) => ({ id: s.id, task: s.task, started: s.started })),
+        waiting: waiting.map((t) => ({ id: t.id, what: t.what, on: t.waitingOn ?? "" })),
+    };
+}
+/** Run the bridge of the tree here, until idle or stopped. */
+export function serveLive(root, env) {
+    const token = crypto.randomBytes(16).toString("hex");
+    const idleMs = Number(env.STROM_LIVE_IDLE_MS ?? 2 * 60 * 60_000);
+    const pollMs = Number(env.STROM_LIVE_POLL_MS ?? 2000);
+    const streams = new Set();
+    let last = Date.now();
+    let ged;
+    const server = http.createServer((req, res) => {
+        last = Date.now();
+        const origin = allowedOrigin(req.headers.origin);
+        if (origin) {
+            res.setHeader("Access-Control-Allow-Origin", origin);
+            res.setHeader("Vary", "Origin");
+        }
+        if (req.method === "OPTIONS") {
+            if (origin) {
+                res.setHeader("Access-Control-Allow-Methods", "GET");
+                res.setHeader("Access-Control-Allow-Headers", String(req.headers["access-control-request-headers"] ?? ""));
+                if (req.headers["access-control-request-private-network"])
+                    res.setHeader("Access-Control-Allow-Private-Network", "true");
+            }
+            res.writeHead(origin ? 204 : 403).end();
+            return;
+        }
+        const [, t, what] = (req.url ?? "").split("?")[0].split("/");
+        if (req.method !== "GET" || t !== token) {
+            res.writeHead(404).end();
+            return;
+        }
+        try {
+            if (what === "status") {
+                res.writeHead(200, { "Content-Type": "application/json; charset=utf-8", "Cache-Control": "no-store" }).end(JSON.stringify(status(root, env)));
+            }
+            else if (what === "tree.ged") {
+                const h = head(root);
+                if (!ged || ged.head !== h) {
+                    const tree = Tree.open(root, env);
+                    ged = { head: h, text: exportGedcom(tree, { for: "strom" }).text };
+                }
+                res.writeHead(200, { "Content-Type": "text/plain; charset=utf-8", "Cache-Control": "no-store", "X-Strom-Head": h }).end(ged.text);
+            }
+            else if (what === "events") {
+                res.writeHead(200, { "Content-Type": "text/event-stream; charset=utf-8", "Cache-Control": "no-store", Connection: "keep-alive" });
+                res.write(`event: hello\ndata: ${JSON.stringify(status(root, env))}\n\n`);
+                streams.add(res);
+                req.on("close", () => streams.delete(res));
+            }
+            else
+                res.writeHead(404).end();
+        }
+        catch (e) {
+            res.writeHead(500, { "Content-Type": "text/plain; charset=utf-8" }).end(e.message);
+        }
+    });
+    return new Promise((resolve) => {
+        server.listen(0, "127.0.0.1", () => {
+            const port = server.address().port;
+            const info = { port, token, pid: process.pid, url: `http://127.0.0.1:${port}/${token}`, started: new Date().toISOString() };
+            fs.mkdirSync(path.dirname(liveFile(root)), { recursive: true });
+            fs.writeFileSync(liveFile(root), JSON.stringify(info, null, 2));
+            // What changed: strom commits every change, so a new commit is news.
+            let seen = head(root);
+            let workers = JSON.stringify(working(root, Tree.open(root, env)));
+            const send = (event, data) => {
+                for (const s of streams)
+                    s.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
+            };
+            const timer = setInterval(() => {
+                if (streams.size)
+                    last = Date.now();
+                if (Date.now() - last > idleMs)
+                    return stop();
+                if (!streams.size)
+                    return;
+                const h = head(root);
+                if (h && h !== seen) {
+                    send("change", { head: h, what: subjects(root, seen, h), at: new Date().toISOString() });
+                    seen = h;
+                }
+                const now = working(root, Tree.open(root, env));
+                const w = JSON.stringify(now);
+                if (w !== workers) {
+                    workers = w;
+                    send("working", now);
+                }
+            }, pollMs);
+            const keepAlive = setInterval(() => {
+                for (const s of streams)
+                    s.write(": still here\n\n");
+            }, 20_000);
+            const stop = () => {
+                clearInterval(timer);
+                clearInterval(keepAlive);
+                for (const s of streams)
+                    s.end();
+                server.close();
+                try {
+                    const now = JSON.parse(fs.readFileSync(liveFile(root), "utf8"));
+                    if (now.pid === process.pid)
+                        fs.rmSync(liveFile(root), { force: true });
+                }
+                catch {
+                    // gone already
+                }
+                resolve();
+            };
+            process.once("SIGTERM", stop);
+            process.once("SIGINT", stop);
+        });
+    });
+}