strom-research 1.0.0

package/dist/core/git.js

@@ -0,0 +1,300 @@
+// Thin git wrapper. Git is the storage backbone but invisible to the user:
+// `strom` initialises repositories, sets a local identity when none exists
+// and commits on its own.
+import { spawnSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { StromError } from "./errors.js";
+import { userHome } from "./paths.js";
+import { which } from "./which.js";
+const GIT_ENV = {
+    GIT_TERMINAL_PROMPT: "0",
+    GIT_OPTIONAL_LOCKS: "0",
+    LC_ALL: "C",
+};
+const PROFILE = process.env.STROM_PROFILE === "1";
+export function runGit(cwd, args, input) {
+    const t0 = PROFILE ? performance.now() : 0;
+    const r = spawnSync(gitProgram() ?? "git", args, {
+        cwd,
+        input,
+        encoding: "utf8",
+        env: { ...process.env, ...GIT_ENV },
+        maxBuffer: 256 * 1024 * 1024,
+        windowsHide: true,
+    });
+    if (r.error) {
+        const code = r.error.code;
+        if (code === "ENOENT")
+            throw gitMissingError();
+        throw r.error;
+    }
+    if (PROFILE)
+        process.stderr.write(`[git ${(performance.now() - t0).toFixed(0)}ms] ${args.slice(0, 3).join(" ")}\n`);
+    return { status: r.status ?? 1, stdout: r.stdout, stderr: r.stderr };
+}
+function git(cwd, args, input) {
+    const r = runGit(cwd, args, input);
+    if (r.status !== 0)
+        throw new StromError(`git ${args[0]} failed: ${r.stderr.trim() || r.stdout.trim()}`);
+    return r.stdout;
+}
+export function gitInstallHint(platform = process.platform) {
+    if (platform === "linux")
+        return "install git with your package manager, e.g. sudo apt install git  /  sudo dnf install git";
+    return "strom doctor --fix installs it";
+}
+export function gitMissingError() {
+    return new StromError("git is not installed — Strom keeps the research history in git", {
+        hint: gitInstallHint(),
+    });
+}
+/** strom's own git on Windows (MinGit, installed by strom doctor --fix — for strom only, no admin, not on PATH). */
+export function ownGitDir(env) {
+    return path.join(env.LOCALAPPDATA ?? path.join(userHome(env), "AppData", "Local"), "Programs", "Strom", "git");
+}
+const programs = new Map();
+/**
+ * Where git is: STROM_GIT; strom's own (Windows); on PATH; where Git for
+ * Windows installs itself (right after its installer this process's PATH does
+ * not have it yet). On macOS /usr/bin/git is only a stand-in until Apple's
+ * command line tools are installed — running it would open Apple's installer
+ * window unasked — so it counts only with the tools there.
+ */
+export function gitProgram(env = process.env, platform = process.platform) {
+    const key = [platform, env.STROM_GIT, env.PATH ?? env.Path, env.LOCALAPPDATA].join("|");
+    if (programs.has(key))
+        return programs.get(key);
+    let found;
+    if (env.STROM_GIT)
+        found = fs.existsSync(env.STROM_GIT) ? env.STROM_GIT : undefined;
+    else {
+        const own = path.join(ownGitDir(env), "cmd", "git.exe");
+        if (platform === "win32" && fs.existsSync(own))
+            found = own;
+        found ??= which("git", env, platform);
+        if (!found && platform === "win32")
+            found = [env.ProgramFiles, env.ProgramW6432, env.LOCALAPPDATA && path.join(env.LOCALAPPDATA, "Programs")]
+                .filter((d) => Boolean(d))
+                .map((d) => path.join(d, "Git", "cmd", "git.exe"))
+                .find((f) => fs.existsSync(f));
+        if (found === "/usr/bin/git" && platform === "darwin" && spawnSync("xcode-select", ["-p"], { stdio: "ignore" }).status !== 0)
+            found = undefined;
+    }
+    programs.set(key, found);
+    return found;
+}
+/** Git version string, or undefined when git is not available. */
+export function gitVersion(env = process.env) {
+    const git = gitProgram(env);
+    if (!git)
+        return undefined;
+    const r = spawnSync(git, ["--version"], { encoding: "utf8", windowsHide: true });
+    if (r.error || r.status !== 0)
+        return undefined;
+    return r.stdout.trim().replace(/^git version /, "");
+}
+export function isRepo(dir) {
+    const r = runGit(dir, ["rev-parse", "--is-inside-work-tree"]);
+    return r.status === 0 && r.stdout.trim() === "true";
+}
+export function initRepo(dir) {
+    git(dir, ["init", "-q"]);
+    // Stable behaviour on every platform and with non-ASCII file names.
+    git(dir, ["config", "core.autocrlf", "false"]);
+    git(dir, ["config", "core.quotepath", "off"]);
+    git(dir, ["config", "commit.gpgsign", "false"]);
+    ensureIdentity(dir);
+}
+/** Non-technical users have no git identity; set a local one if needed. */
+export function ensureIdentity(dir) {
+    const name = runGit(dir, ["config", "user.name"]).stdout.trim();
+    const email = runGit(dir, ["config", "user.email"]).stdout.trim();
+    if (!name)
+        git(dir, ["config", "user.name", "Strom"]);
+    if (!email)
+        git(dir, ["config", "user.email", "strom@localhost"]);
+}
+// HEAD only changes when strom commits, so it is read once per command.
+const headCache = new Map();
+/** Forget cached git state (at the start of every command, and when a tree is reopened). */
+export function resetCache(dir) {
+    if (dir)
+        headCache.delete(dir);
+    else {
+        headCache.clear();
+        programs.clear();
+    }
+}
+/** HEAD commit, its tree hash and message (cached until the next commit). */
+export function headInfo(dir) {
+    const cached = headCache.get(dir);
+    if (cached !== undefined)
+        return cached ?? undefined;
+    const r = runGit(dir, ["log", "-1", "--format=%H%x00%T%x00%B"]);
+    let info = null;
+    if (r.status === 0 && r.stdout) {
+        const [hash = "", tree = "", ...rest] = r.stdout.split("\0");
+        info = { hash: hash.trim(), tree: tree.trim(), message: rest.join("\0") };
+    }
+    headCache.set(dir, info);
+    return info ?? undefined;
+}
+export function hasHead(dir) {
+    return headInfo(dir) !== undefined;
+}
+export function head(dir) {
+    return headInfo(dir)?.hash;
+}
+function tracked(dir, p) {
+    return runGit(dir, ["ls-files", "--error-unmatch", "--", p]).status === 0;
+}
+const FALLBACK_IDENTITY = {
+    GIT_AUTHOR_NAME: "Strom",
+    GIT_AUTHOR_EMAIL: "strom@localhost",
+    GIT_COMMITTER_NAME: "Strom",
+    GIT_COMMITTER_EMAIL: "strom@localhost",
+};
+function commitTree(dir, tree, parent, message) {
+    const args = ["commit-tree", tree, ...(parent ? ["-p", parent] : []), "-F", "-"];
+    const r = runGit(dir, args, message);
+    if (r.status === 0 || !/identity|tell me who you are|user\.email/i.test(r.stderr))
+        return r;
+    // No git identity at all (non-technical users): commit as "Strom".
+    const retry = spawnSync(gitProgram() ?? "git", args, {
+        cwd: dir,
+        input: message,
+        encoding: "utf8",
+        env: { ...process.env, ...GIT_ENV, ...FALLBACK_IDENTITY },
+        windowsHide: true,
+    });
+    return { status: retry.status ?? 1, stdout: retry.stdout, stderr: retry.stderr };
+}
+/**
+ * Commit everything under the given paths. `seal` receives the git tree hash
+ * of the commit-to-be and returns trailer lines (the commit seal).
+ * Plumbing (write-tree, commit-tree, update-ref) instead of `git commit`:
+ * fewer processes, no hooks, no editor. Returns the hash, or undefined if
+ * nothing changed.
+ */
+export function commitAll(dir, message, pathspec = ["."], seal) {
+    // Paths that no longer exist are fine for `add -A` only if git knows them;
+    // filter missing, untracked-never-seen paths out to keep `add` from failing.
+    const specs = pathspec.filter((p) => p === "." || fs.existsSync(path.join(dir, p)) || tracked(dir, p));
+    if (specs.length)
+        git(dir, ["add", "-A", "--", ...specs]);
+    const tree = git(dir, ["write-tree"]).trim();
+    const parent = headInfo(dir);
+    if (parent && parent.tree === tree)
+        return undefined;
+    const full = seal ? `${message}\n\n${seal(tree)}\n` : `${message}\n`;
+    const r = commitTree(dir, tree, parent?.hash, full);
+    if (r.status !== 0)
+        throw new StromError(`git commit failed: ${r.stderr.trim()}`);
+    const hash = r.stdout.trim();
+    git(dir, ["update-ref", "HEAD", hash, ...(parent ? [parent.hash] : [])]);
+    headCache.set(dir, { hash, tree, message: full });
+    tidy(dir);
+    return hash;
+}
+/**
+ * Pack loose objects now and then. Plumbing never runs git's own `gc --auto`,
+ * so every change of every record would stay a loose file for good (a migrated
+ * tree: 30 000 of them, 900 MB). Git's own estimate first — the objects in one
+ * of its 256 folders (gc.auto 6700 / 256) — so a commit costs no extra process;
+ * `gc --auto` then packs in the background, as `git commit` would.
+ */
+function tidy(dir) {
+    let n;
+    try {
+        n = fs.readdirSync(path.join(dir, ".git", "objects", "17")).length;
+    }
+    catch {
+        return;
+    }
+    if (n > 27)
+        runGit(dir, ["gc", "--auto", "--quiet"]);
+}
+/**
+ * Read many files at one revision with a single git process.
+ * Returns path -> content (undefined when the file did not exist there).
+ */
+export function showFiles(dir, rev, files) {
+    const out = new Map();
+    if (files.length === 0)
+        return out;
+    const r = spawnSync(gitProgram() ?? "git", ["cat-file", "--batch"], {
+        cwd: dir,
+        input: files.map((f) => `${rev}:${f.replace(/\\/g, "/")}`).join("\n") + "\n",
+        env: { ...process.env, ...GIT_ENV },
+        maxBuffer: 1024 * 1024 * 1024,
+        windowsHide: true,
+    });
+    if (r.error)
+        throw r.error;
+    const buf = r.stdout;
+    let pos = 0;
+    for (const file of files) {
+        const nl = buf.indexOf(10, pos);
+        const header = buf.subarray(pos, nl).toString("utf8");
+        pos = nl + 1;
+        if (header.endsWith(" missing") || header.endsWith(" ambiguous")) {
+            out.set(file, undefined);
+            continue;
+        }
+        const size = Number(header.split(" ")[2]);
+        out.set(file, buf.subarray(pos, pos + size).toString("utf8"));
+        pos += size + 1; // content is followed by a newline
+    }
+    return out;
+}
+/** Uncommitted changes with their status codes. */
+export function changes(dir, pathspec = []) {
+    const out = git(dir, ["status", "--porcelain=v1", "-z", "--untracked-files=all", "--", ...pathspec]);
+    const items = out.split("\0").filter(Boolean);
+    const list = [];
+    for (let i = 0; i < items.length; i++) {
+        const item = items[i];
+        const code = item.slice(0, 2);
+        list.push({ path: item.slice(3), code });
+        if (code.startsWith("R") || code.startsWith("C"))
+            i++;
+    }
+    return list;
+}
+/** File content at a revision, or undefined if it did not exist there. */
+export function showFile(dir, rev, file) {
+    const r = runGit(dir, ["show", `${rev}:${file.replace(/\\/g, "/")}`]);
+    return r.status === 0 ? r.stdout : undefined;
+}
+/** Files under a directory at a revision. */
+export function listFiles(dir, rev, subdir) {
+    const r = runGit(dir, ["ls-tree", "-r", "--name-only", "-z", rev, "--", subdir]);
+    if (r.status !== 0)
+        return [];
+    return r.stdout.split("\0").filter(Boolean);
+}
+/** Newest commit (within `limit`) whose tree hash and message satisfy `ok`. */
+export function findCommit(dir, ok, limit = 1000) {
+    const r = runGit(dir, ["log", `-n${limit}`, "--format=%H%x00%T%x00%B%x1e"]);
+    if (r.status !== 0)
+        return undefined;
+    for (const entry of r.stdout.split("\x1e")) {
+        const [hash, tree, message] = entry.replace(/^\n/, "").split("\0");
+        if (hash && tree && message !== undefined && ok(tree, message))
+            return hash;
+    }
+    return undefined;
+}
+export function log(dir, limit, pathspec = []) {
+    if (!hasHead(dir))
+        return [];
+    const out = git(dir, ["log", `-n${limit}`, "--format=%H%x1f%aI%x1f%s", "--", ...pathspec]);
+    return out
+        .split("\n")
+        .filter(Boolean)
+        .map((line) => {
+        const [hash = "", at = "", subject = ""] = line.split("\x1f");
+        return { hash, at, subject };
+    });
+}

package/dist/core/guard.js

@@ -0,0 +1,124 @@
+// `strom guard` — did anything get LOST since the last commit?
+//
+// Nothing in Strom is deleted: records and facts are retracted with a
+// reason. So a record file that disappeared, or an event, note, name,
+// partner or child that vanished from a record, is a loss — unless an
+// operation logged since the last commit names it with a reason.
+// Only files changed since HEAD are examined (one git status + one batched
+// read), so the cost does not grow with the size of the tree.
+import fs from "node:fs";
+import path from "node:path";
+import * as git from "./git.js";
+import { RECORD_TYPES } from "./model.js";
+import { snapshot } from "./integrity.js";
+/** Array fields whose items must never disappear, with the key identifying an item. */
+const KEPT_ARRAYS = {
+    events: (i) => String(i.id),
+    notes: (i) => JSON.stringify(i),
+    // a person's name is its words (its kind or a citation added is no change of it); a place's name is all of it
+    names: (i) => {
+        const n = i;
+        if (n.given === undefined && n.surname === undefined)
+            return JSON.stringify(i);
+        return `${n.given ?? ""} /${n.surname ?? ""}/`;
+    },
+    partners: (i) => String(i),
+    children: (i) => String(i.person),
+};
+const RECORD_DIRS = new Set(Object.values(RECORD_TYPES).map((d) => `data/${d.dir}/`));
+function parse(text) {
+    if (text === undefined)
+        return undefined;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+export function guard(tree, snap = snapshot(tree)) {
+    const out = [];
+    if (!git.hasHead(tree.root))
+        return out;
+    const changes = snap.changes.filter((c) => c.path.startsWith("data/"));
+    if (changes.length === 0)
+        return out;
+    const before = snap.head;
+    // Operations logged since HEAD (the new tail of each changed log).
+    const justified = new Set();
+    for (const c of changes) {
+        if (!c.path.startsWith("data/ops/"))
+            continue;
+        const abs = path.join(tree.root, c.path);
+        const old = before.get(c.path) ?? "";
+        if (!fs.existsSync(abs)) {
+            if (old)
+                out.push({ level: "error", code: "ops-deleted", file: c.path, message: "operation log was deleted" });
+            continue;
+        }
+        const now = fs.readFileSync(abs, "utf8");
+        if (!now.startsWith(old)) {
+            out.push({ level: "error", code: "ops-rewritten", file: c.path, message: "operation log was rewritten (it is append-only)" });
+            continue;
+        }
+        for (const line of now.slice(old.length).split("\n")) {
+            if (!line.trim())
+                continue;
+            try {
+                const op = JSON.parse(line);
+                if (op.reason)
+                    for (const t of op.targets)
+                        justified.add(t);
+            }
+            catch {
+                // reported by verify
+            }
+        }
+    }
+    for (const c of changes) {
+        const dir = c.path.slice(0, c.path.lastIndexOf("/") + 1);
+        if (!RECORD_DIRS.has(dir) || !c.path.endsWith(".json"))
+            continue;
+        const prev = parse(before.get(c.path));
+        if (!prev)
+            continue; // new record: nothing to lose
+        const id = prev.id ?? path.basename(c.path, ".json");
+        const abs = path.join(tree.root, c.path);
+        if (!fs.existsSync(abs)) {
+            if (!justified.has(id))
+                out.push({
+                    level: "error",
+                    code: "record-deleted",
+                    id,
+                    file: c.path,
+                    message: "record was deleted",
+                    hint: "records are never deleted — strom repair, then retract it with --reason",
+                });
+            continue;
+        }
+        const cur = parse(fs.readFileSync(abs, "utf8"));
+        if (!cur)
+            continue;
+        for (const [field, keyOf] of Object.entries(KEPT_ARRAYS)) {
+            const a = prev[field];
+            if (!Array.isArray(a))
+                continue;
+            const b = cur[field];
+            const remaining = new Set(Array.isArray(b) ? b.map(keyOf) : []);
+            for (const item of a) {
+                const key = keyOf(item);
+                if (remaining.has(key) || justified.has(id) || justified.has(key))
+                    continue;
+                out.push({
+                    level: "error",
+                    code: "item-removed",
+                    id,
+                    file: c.path,
+                    message: `${field} item removed: ${key.slice(0, 80)}`,
+                    hint: "retract instead of deleting, or give --reason",
+                });
+            }
+        }
+    }
+    return out;
+}

package/dist/core/http2.js

@@ -0,0 +1,76 @@
+// One request over HTTP/2, answered as fetch() answers. Node's fetch speaks only
+// HTTP/1.1, and some servers answer that with 426 (Upgrade Required): their
+// image servers, for one. strom then asks them again the way a browser does,
+// over HTTP/2 — with the same user agent, the same pace, the same cookies.
+import http2 from "node:http2";
+import zlib from "node:zlib";
+/** Headers of a connection, not of a request: HTTP/2 has none of them. */
+const CONNECTION_HEADERS = new Set(["connection", "keep-alive", "proxy-connection", "transfer-encoding", "upgrade", "host"]);
+/** GET, POST or HEAD over HTTP/2 (https: by ALPN, http: by prior knowledge); redirects are not followed. */
+export function fetchH2(url, init) {
+    const u = new URL(url);
+    return new Promise((resolve, reject) => {
+        let settled = false;
+        const session = http2.connect(u.origin);
+        const done = (err, res) => {
+            if (settled)
+                return;
+            settled = true;
+            session.close();
+            if (err)
+                reject(err);
+            else
+                resolve(res);
+        };
+        session.on("error", (e) => done(e));
+        const headers = { ":method": init.method, ":path": u.pathname + u.search };
+        for (const [k, v] of Object.entries(init.headers))
+            if (!CONNECTION_HEADERS.has(k.toLowerCase()))
+                headers[k.toLowerCase()] = v;
+        const req = session.request(headers);
+        const abort = () => {
+            req.close(http2.constants.NGHTTP2_CANCEL);
+            done(init.signal?.reason ?? new Error("aborted"));
+        };
+        if (init.signal?.aborted)
+            return abort();
+        init.signal?.addEventListener("abort", abort, { once: true });
+        let head = {};
+        const chunks = [];
+        req.on("response", (h) => (head = h));
+        req.on("data", (c) => chunks.push(c));
+        req.on("error", (e) => done(e));
+        req.on("end", () => {
+            init.signal?.removeEventListener("abort", abort);
+            try {
+                const status = Number(head[":status"]);
+                const out = new Headers();
+                for (const [k, v] of Object.entries(head)) {
+                    if (k.startsWith(":") || v === undefined)
+                        continue;
+                    for (const one of Array.isArray(v) ? v : [String(v)])
+                        out.append(k, one);
+                }
+                let body = Buffer.concat(chunks);
+                // as fetch does: a compressed answer is read uncompressed
+                const enc = String(head["content-encoding"] ?? "").toLowerCase();
+                if (body.length && enc === "gzip")
+                    body = zlib.gunzipSync(body);
+                else if (body.length && enc === "br")
+                    body = zlib.brotliDecompressSync(body);
+                else if (body.length && enc === "deflate")
+                    body = zlib.inflateSync(body);
+                if (init.method === "HEAD" || status === 204 || status === 304)
+                    body = null;
+                done(undefined, new Response(body, { status, headers: out }));
+            }
+            catch (e) {
+                done(e);
+            }
+        });
+        if (init.body !== undefined && init.method === "POST")
+            req.end(init.body);
+        else
+            req.end();
+    });
+}