strom-research 1.0.0

package/dist/core/lock.js

@@ -0,0 +1,112 @@
+// Cross-platform file lock: exclusive create ("wx") + owner info.
+// A lock whose process is gone (same host) or which is older than `staleMs`
+// is taken over. Works the same on Windows, macOS and Linux.
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { LockedError } from "./errors.js";
+function sleepSync(ms) {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+function processAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code === "EPERM";
+    }
+}
+export function readLock(file) {
+    try {
+        return JSON.parse(fs.readFileSync(file, "utf8"));
+    }
+    catch {
+        return undefined;
+    }
+}
+function isStale(info, staleMs) {
+    if (!info)
+        return true; // unreadable: locks appear whole, so this one was damaged (a crash, a full disk)
+    // On this computer the process decides: a live holder keeps its lock however
+    // long it works (a big intake), a dead one loses it at once.
+    if (info.host === os.hostname())
+        return !processAlive(info.pid);
+    return Date.now() - Date.parse(info.at) > staleMs;
+}
+/** Who holds the lock now, if anyone alive does. */
+export function liveHolder(file, staleMs = 24 * 3600_000) {
+    if (!fs.existsSync(file))
+        return undefined;
+    const info = readLock(file);
+    return isStale(info, staleMs) ? undefined : info;
+}
+/** Acquire the lock or throw LockedError. Returns the release function. */
+export function acquireLock(file, opts) {
+    const waitMs = opts.waitMs ?? 5000;
+    const staleMs = opts.staleMs ?? 10 * 60 * 1000;
+    const deadline = Date.now() + waitMs;
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    const info = { pid: process.pid, host: os.hostname(), at: new Date().toISOString(), owner: opts.owner };
+    // The lock appears whole or not at all: written aside, then linked in under its name
+    // (a link fails if the name exists). A lock created empty and filled a moment later
+    // looked abandoned to a process that read it in that moment — and two held it.
+    const body = JSON.stringify(info);
+    const aside = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
+    fs.writeFileSync(aside, body);
+    try {
+        for (;;) {
+            try {
+                fs.linkSync(aside, file);
+                return () => {
+                    const cur = readLock(file);
+                    if (cur && cur.pid === info.pid && cur.at === info.at)
+                        fs.rmSync(file, { force: true });
+                };
+            }
+            catch (err) {
+                if (err.code !== "EEXIST")
+                    throw err;
+            }
+            const holder = readLock(file);
+            if (isStale(holder, staleMs)) {
+                // Taken out of the way in one step; of several waiters only one moves it, and only
+                // the lock it judged dead — another one is put back.
+                const dead = `${file}.dead.${process.pid}.${Math.random().toString(36).slice(2)}`;
+                try {
+                    fs.renameSync(file, dead);
+                    const moved = readLock(dead);
+                    if (holder && moved && (moved.pid !== holder.pid || moved.at !== holder.at)) {
+                        try {
+                            fs.linkSync(dead, file);
+                        }
+                        catch {
+                            // someone took the name meanwhile: that one holds it now
+                        }
+                    }
+                    fs.rmSync(dead, { force: true });
+                }
+                catch {
+                    // someone else moved it first
+                }
+                continue;
+            }
+            if (Date.now() >= deadline) {
+                throw new LockedError(`locked by ${holder?.owner ?? "another process"} (pid ${holder?.pid ?? "?"} since ${holder?.at ?? "?"})`, "wait for it to finish, or check `strom status`");
+            }
+            sleepSync(50);
+        }
+    }
+    finally {
+        fs.rmSync(aside, { force: true });
+    }
+}
+export function withLock(file, opts, fn) {
+    const release = acquireLock(file, opts);
+    try {
+        return fn();
+    }
+    finally {
+        release();
+    }
+}

package/dist/core/logins.js

@@ -0,0 +1,67 @@
+// Logins of the user to archive portals: an account they have, perhaps one they
+// paid for. The user types them in on a terminal (strom login <connector>),
+// never an agent. They stay on this computer, in the user's config folder (which
+// agents may not read) — never in a tree, the shared folder or git. strom puts a
+// value into a request of the connector itself, only to the hosts it was saved
+// for and only over https: the connector's code never sees it, nor does the
+// agent that runs it, and it is taken out of every answer the connector reads.
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { configDir } from "./paths.js";
+import { readJsonIfExists, stringifyCanonical } from "./json.js";
+/** Fields typed in the open; every other one (a password, a key, a token) is hidden, and kept out of answers. */
+export const VISIBLE_FIELDS = new Set(["user", "email"]);
+export function loginsFile(env) {
+    return path.join(configDir(env), "logins.json");
+}
+export function loadLogins(env) {
+    return readJsonIfExists(loginsFile(env)) ?? {};
+}
+/** Written readable by the user alone, and never half. */
+function write(env, all) {
+    const file = loginsFile(env);
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    const tmp = path.join(path.dirname(file), `.logins.${process.pid}.${crypto.randomBytes(4).toString("hex")}.tmp`);
+    fs.writeFileSync(tmp, stringifyCanonical(all), { encoding: "utf8", mode: 0o600 });
+    try {
+        fs.renameSync(tmp, file);
+    }
+    catch (err) {
+        fs.rmSync(tmp, { force: true });
+        throw err;
+    }
+}
+export function saveLogin(env, name, login) {
+    write(env, { ...loadLogins(env), [name]: login });
+}
+export function removeLogin(env, name) {
+    const all = loadLogins(env);
+    if (!all[name])
+        return false;
+    delete all[name];
+    write(env, all);
+    return true;
+}
+/** The login saved for this connector — for its folder, not for another one of the same name. */
+export function loginOf(env, c) {
+    const l = loadLogins(env)[c.name];
+    return l && path.resolve(l.dir) === path.resolve(c.dir) ? l : undefined;
+}
+/** The forms a value takes in a page, a URL, a form or JSON: each one comes out of an answer. */
+function spellings(v) {
+    const html = v.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+    const form = new URLSearchParams({ v }).toString().slice(2);
+    return [...new Set([v, encodeURIComponent(v), form, html, JSON.stringify(v).slice(1, -1)])].sort((a, b) => b.length - a.length);
+}
+/** Takes the hidden values of a login out of a text: "[login]" stands in their place. */
+export function redactor(login) {
+    const secrets = login
+        ? Object.entries(login.values)
+            .filter(([k, v]) => !VISIBLE_FIELDS.has(k) && v.length >= 4)
+            .flatMap(([, v]) => spellings(v))
+        : [];
+    if (!secrets.length)
+        return (s) => s;
+    return (s) => secrets.reduce((t, v) => t.split(v).join("[login]"), s);
+}

package/dist/core/media.js

@@ -0,0 +1,223 @@
+// Files that come into a research: stored unchanged, found again by their
+// SHA-256. Small material from the family goes into the tree (and git) —
+// it cannot be downloaded again; large files go to the shared media store.
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { Settings } from "./config.js";
+import { UsageError } from "./errors.js";
+import { safeFolderName } from "./text.js";
+export const MAX_IN_TREE = 20 * 1024 * 1024;
+export function fileSha256(file) {
+    const h = crypto.createHash("sha256");
+    const fd = fs.openSync(file, "r");
+    try {
+        const buf = Buffer.allocUnsafe(1 << 20);
+        let n;
+        while ((n = fs.readSync(fd, buf, 0, buf.length, null)) > 0)
+            h.update(buf.subarray(0, n));
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+    return h.digest("hex");
+}
+/** Shared store path for a hash: media/ab/cd/abcd….ext */
+export function sharedMediaPath(sharedDir, sha, ext) {
+    return path.join(sharedDir, "media", sha.slice(0, 2), sha.slice(2, 4), `${sha}${ext.toLowerCase()}`);
+}
+/** Copy into the shared store (no-op when the same content is already there). */
+export function storeShared(sharedDir, file, sha) {
+    const dest = sharedMediaPath(sharedDir, sha, path.extname(file));
+    if (!fs.existsSync(dest)) {
+        fs.mkdirSync(path.dirname(dest), { recursive: true });
+        // a clone where Node can make one (Linux on Btrfs or XFS: no space taken until one of them changes), else a copy —
+        // on macOS Node copies (it cannot clone), so an archive of scans takes its size again
+        fs.copyFileSync(file, dest, fs.constants.COPYFILE_FICLONE);
+    }
+    return dest;
+}
+const MIME = {
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".png": "image/png",
+    ".tif": "image/tiff",
+    ".tiff": "image/tiff",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".heic": "image/heic",
+    ".pdf": "application/pdf",
+    ".txt": "text/plain",
+    ".md": "text/markdown",
+    ".rtf": "application/rtf",
+    ".doc": "application/msword",
+    ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    ".odt": "application/vnd.oasis.opendocument.text",
+    ".ged": "text/x-gedcom",
+    ".json": "application/json",
+    ".html": "text/html",
+    ".csv": "text/csv",
+    ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    ".mp3": "audio/mpeg",
+    ".m4a": "audio/mp4",
+};
+export function mimeOf(file) {
+    return MIME[path.extname(file).toLowerCase()] ?? "application/octet-stream";
+}
+/** Every file under the given paths (folders recursively), hidden files skipped. */
+export function collectFiles(paths) {
+    const out = [];
+    const walk = (p) => {
+        const base = path.basename(p);
+        if (base.startsWith(".") || base === "Thumbs.db" || base === "desktop.ini")
+            return;
+        const st = fs.statSync(p);
+        if (st.isDirectory())
+            for (const e of fs.readdirSync(p).sort())
+                walk(path.join(p, e));
+        else if (st.isFile())
+            out.push(p);
+    };
+    for (const p of paths)
+        walk(p);
+    return out;
+}
+const digitGroups = (file) => path.basename(file, path.extname(file)).match(/\d+/g) ?? [];
+/**
+ * The image number in one file name: the last zero-padded group of digits
+ * ("s0057.jpg" → 57, "…_00141_sign-4411.jpg" → 141), else the last group.
+ */
+export function numberInName(file) {
+    const groups = digitGroups(file);
+    const g = groups.filter((d) => d.length > 1 && d.startsWith("0")).at(-1) ?? groups.at(-1);
+    return g === undefined ? undefined : Number(g);
+}
+/** The folder of the shared inbox where the user saves images of a record set by hand: "B0003 <its title>". */
+export function inboxFolderFor(b) {
+    const name = [...safeFolderName(`${b.id} ${b.title}`)];
+    return name.length <= 80 ? name.join("") : name.slice(0, 80).join("").replace(/[. ]+$/, "");
+}
+/** A list of image numbers as people write it: "9", "9-12", "9–12, 15" → [9, 10, 11, 12, 15]; undefined if it is not one. */
+export function parseImageList(s, max = 2000) {
+    const parts = s.split(",").map((p) => p.trim());
+    const out = new Set();
+    for (const p of parts) {
+        const m = /^(\d+)(?:\s*[-–]\s*(\d+))?$/.exec(p);
+        if (!m)
+            return undefined;
+        const a = Number(m[1]);
+        const z = Number(m[2] ?? m[1]);
+        if (a < 1 || z < a || z - a >= max)
+            return undefined;
+        for (let n = a; n <= z && out.size <= max; n++)
+            out.add(n);
+    }
+    return out.size && out.size <= max ? [...out].sort((x, y) => x - y) : undefined;
+}
+/**
+ * Image numbers of the files of one download: the group of digits that changes
+ * from file to file (an archive's file names also carry a call number or a
+ * fonds code that stays the same).
+ */
+export function imageNumbers(files) {
+    const groups = files.map(digitGroups);
+    const n = groups[0]?.length ?? 0;
+    if (files.length > 1 && n > 0 && groups.every((g) => g.length === n)) {
+        const varying = [...Array(n).keys()].filter((i) => new Set(groups.map((g) => Number(g[i]))).size > 1);
+        const i = varying.at(-1);
+        if (i !== undefined)
+            return groups.map((g) => Number(g[i]));
+    }
+    return files.map(numberInName);
+}
+/** The shared inbox by folder ("" = files lying in the inbox itself): one download, one record set. */
+export function inboxFolders(inbox) {
+    if (!fs.existsSync(inbox))
+        return [];
+    const by = new Map();
+    for (const f of collectFiles([inbox])) {
+        const rel = path.relative(inbox, f).split(path.sep);
+        const folder = rel.length > 1 ? rel[0] : "";
+        by.set(folder, [...(by.get(folder) ?? []), f]);
+    }
+    return [...by].map(([folder, files]) => ({ folder, files }));
+}
+/**
+ * Image n of a record set: the whole image, before any part of it fetched sharper —
+ * of several copies of the whole image (a reduced download, then the full one), the sharpest.
+ */
+export function findImage(all, recordset, image) {
+    const of = all.filter((m) => m.recordset === recordset && m.image === image);
+    const pixels = (m) => (m.width ?? 0) * (m.height ?? 0);
+    return of.filter((m) => !m.part).sort((a, b) => pixels(b) - pixels(a))[0] ?? of[0];
+}
+/**
+ * Image n as a reference names it ("B0001:57"): the whole image. When only parts of it are
+ * registered (its two halves), "B0001:57" says nothing of which: the parts are named, none is picked.
+ */
+export function imageOfRef(all, recordset, image) {
+    const m = findImage(all, recordset, image);
+    if (!m?.part)
+        return m;
+    const parts = all.filter((x) => x.recordset === recordset && x.image === image && x.part);
+    if (parts.length < 2)
+        return m;
+    throw new UsageError(`image ${image} of ${recordset} is registered only in parts: ${parts.map((p) => `${p.id} (part ${regionText(p.part)})`).join(", ")}`, {
+        hint: `name the part the record is on: ${parts.map((p) => p.id).join(" or ")}`,
+    });
+}
+/** The other copies of the same whole image, sharpest first. */
+export function otherCopies(all, m) {
+    if (m.part || m.recordset === undefined || m.image === undefined)
+        return [];
+    return all
+        .filter((x) => x.id !== m.id && !x.part && x.recordset === m.recordset && x.image === m.image)
+        .sort((a, b) => (b.width ?? 0) * (b.height ?? 0) - (a.width ?? 0) * (a.height ?? 0));
+}
+/** The same part of an image (to a thousandth of it). */
+export function sameRegion(a, b) {
+    return Math.abs(a.x - b.x) < 0.001 && Math.abs(a.y - b.y) < 0.001 && Math.abs(a.w - b.w) < 0.001 && Math.abs(a.h - b.h) < 0.001;
+}
+/** "0.5,0,0.5,0.5" — a region as --crop takes it. */
+export function regionText(r) {
+    return [r.x, r.y, r.w, r.h].map((n) => String(Math.round(n * 1000) / 1000)).join(",");
+}
+/**
+ * A registered part of a whole image that shows a region of it (fractions of the
+ * whole) with more detail than the whole image has: the sharpest one, and where
+ * the region is in it. Worth it from a fifth more pixels on. Another copy of the
+ * whole image with more pixels counts as a part that covers all of it.
+ */
+export function sharperPart(all, whole, r) {
+    if (!whole.width || whole.part || whole.recordset === undefined || whole.image === undefined)
+        return undefined;
+    const e = 0.002;
+    let best;
+    for (const m of all) {
+        const p = m.part ?? (m.id !== whole.id ? { x: 0, y: 0, w: 1, h: 1 } : undefined);
+        if (!p || !m.width || m.recordset !== whole.recordset || m.image !== whole.image)
+            continue;
+        if (r.x < p.x - e || r.y < p.y - e || r.x + r.w > p.x + p.w + e || r.y + r.h > p.y + p.h + e)
+            continue;
+        const gain = m.width / p.w / whole.width;
+        if (gain < 1.2 || (best && gain <= best.gain))
+            continue;
+        const f = (n) => Math.min(1, Math.max(0, n));
+        const x = f((r.x - p.x) / p.w);
+        const y = f((r.y - p.y) / p.h);
+        best = { part: m, gain, crop: { x, y, w: Math.min(1 - x, r.w / p.w), h: Math.min(1 - y, r.h / p.h) } };
+    }
+    return best;
+}
+/** Where an input's file is on disk (in the tree, or in the shared media store). */
+export function inputPath(tree, input) {
+    if (!input.file)
+        return undefined;
+    if (input.file.startsWith("inputs/"))
+        return path.join(tree.root, input.file);
+    if (input.file.startsWith("media:")) {
+        const shared = new Settings(tree.env, {}).shared()?.value;
+        return shared ? path.join(shared, input.file.slice(6)) : undefined;
+    }
+    return undefined;
+}

package/dist/core/model.js

@@ -0,0 +1,101 @@
+// Data model. One record = one JSON file under data/<collection>/<ID>.json.
+// Facts are "conclusions" backed by citations; nothing is ever deleted,
+// records and facts are retracted with a reason instead.
+export const RECORD_TYPES = {
+    research: { prefix: "G", dir: "researches" },
+    person: { prefix: "P", dir: "persons" },
+    family: { prefix: "F", dir: "families" },
+    source: { prefix: "S", dir: "sources" },
+    repository: { prefix: "R", dir: "repositories" },
+    recordset: { prefix: "B", dir: "recordsets" },
+    place: { prefix: "L", dir: "places" },
+    search: { prefix: "Q", dir: "searches" },
+    task: { prefix: "T", dir: "tasks" },
+    conflict: { prefix: "X", dir: "conflicts" },
+    hypothesis: { prefix: "H", dir: "hypotheses" },
+    lesson: { prefix: "K", dir: "lessons" },
+    input: { prefix: "I", dir: "inputs" },
+    session: { prefix: "N", dir: "sessions" },
+    media: { prefix: "M", dir: "images" },
+};
+/** Prefixes of IDs that are not files of their own (embedded records). */
+export const EMBEDDED_PREFIXES = {
+    event: "E",
+};
+export const ALL_PREFIXES = {
+    ...Object.fromEntries(Object.entries(RECORD_TYPES).map(([t, v]) => [t, v.prefix])),
+    ...EMBEDDED_PREFIXES,
+};
+/** Certainty of the conclusion (a fact). The quality of each piece of evidence is on its citation (GEDCOM QUAY). */
+export const STATUSES = ["proven", "probable", "possible", "lead", "disproven", "retracted"];
+export const PARTICIPANT_ROLES = ["godparent", "witness", "officiant", "informant", "midwife", "other"];
+export const NAME_KINDS = ["birth", "married", "religious", "alias"];
+export const CHILD_RELATIONS = ["birth", "adopted", "step", "foster", "unknown"];
+export const DIRECTIONS = ["ancestors", "descendants", "person", "question"];
+export const RESEARCH_STATES = ["active", "paused", "done"];
+export const SOURCE_KINDS = [
+    "baptism", "birth", "marriage", "death", "burial", "census", "civil", "church", "land", "tax",
+    "military", "court", "notarial", "newspaper", "index", "letter", "photo", "family-tree",
+    "family-memory", "certificate", "book", "web", "other",
+];
+export const AUTOMATION = ["allowed", "manual", "forbidden", "unknown"];
+export const ACCESS = ["online-free", "online-login", "onsite", "request", "lost", "unknown"];
+export const JURISDICTIONS = ["parish", "civil", "diocese", "manor", "district", "county", "state", "other"];
+export const SEARCH_METHODS = ["index", "page-by-page", "full-text", "catalog", "web", "request", "other"];
+export const SEARCH_RESULTS = ["found", "negative", "partial", "inconclusive"];
+/** How the task queue is ordered (core/queue.ts). */
+export const STRATEGIES = ["balanced", "depth", "priority"];
+export const TASK_LEVELS = ["intake", "locate", "link", "verify", "enrich", "request", "narrate"];
+export const TASK_STATES = ["open", "doing", "done", "parked", "waiting", "dropped"];
+export const LESSON_SCOPES = ["method", "repository", "recordset", "place", "project"];
+export const LESSON_MAX = 200;
+export const INPUT_KINDS = ["document", "photo", "tree", "text", "other"];
+export const SCHEMA_VERSION = 1;
+export const NOTE_MAX = 500;
+/** Event tags accepted by `event add`, with friendly aliases. */
+export const EVENT_KINDS = {
+    BIRT: "birth",
+    CHR: "christening",
+    BAPM: "baptism",
+    DEAT: "death",
+    BURI: "burial",
+    CREM: "cremation",
+    MARR: "marriage",
+    MARB: "marriage banns",
+    MARC: "marriage contract",
+    MARL: "marriage license",
+    MARS: "marriage settlement",
+    ANUL: "annulment",
+    DIVF: "divorce filed",
+    ADOP: "adoption",
+    GRAD: "graduation",
+    RETI: "retirement",
+    CHRA: "adult christening",
+    DIV: "divorce",
+    ENGA: "engagement",
+    OCCU: "occupation",
+    RESI: "residence",
+    EDUC: "education",
+    RELI: "religion",
+    EMIG: "emigration",
+    IMMI: "immigration",
+    NATU: "naturalization",
+    CENS: "census",
+    PROB: "probate",
+    WILL: "will",
+    CONF: "confirmation",
+    FCOM: "first communion",
+    ORDN: "ordination",
+    NATI: "nationality",
+    TITL: "title",
+    MILI: "military service",
+    EVEN: "other event",
+};
+export const FAMILY_EVENT_KINDS = new Set(["MARR", "MARB", "MARC", "MARL", "MARS", "DIV", "DIVF", "ANUL", "ENGA", "EVEN", "RESI", "CENS"]);
+const ALIASES = Object.fromEntries(Object.entries(EVENT_KINDS).map(([tag, label]) => [label.replace(/ /g, "-"), tag]));
+export function eventKind(input) {
+    const up = input.toUpperCase();
+    if (up in EVENT_KINDS)
+        return up;
+    return ALIASES[input.toLowerCase().replace(/ /g, "-")];
+}