strom-research 1.0.0 → 1.1.0

package/dist/commands/transcripts.js

@@ -0,0 +1,177 @@
+// strom transcripts — the words of entries already on their scans, for sources
+// that have a clip but no transcript (read before strom asked for the words, or
+// brought in from an older research): a reader writes them from the entry cut
+// out of its scan, a second reader holds them against the same cut-out; what
+// passes becomes the source's transcript, with a note of how it was made. Run
+// again, it takes up only what is still missing.
+import fs from "node:fs";
+import path from "node:path";
+import { register } from "../cli/registry.js";
+import { lines, truncate } from "../cli/format.js";
+import { UsageError } from "../core/errors.js";
+import { EXCERPT_SCOPES } from "../core/model.js";
+import { normId, requireRecord, update } from "../core/records.js";
+import { makeView } from "../core/views.js";
+import { inputPath, regionText, sharperPart } from "../core/media.js";
+import { SCOPE_MAX, sourceNearness, withMargin } from "../core/excerpt.js";
+import { batches } from "../core/reader.js";
+import { checkTranscriptPrompt, parseTranscriptChecks, parseTranscribed, transcribePrompt } from "../core/transcriber.js";
+import { makeNote } from "../core/actions.js";
+import { phrase } from "../core/phrases.js";
+import { verifyFast } from "../core/integrity.js";
+import { pool } from "./read.js";
+import { readers } from "./readers.js";
+import { namesIn } from "./clips.js";
+/** Entries per reader. */
+const PER_READER = 8;
+/** A cut-out as the reader sees it: sharp enough to read a hand, small enough to open. */
+const CUT_MAX = 2000;
+register({
+    path: ["transcripts"],
+    summary: "Write the words of entries already cut out of their scans (sources with a clip, no transcript): readers read them, a second look checks",
+    group: "sources",
+    tree: true,
+    writes: true,
+    lock: "sections",
+    description: "For sources with a clip (or a document the user gave as a scan) and no transcript — read before strom asked for the\n" +
+        "words, or brought in from an older research. A reader writes the entry's words from its cut-out, as they stand (its\n" +
+        "own language and spelling, unread letters [?]); a second reader holds them against the same cut-out and corrects\n" +
+        "them: only then they become the source's transcript, with a note of how they were made. The Strom app shows them next\n" +
+        "to the entry's image. Paid model work, two looks per entry: --dry-run says how many first. Nearest the research first.",
+    args: [{ name: "sources", description: "only these sources (S…)", variadic: true }],
+    options: [
+        { name: "for", type: "string", value: "<whose>", description: "line, family, connected or all (default: all — the nearest to the research first)" },
+        { name: "limit", type: "string", value: "<n>", description: "at most this many sources this time" },
+        { name: "batch", type: "string", value: "<n>", description: `entries per reader (default ${PER_READER})` },
+        { name: "parallel", type: "string", value: "<n>", description: "readers at the same time (default 3)" },
+        { name: "model", type: "string", value: "<model>", description: "model of the readers (default: model.vision — it reads the handwriting)" },
+        { name: "minutes", type: "string", value: "<n>", description: "time limit of one reader (default 15)" },
+    ],
+    examples: ["strom transcripts --dry-run", "strom transcripts --for family --limit 20", "strom transcripts S0001 S0002"],
+    run: async (ctx, { args, opts }) => {
+        const tree = ctx.tree();
+        const shared = ctx.settings.shared()?.value;
+        if (!shared)
+            throw new UsageError("no shared folder with the images is set", { hint: "strom setup" });
+        const scope = (typeof opts.for === "string" ? opts.for : "all");
+        if (!EXCERPT_SCOPES.includes(scope) || scope === "none")
+            throw new UsageError(`invalid --for "${opts.for}"`, { hint: "line, family, connected or all" });
+        const all = tree.list("media");
+        const byId = new Map(all.map((m) => [m.id, m]));
+        const only = new Set(args.map((a) => requireRecord(tree, normId(a), "source").id));
+        const near = sourceNearness(tree);
+        const rank = (s) => near.get(s.id) ?? 4;
+        /** The cut-outs of a source, from the sharpest image of the place there is; none when its image is not here. */
+        const cutsOf = (s) => {
+            if (s.clips?.length)
+                return s.clips.flatMap((c) => {
+                    const m = byId.get(c.media);
+                    if (!m)
+                        return [];
+                    const wanted = withMargin(c.region);
+                    const sharper = sharperPart(all, m, wanted);
+                    const src = sharper?.part ?? m;
+                    const file = path.join(shared, src.file);
+                    return /^image\/(jpeg|png)$/.test(src.mime) && fs.existsSync(file) ? [{ media: m.id, file, crop: regionText(sharper?.crop ?? wanted) }] : [];
+                });
+            const input = s.input ? tree.get(s.input) : undefined;
+            const file = input ? inputPath(tree, input) : undefined;
+            return input && file && /\.(jpe?g|png)$/i.test(file) && fs.existsSync(file) ? [{ media: input.id, file }] : [];
+        };
+        let todo = tree
+            .list("source")
+            .filter((s) => !s.retracted && !s.transcript?.trim() && (s.clips?.length || s.input))
+            .filter((s) => (only.size ? only.has(s.id) : rank(s) <= SCOPE_MAX[scope]))
+            .sort((a, b) => rank(a) - rank(b) || a.id.localeCompare(b.id));
+        const away = todo.filter((s) => !cutsOf(s).length);
+        todo = todo.filter((s) => cutsOf(s).length);
+        const limit = opts.limit === undefined ? undefined : Number(opts.limit);
+        if (limit !== undefined && !(Number.isInteger(limit) && limit > 0))
+            throw new UsageError("--limit must be a positive number");
+        if (limit)
+            todo = todo.slice(0, limit);
+        const size = opts.batch === undefined ? PER_READER : Number(opts.batch);
+        if (!Number.isInteger(size) || size < 1)
+            throw new UsageError("--batch must be a positive number");
+        const groups = batches(todo, size);
+        if (!todo.length)
+            return { text: lines("nothing to do: every entry cut out of its scan has its words", away.length ? `${away.length} source(s) on images not on this computer` : undefined), data: { sources: 0 } };
+        if (tree.dryRun)
+            return {
+                text: lines(`${todo.length} source(s) to transcribe from their cut-outs — ${groups.length} reader(s) to read them, as many to check`, away.length ? `${away.length} more on images not on this computer` : undefined, "(dry run — no reader started)"),
+                data: { sources: todo.length, readers: groups.length, away: away.map((s) => s.id) },
+            };
+        const r0 = readers(ctx, tree, shared, "transcripts", "read", opts);
+        const { parallel, reportsDir, failed, progress } = r0;
+        const facts = (s) => [
+            s.locator || s.date ? `where: ${[s.locator, s.date ? `recorded ${s.date}` : ""].filter(Boolean).join("; ")}` : "",
+            namesIn(tree, s.id).length ? `people in it: ${namesIn(tree, s.id).join(", ")}` : "",
+            s.language ? `language: ${s.language}` : "",
+        ].filter(Boolean);
+        const job = (s) => ({
+            source: s.id,
+            title: s.title,
+            facts: facts(s),
+            views: cutsOf(s).map((c) => ({ media: c.media, view: makeView(tree, c.file, c.media, { ...(c.crop ? { crop: c.crop } : {}), max: CUT_MAX }).file })),
+        });
+        // 1. Read: the words from the cut-out.
+        const jobs = new Map(todo.map((s) => [s.id, job(s)]));
+        const read = (await pool(groups, parallel, async (group, k) => {
+            const js = group.map((s) => jobs.get(s.id));
+            progress(`▶ reading ${k + 1}/${groups.length}: ${group.map((s) => s.id).join(" ")}`);
+            const r = await r0.read(`read-${k + 1}`, js.flatMap((j) => j.views.map((v) => v.view)), `The words of the entries · batch ${k + 1} of ${groups.length}`, (report) => transcribePrompt(js, report));
+            const got = parseTranscribed(r.text).filter((x) => jobs.has(x.source));
+            progress(`■ reading ${k + 1}: ${r.outcome} · ${got.filter((x) => x.result === "read").length} read`);
+            return got;
+        })).flat();
+        const words = read.filter((x) => x.result === "read");
+        // 2. Check: the same cut-out, word by word.
+        const checks = words.map((x) => ({ ...jobs.get(x.source), transcript: x.transcript }));
+        const checkGroups = batches(checks, size);
+        const verdicts = (await pool(checkGroups, parallel, async (group, k) => {
+            progress(`▶ checking ${k + 1}/${checkGroups.length}: ${group.length} transcripts`);
+            const r = await r0.read(`check-${k + 1}`, group.flatMap((j) => j.views.map((v) => v.view)), `Checking the words · batch ${k + 1} of ${checkGroups.length}`, (report) => checkTranscriptPrompt(group, report));
+            const v = parseTranscriptChecks(r.text);
+            progress(`■ checking ${k + 1}: ${r.outcome} · ${v.filter((x) => x.verdict !== "wrong").length} passed`);
+            return v;
+        })).flat();
+        // 3. What passed becomes the source's transcript, with how it was made.
+        const passed = new Map();
+        for (const v of verdicts) {
+            const first = words.find((x) => x.source === v.source);
+            if (!first || passed.has(v.source))
+                continue;
+            if (v.verdict === "ok")
+                passed.set(v.source, first.transcript);
+            else if (v.verdict === "fixed")
+                passed.set(v.source, v.transcript);
+        }
+        const day = new Date().toISOString().slice(0, 10);
+        const how = phrase(tree.lang, "transcript.read", { model: r0.model ?? "?", date: day });
+        for (const [id, transcript] of passed)
+            update(tree, id, "source", (s) => ({ ...s, transcript, notes: [...s.notes, makeNote(tree, how)] }), {
+                op: "source.edit",
+                summary: `${id} transcript (${transcript.split("\n").length} line(s), read by a reader, checked)`,
+            });
+        if (verifyFast(tree).findings.every((f) => f.level !== "error"))
+            tree.withTreeLock(() => tree.commit(`Transcripts: ${passed.size} of ${todo.length} entries written from their cut-outs`, [tree.relative(reportsDir)]));
+        const rejected = verdicts.filter((v) => v.verdict === "wrong");
+        const unread = read.filter((x) => x.result !== "read");
+        const missed = todo.filter((s) => !passed.has(s.id));
+        return {
+            text: lines(`${passed.size} of ${todo.length} source(s) got their words${r0.model ? ` · ${r0.model}` : ""}${r0.cost() ? ` · $${r0.cost().toFixed(2)}` : ""}`, verdicts.some((v) => v.verdict === "fixed") ? `the check corrected ${verdicts.filter((v) => v.verdict === "fixed").length} of them` : undefined, rejected.length ? `the check turned down: ${rejected.map((v) => `${v.source}${v.why ? ` (${truncate(v.why, 40)})` : ""}`).join(", ")}` : undefined, unread.length ? `not read (unreadable, or not this entry in the cut-out): ${unread.map((x) => `${x.source} ${x.result}`).slice(0, 20).join(", ")}` : undefined, missed.length ? `without words: ${missed.map((s) => s.id).slice(0, 20).join(", ")}${missed.length > 20 ? ` … (${missed.length})` : ""} — strom transcripts again, or strom source edit S… --transcript @<file>` : undefined, away.length ? `on images not on this computer: ${away.length}` : undefined, failed.length ? `readers that failed: ${failed.join(", ")}` : undefined, "the words go into the Strom app with the entry's image: strom app (output/tree-strom.ged)"),
+            data: {
+                sources: todo.length,
+                transcribed: [...passed.keys()],
+                corrected: verdicts.filter((v) => v.verdict === "fixed").map((v) => v.source),
+                rejected,
+                unread: unread.map((x) => ({ source: x.source, result: x.result })),
+                missed: missed.map((s) => s.id),
+                away: away.map((s) => s.id),
+                failed,
+                costUsd: r0.cost() || undefined,
+            },
+            exitCode: failed.length ? 1 : 0,
+        };
+    },
+});

package/dist/core/chromium.js

@@ -19,31 +19,26 @@ const BROWSERS = [
 ];
 /** A Chromium browser on this computer, the most common first. STROM_APP_DIRS: folders to look in instead (tests). */
 export function chromiumBrowser(env, platform = process.platform) {
+    return chromiumBrowsers(env, platform)[0];
+}
+/** Every Chromium browser on this computer, the most common first. */
+export function chromiumBrowsers(env, platform = process.platform) {
+    const found = [];
     const own = env.STROM_APP_DIRS !== undefined ? env.STROM_APP_DIRS.split(path.delimiter).filter(Boolean) : undefined;
     for (const b of BROWSERS) {
-        if (platform === "darwin") {
-            for (const d of own ?? ["/Applications", path.join(userHome(env), "Applications")])
-                if (fs.existsSync(path.join(d, b.mac)))
-                    return { name: b.name, path: path.join(d, b.mac) };
-        }
+        let hit;
+        if (platform === "darwin")
+            hit = (own ?? ["/Applications", path.join(userHome(env), "Applications")]).map((d) => path.join(d, b.mac)).find((f) => fs.existsSync(f));
         else if (platform === "win32") {
             const roots = own ?? [env.LOCALAPPDATA, env.ProgramFiles, env["ProgramFiles(x86)"]].filter((r) => Boolean(r));
-            for (const root of roots)
-                for (const rel of b.win) {
-                    const exe = own ? path.join(root, path.win32.basename(rel)) : path.join(root, rel);
-                    if (fs.existsSync(exe))
-                        return { name: b.name, path: exe };
-                }
-        }
-        else {
-            for (const cmd of b.linux) {
-                const hit = own ? own.map((d) => path.join(d, cmd)).find((f) => fs.existsSync(f)) : which(cmd, env, platform);
-                if (hit)
-                    return { name: b.name, path: hit };
-            }
+            hit = roots.flatMap((root) => b.win.map((rel) => (own ? path.join(root, path.win32.basename(rel)) : path.join(root, rel)))).find((f) => fs.existsSync(f));
         }
+        else
+            hit = b.linux.map((cmd) => (own ? own.map((d) => path.join(d, cmd)).find((f) => fs.existsSync(f)) : which(cmd, env, platform))).find(Boolean);
+        if (hit)
+            found.push({ name: b.name, path: hit });
     }
-    return undefined;
+    return found;
 }
 function launch(cmd, args, env) {
     try {
@@ -56,6 +51,52 @@ function launch(cmd, args, env) {
         return false;
     }
 }
+/** Where each browser keeps its profiles, under the system's folder for application data. */
+const USER_DATA = {
+    "Google Chrome": { mac: "Google/Chrome", win: "Google\\Chrome\\User Data", linux: "google-chrome" },
+    "Microsoft Edge": { mac: "Microsoft Edge", win: "Microsoft\\Edge\\User Data", linux: "microsoft-edge" },
+    Brave: { mac: "BraveSoftware/Brave-Browser", win: "BraveSoftware\\Brave-Browser\\User Data", linux: "BraveSoftware/Brave-Browser" },
+    Chromium: { mac: "Chromium", win: "Chromium\\User Data", linux: "chromium" },
+    Vivaldi: { mac: "Vivaldi", win: "Vivaldi\\User Data", linux: "vivaldi" },
+};
+/** The profile of the browser an installed web app belongs to: the one that holds its files (only folder names are looked at). */
+export function webAppProfile(browser, appId, env, platform = process.platform) {
+    const d = USER_DATA[browser];
+    if (!d)
+        return undefined;
+    const home = userHome(env);
+    const root = platform === "darwin"
+        ? path.join(home, "Library", "Application Support", d.mac)
+        : platform === "win32"
+            ? path.join(env.LOCALAPPDATA ?? path.join(home, "AppData", "Local"), d.win)
+            : path.join(env.XDG_CONFIG_HOME ?? path.join(home, ".config"), d.linux);
+    let dirs = [];
+    try {
+        dirs = fs.readdirSync(root);
+    }
+    catch {
+        return undefined;
+    }
+    return dirs.find((p) => fs.existsSync(path.join(root, p, "Web Applications", "Manifest Resources", appId)));
+}
+/**
+ * Open an address in a web app installed from a Chromium browser — its own window, not a tab: the browser
+ * is asked to launch the app (its id, its profile) at that address, which must lie within the app.
+ */
+export function openWebApp(app, url, env, platform = process.platform) {
+    if (env.STROM_NO_OPEN === "1")
+        return false;
+    const b = chromiumBrowsers(env, platform).find((x) => x.name === app.browser);
+    if (!b)
+        return false;
+    const profile = app.profile ?? webAppProfile(app.browser, app.appId, env, platform);
+    const args = [...(profile ? [`--profile-directory=${profile}`] : []), `--app-id=${app.appId}`, `--app-launch-url-for-shortcuts-menu-item=${url}`];
+    // macOS: the browser's own program — it hands the request to the browser already running.
+    const exe = platform === "darwin" ? path.join(b.path, "Contents", "MacOS", path.basename(b.path, ".app")) : b.path;
+    if (platform !== "darwin" && platform !== "win32" && !env.DISPLAY && !env.WAYLAND_DISPLAY)
+        return false;
+    return launch(exe, args, env);
+}
 /** Open an address in that browser; false when this computer cannot (no desktop) or tests ask not to. */
 export function openInBrowser(browser, url, env, platform = process.platform) {
     if (env.STROM_NO_OPEN === "1")

package/dist/core/clipfinder.js

@@ -0,0 +1,118 @@
+// Finding where entries already read are on their scans (the clips of old
+// sources): a reader looks at each image with a grid and says where the entry
+// is; strom cuts that out and a second reader checks the cut-out shows the
+// whole entry, and that entry. Only what passes the check becomes a clip.
+// Readers never write to the research; strom does, from their reports.
+const RULES = `You are a READER for a genealogical research. You are not the researcher: you do not decide
+anything and you do not write anywhere except your report file. Do not run any commands.`;
+export function locatePrompt(jobs, report) {
+    return `${RULES}
+
+Each entry below was read earlier from the image(s) named with it. Open each image (one at a time,
+in this order), find that entry on it, and say where it is. The images carry a grid of tenths with
+labels (0.1 … 0.9 across and down): read the position off it, to hundredths.
+
+THE ENTRIES
+${jobs
+        .map((j) => [
+        `### ${j.source} — ${j.title}`,
+        ...j.facts,
+        j.words ? `words: ${j.words}` : undefined,
+        ...j.images.map((i) => `image ${i.media}: ${i.view}`),
+    ]
+        .filter(Boolean)
+        .join("\n"))
+        .join("\n\n")}
+
+THE REPORT
+Write your report to ${report} AS YOU GO — after every entry, not at the end. For each entry and image:
+
+## <S… id> · <M… id>
+result: found | nothing | unclear | many
+region: x,y,w,h
+
+x,y is the top left corner, w,h the width and height — all as fractions of the whole image (0–1).
+The region holds the whole entry: every column of its row or rows, its notes in the margin — rather
+a little generous than cut, and no more of the entries above and below than it has to. An entry
+that runs over to the next page: a block for each image. "nothing" when the entry is not on the
+image, "unclear" when you cannot tell which one it is — never a guess. "many" when it is not one entry
+at all but a search over many entries or pages (a section read through, a whole book, "not found in
+…", a description of the book): no region then — a page picked out of many would show nothing of it.
+The words "result", "found", "nothing", "unclear", "many" and "region" stay in English: strom reads them.
+When all are done, finish with one line: "done: <n> entries".`;
+}
+export function checkPrompt(jobs, report) {
+    return `${RULES}
+
+Each file below is a cut-out of a scan that should show one entry of a register. Open each (one at
+a time, in this order) and check it against what the entry says.
+
+THE CUT-OUTS
+${jobs.map((j) => [`### ${j.source} · ${j.media} — ${j.title}`, j.words ? `words: ${j.words}` : undefined, `cut-out: ${j.view}`].filter(Boolean).join("\n")).join("\n\n")}
+
+THE REPORT
+Write your report to ${report} AS YOU GO. For each cut-out:
+
+## <S… id> · <M… id>
+verdict: ok | cut | wrong
+missing: top, bottom, left, right (only with cut: the sides where a part of the entry is cut off)
+why: a few words
+
+ok: the entry is there whole (a little of its neighbours does not matter). cut: it is there, but a
+part of it is missing — strom cuts it out wider on those sides and asks again. wrong: another entry,
+or none. The words "verdict", "ok", "cut", "wrong", "missing", "top", "bottom", "left" and "right"
+stay in English: strom reads them.
+When all are done, finish with one line: "done: <n> cut-outs".`;
+}
+/** The blocks of a report: "## S0012 · M0034" and its lines. */
+function blocks(text) {
+    const out = [];
+    for (const b of text.split(/^##\s+/m).slice(1)) {
+        const head = b.split("\n")[0] ?? "";
+        const source = /\b(S\d{4,})\b/.exec(head)?.[1];
+        const media = /\b(M\d{4,})\b/.exec(head)?.[1];
+        if (source && media)
+            out.push({ source, media, body: b });
+    }
+    return out;
+}
+export function parseLocated(text) {
+    return blocks(text).map(({ source, media, body }) => {
+        const r = /^result:\s*(\p{L}+)/imu.exec(body)?.[1]?.toLowerCase();
+        const result = r === "found" ? "found" : r === "nothing" ? "nothing" : r === "many" ? "many" : "unclear";
+        const nums = /^region:\s*([\d.]+)\s*[,;\s]\s*([\d.]+)\s*[,;\s]\s*([\d.]+)\s*[,;\s]\s*([\d.]+)/im.exec(body)?.slice(1).map(Number);
+        const ok = nums && nums.every((n) => Number.isFinite(n) && n >= 0 && n <= 1) && nums[2] > 0.005 && nums[3] > 0.003;
+        if (result !== "found" || !ok)
+            return { source, media, result: result === "found" ? "unclear" : result };
+        const [x, y, w, h] = nums;
+        return { source, media, result, region: { x: x, y: y, w: Math.min(w, 1 - x), h: Math.min(h, 1 - y) } };
+    });
+}
+export function parseChecked(text) {
+    return blocks(text).map(({ source, media, body }) => {
+        const v = /^verdict:\s*(\p{L}+)/imu.exec(body)?.[1]?.toLowerCase();
+        const why = /^why:\s*(.+)$/im.exec(body)?.[1]?.trim();
+        const sides = (/^missing:\s*(.+)$/im.exec(body)?.[1] ?? "").toLowerCase().match(/\b(top|bottom|left|right)\b/g);
+        const verdict = v === "ok" ? "ok" : v === "cut" ? "cut" : "wrong";
+        return { source, media, verdict, ...(verdict === "cut" && sides?.length ? { missing: [...new Set(sides)] } : {}), ...(why ? { why } : {}) };
+    });
+}
+/** A region cut out wider where the check found the entry cut off: half as much again on each such side (all round when none was said). */
+export function widen(r, missing) {
+    const sides = missing?.length ? missing : ["top", "bottom", "left", "right"];
+    const dx = Math.max(0.02, r.w * 0.5);
+    const dy = Math.max(0.02, r.h * 0.5);
+    let [x0, y0, x1, y1] = [r.x, r.y, r.x + r.w, r.y + r.h];
+    if (sides.includes("left"))
+        x0 -= dx;
+    if (sides.includes("right"))
+        x1 += dx;
+    if (sides.includes("top"))
+        y0 -= dy;
+    if (sides.includes("bottom"))
+        y1 += dy;
+    const f = (n) => Math.min(1, Math.max(0, n));
+    const k = (n) => Math.round(n * 1e4) / 1e4;
+    [x0, y0, x1, y1] = [f(x0), f(y0), f(x1), f(y1)];
+    return { x: k(x0), y: k(y0), w: k(x1 - x0), h: k(y1 - y0) };
+}

package/dist/core/config.js

@@ -10,9 +10,10 @@ import { readJsonIfExists, writeJson } from "./json.js";
 import { detectLang, isValidLang } from "./lang.js";
 import { UsageError } from "./errors.js";
 import { DEFAULT_AGENT, PROFILES, TIERS } from "../agents/profiles.js";
-import { STRATEGIES } from "./model.js";
+import { EXCERPT_QUALITIES, EXCERPT_SCOPES, EXCERPTS_MAX_MB, STRATEGIES } from "./model.js";
 import { downloadsDir } from "./browser.js";
 import { acquireLock } from "./lock.js";
+import { isStromAppOrigin } from "./stromapp.js";
 export const PERMISSION_LEVELS = ["ask", "auto", "full"];
 /** Earlier names of the levels, still understood. */
 const PERMISSION_ALIASES = { list: "auto", bypass: "full" };
@@ -41,6 +42,10 @@ export const SETTINGS = [
     { key: "queue.strategy", env: "STROM_QUEUE_STRATEGY", tree: true, kind: "choice", choices: STRATEGIES, description: "order of the task queue: balanced (default — nearest ancestors first, spread over the lines, nothing taken forever), depth (stay on one line), priority (strict priority)" },
     { key: "gedcom.for", env: "STROM_GEDCOM_FOR", tree: true, kind: "choice", choices: ["both", "standard", "strom"], description: "GEDCOM files written: both (default), standard (any program), strom (the Strom app)" },
     { key: "stories", env: "STROM_STORIES", tree: true, kind: "choice", choices: ["yes", "no"], description: "stories of the ancestors for the family, written from the facts: yes (default — strom proposes one once a person's life is told by records), no — the user is told when the research starts and may say no" },
+    { key: "main.person", env: "STROM_MAIN_PERSON", tree: true, kind: "person", description: "the main person of the tree (P…): first in the GEDCOM files — the Strom app opens on them (default: the nearest person descended from every research's focus, else the first research's focus)" },
+    { key: "excerpts.quality", env: "STROM_EXCERPTS_QUALITY", tree: true, kind: "choice", choices: EXCERPT_QUALITIES, description: "the entries cut out of their scans in the file for the Strom app (output/tree-strom.ged): small (1000 px, grey — half the size), normal (default, 1200 px), sharp (1600 px)" },
+    { key: "excerpts.for", env: "STROM_EXCERPTS_FOR", tree: true, kind: "choice", choices: EXCERPT_SCOPES, description: "whose entries get their image in the Strom app: none (the file without images), line (the ancestors), family (default: the ancestors and their families), connected (anyone linked to them), all" },
+    { key: "excerpts.mb", env: "STROM_EXCERPTS_MB", tree: true, kind: "number", description: `limit of all the images in one file for the Strom app, MB (default ${EXCERPTS_MAX_MB}): over it they are made smaller, then the farthest from the research left out — strom says so` },
     { key: "strom.version", env: "STROM_APP_VERSION", tree: true, kind: "version", description: "version of the Strom app the Strom GEDCOM is for (set by the app)" },
     { key: "connectors.consent", env: "STROM_CONNECTORS_CONSENT", tree: false, kind: "choice", choices: ["off", "on"], description: "off (default): connectors run without asking — paced by strom, only to their hosts; on: each needs the user's yes, in their terminal, to it and to each archive host" },
     { key: "browser.downloads", env: "STROM_BROWSER_DOWNLOADS", tree: false, kind: "path", description: "the folder your browser saves downloads into (default: the system's Downloads folder) — strom takes a connector's images over from there" },
@@ -49,6 +54,7 @@ export const SETTINGS = [
     { key: "agent.where", env: "STROM_AGENT_WHERE", tree: false, kind: "choice", choices: ["app", "terminal"], description: "where you talk with the agent: app (its desktop app — the easiest), terminal (its CLI) — unset: the app when it is installed" },
     { key: "updates", env: "STROM_UPDATES", tree: false, kind: "choice", choices: ["check", "off"], description: "look for new versions of strom: check (default — at most once a day, one small file from the project's releases; strom says so, strom update installs it) or off" },
     { key: "strom.app", env: "", tree: false, kind: "choice", choices: ["yes", "no"], description: "you use the Strom app: yes (strom says which file to import into it), no (strom never mentions it) — unset: strom notices it itself" },
+    { key: "strom.app.url", env: "STROM_APP_URL", tree: false, kind: "url", description: "another copy of the Strom app to open instead of https://stromapp.info/run/ — its beta (https://beta.stromapp.info/run/), its development (http://127.0.0.1:8080/); installed from a browser, that copy opens as its own app" },
 ];
 /** Fields of the stored settings whose key is not the field name. */
 const FIELDS = {
@@ -62,6 +68,11 @@ const FIELDS = {
     "agent.permissions": "agentPermissions",
     "agent.where": "agentWhere",
     "strom.app": "stromApp",
+    "strom.app.url": "stromAppUrl",
+    "main.person": "mainPerson",
+    "excerpts.quality": "excerptsQuality",
+    "excerpts.for": "excerptsFor",
+    "excerpts.mb": "excerptsMb",
 };
 /** Environment variables that are not settings but steer strom. */
 export const OTHER_ENV = [
@@ -69,7 +80,6 @@ export const OTHER_ENV = [
     { env: "STROM_CONFIG_DIR", description: "where the user config and seal keys live" },
     { env: "STROM_NONINTERACTIVE", description: "1 = never ask, fail with needs-input instead" },
     { env: "STROM_DOCUMENTS", description: "the Documents folder, where strom suggests its home (default: the system's)" },
-    { env: "STROM_APP_URL", description: "another copy of the Strom app to open (its development: http://127.0.0.1:8080/)" },
     { env: "STROM_APP", description: "set by the Strom app when it starts strom (or an agent for it): strom then knows the app is there" },
 ];
 export function settingDef(key) {
@@ -118,6 +128,23 @@ export function checkValue(def, raw, resolvePath) {
                 throw new UsageError(`invalid ${def.key} "${raw}"`, { hint: def.choices?.join(", ") });
             return c;
         }
+        case "person":
+            if (!/^[Pp]\d{4,}$/.test(v))
+                throw new UsageError(`invalid ${def.key} "${raw}"`, { hint: "a person's ID, e.g. P0009 (strom find <name>)" });
+            return v.toUpperCase();
+        case "url": {
+            // A copy of the Strom app the bridge lets in (core/live.ts) — the bridge's address goes to it.
+            let origin = "";
+            try {
+                origin = new URL(v).origin;
+            }
+            catch {
+                // not an address
+            }
+            if (!isStromAppOrigin(origin))
+                throw new UsageError(`invalid ${def.key} "${raw}"`, { hint: "the Strom app: https://stromapp.info/run/, its beta https://beta.stromapp.info/run/, or a copy on this computer http://127.0.0.1:<port>/" });
+            return v;
+        }
         case "version":
             if (!/^\d+(\.\d+){0,2}$/.test(v))
                 throw new UsageError(`invalid ${def.key} "${raw}"`, { hint: "a version like 1.4.0" });
@@ -314,6 +341,19 @@ export class Settings {
         const v = this.resolve("stories", tree)?.value;
         return { on: v !== "no", said: v === "yes" || v === "no" };
     }
+    /** The images for the Strom app: how sharp, whose, and the limit of all together (MB). */
+    excerpts(tree) {
+        return {
+            quality: String(this.resolve("excerpts.quality", tree)?.value ?? "normal"),
+            for: String(this.resolve("excerpts.for", tree)?.value ?? "family"),
+            mb: this.number("excerpts.mb", tree, EXCERPTS_MAX_MB),
+        };
+    }
+    /** The main person of the tree as set (strom config set main.person P…), if anybody did. */
+    mainPerson(tree) {
+        const r = this.resolve("main.person", tree);
+        return r ? String(r.value) : undefined;
+    }
     stromVersion(tree) {
         const r = this.resolve("strom.version", tree);
         return r ? String(r.value) : undefined;

package/dist/core/connector.js

@@ -25,12 +25,19 @@ import { loadUserConfig, Settings } from "./config.js";
 import { readJsonIfExists, writeJson } from "./json.js";
 import { readAsset } from "./assets.js";
 import { foldText } from "./text.js";
-import { CookieJar, hostAllowed, NetError, politeRequest } from "./net.js";
+import { CookieJar, hostAllowed, NetError, paceOf, paceText, politeRequest } from "./net.js";
 import { loginOf, redactor } from "./logins.js";
 import { decodeImage, encodeImage, imageSize, imageSizeOfFile } from "../image/index.js";
 import { blank, paste, toRgb } from "../image/image.js";
 import { botCheck } from "./browser.js";
+/** The newest version of the contract. */
 export const INTERFACE = 1;
+/**
+ * Every version of the contract this strom runs — each one it ever had: a connector someone built stays
+ * working. Within a version the contract only grows (optional fields, capabilities, routes); what a newer
+ * strom added, an older one leaves out and runs the rest (ConnectorManifest.newer).
+ */
+export const INTERFACES = [1];
 export const MANIFEST = "connector.json";
 export const CAPABILITIES = ["find", "list", "fetch", "part", "locate"];
 /** How its images come: through strom's limiter (direct), or through the user's own browser. */
@@ -109,8 +116,24 @@ export function readManifest(dir) {
         throw new UsageError(`${MANIFEST} is not valid JSON: ${err.message}`);
     }
     const problems = [];
-    if (m.interface !== INTERFACE)
+    if (typeof m.interface === "number" && m.interface > Math.max(...INTERFACES))
+        throw new UsageError(`${MANIFEST}: written for version ${m.interface} of the contract — this strom runs ${INTERFACES.join(", ")}`, { hint: "a newer strom runs it: strom update" });
+    if (!INTERFACES.includes(m.interface))
         problems.push(`interface: ${INTERFACE} — the version of the contract it is written for`);
+    // Capabilities and routes of a newer contract: left out, the rest runs (an older strom, a connector built for a newer one).
+    const newer = [];
+    const known = (list, all, what) => {
+        if (!Array.isArray(list) || list.some((x) => typeof x !== "string"))
+            return list;
+        const unknown = list.filter((x) => !all.includes(x) && /^[a-z][a-z-]*$/.test(x));
+        newer.push(...unknown.map((x) => `${what} ${x}`));
+        return list.filter((x) => !unknown.includes(x));
+    };
+    m.can = known(m.can, CAPABILITIES, "can");
+    if (m.routes !== undefined)
+        m.routes = known(m.routes, ROUTES, "route");
+    if (newer.length)
+        m.newer = newer;
     if (!m.title || typeof m.title !== "string")
         problems.push("title: the archive or portal");
     if (!Array.isArray(m.run) || !m.run.length || m.run.some((x) => typeof x !== "string"))
@@ -177,7 +200,7 @@ export function scanConnectors(shared) {
             connectors.push({ name: e.name, dir, manifest: readManifest(dir) });
         }
         catch (err) {
-            broken.push({ name: e.name, dir, problem: err.message });
+            broken.push({ name: e.name, dir, problem: err.message, ...(err instanceof UsageError && err.hint ? { hint: err.hint } : {}) });
         }
     }
     return { connectors, broken };
@@ -195,10 +218,35 @@ export function routeOf(env, c) {
     const routes = routesOf(c);
     return chosen && routes.includes(chosen.via) ? { via: chosen.via, chosen } : { via: routes[0] };
 }
-/** Connectors whose images come through the user's own browser on this computer: the agent gets browser tools for their hosts. */
+/** Connectors whose images come through the user's own browser on this computer. */
 export function browserConnectors(env, shared) {
     return listConnectors(shared).filter((c) => c.manifest.policy.automation !== "manual" && routeOf(env, c).via === "browser");
 }
+/** A site as written anywhere: lower case, without "*." or "www.". */
+function siteOf(host) {
+    return bareHost(host).replace(/^www\./, "");
+}
+/**
+ * The browser connectors a tree works with — the agent gets browser tools for their hosts, and only there: images
+ * fetched through one, or an archive, book or record of the tree on its site. The plugins folder is shared by every
+ * tree; a tree whose research never goes to that archive gets no browser.
+ */
+export function treeBrowserConnectors(tree, shared) {
+    const all = browserConnectors(tree.env, shared);
+    if (!all.length)
+        return [];
+    const fetchedBy = new Set(tree.list("media").flatMap((m) => (m.fetched ? [m.fetched.connector] : [])));
+    const hosts = new Set();
+    for (const r of [...tree.list("repository"), ...tree.list("recordset"), ...tree.list("source")])
+        if (r.url && !r.retracted)
+            try {
+                hosts.add(siteOf(new URL(r.url).hostname));
+            }
+            catch {
+                // not an address
+            }
+    return all.filter((c) => fetchedBy.has(c.name) || c.manifest.hosts.some((h) => hosts.has(siteOf(h))));
+}
 export function findConnector(shared, name) {
     if (!shared)
         throw new StromError("Strom is not set up yet", { hint: "strom setup" });
@@ -208,7 +256,7 @@ export function findConnector(shared, name) {
         return hit;
     const bad = broken.find((b) => b.name === name);
     if (bad)
-        throw new UsageError(`connector ${name} cannot run: ${bad.problem}`, { hint: `fix it in ${bad.dir} — the contract is ${path.join(connectorsDir(shared), "README.md")}` });
+        throw new UsageError(`connector ${name} cannot run: ${bad.problem}`, { hint: bad.hint ?? `fix it in ${bad.dir} — the contract is ${path.join(connectorsDir(shared), "README.md")}` });
     const base = connectorsDir(shared);
     if (/[\\/]/.test(name) || fs.existsSync(path.join(name, MANIFEST)))
         throw new UsageError(`a connector runs from the plugins folder, by its name — not from ${name}`, { hint: `copy the folder into ${base} (or: strom connector add ${name}), then use its folder's name` });
@@ -345,14 +393,14 @@ export function readyConnectors(env, shared, url) {
     });
 }
 /** The warning the user reads before allowing automated access to an archive. */
-export function hostWarning(host, c) {
+export function hostWarning(host, c, pace = paceOf(c?.manifest.policy.pace)) {
     const p = c?.manifest.policy;
     return [
         `Automated access to ${host}${c ? ` (${c.manifest.title}, connector ${c.name})` : ""}`,
         "  · Archives run small servers: going too fast slows the archive for everyone and gets your IP blocked.",
         `  · Terms of use: ${p?.terms || "not found yet — read them on the portal before you allow this"}${p?.termsSummary ? `\n    ${p.termsSummary}` : ""}`,
         p ? `  · The connector's reading of them: automation ${p.automation}${p.officialExport ? `; official export: ${p.officialExport}` : ""}` : undefined,
-        "  · strom sends one request at a time, at least 2 s apart, at most 400 an hour; it stops at the first",
+        `  · strom sends one request at a time, ${paceText(pace)}; it stops at the first`,
         "    refusal (401/403) and leaves the archive alone for a day, and for an hour when it asks twice to",
         "    slow down or stops answering.",
         "  · You are responsible for how you use the archive. Prefer its official export where there is one;",
@@ -582,7 +630,7 @@ export async function runConnector(c, request, opts) {
         report.stopped ??= why;
         child.kill();
     };
-    send({ interface: INTERFACE, ...request, ...(login ? { login: true } : {}) });
+    send({ interface: c.manifest.interface, ...request, ...(login ? { login: true } : {}) });
     const lines = readline.createInterface({ input: child.stdout });
     let queue = Promise.resolve();
     lines.on("line", (line) => {