strom-research 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +13 -5
  2. package/assets/icon/NOTICE.md +26 -0
  3. package/assets/icon/favicon.svg +1 -0
  4. package/assets/icon/icon-256.png +0 -0
  5. package/assets/icon/icon-512.png +0 -0
  6. package/assets/icon/icon.ico +0 -0
  7. package/assets/icon/icon.svg +1 -0
  8. package/assets/lang/cs.json +184 -8
  9. package/assets/lang/de.json +184 -8
  10. package/assets/method/reading.md +4 -2
  11. package/assets/method/recording.md +10 -1
  12. package/assets/plugins/connectors/DISCOVERY.md +6 -2
  13. package/assets/plugins/connectors/README.md +27 -10
  14. package/dist/agents/files.js +27 -18
  15. package/dist/agents/global.js +15 -4
  16. package/dist/brief/brief.js +4 -0
  17. package/dist/cli/context.js +10 -4
  18. package/dist/cli/human.js +116 -0
  19. package/dist/cli/main.js +9 -0
  20. package/dist/cli/menu.js +254 -38
  21. package/dist/cli/ui.js +156 -7
  22. package/dist/cli/wizard.js +36 -9
  23. package/dist/commands/browse.js +243 -0
  24. package/dist/commands/clips.js +190 -0
  25. package/dist/commands/connectors.js +93 -28
  26. package/dist/commands/guide.js +28 -7
  27. package/dist/commands/index.js +3 -0
  28. package/dist/commands/media.js +42 -8
  29. package/dist/commands/meta.js +33 -9
  30. package/dist/commands/output.js +103 -13
  31. package/dist/commands/read.js +1 -1
  32. package/dist/commands/readers.js +62 -0
  33. package/dist/commands/research.js +62 -0
  34. package/dist/commands/session.js +76 -47
  35. package/dist/commands/setup.js +17 -10
  36. package/dist/commands/sources.js +88 -16
  37. package/dist/commands/start.js +57 -37
  38. package/dist/commands/tasks.js +43 -6
  39. package/dist/commands/transcripts.js +177 -0
  40. package/dist/core/chromium.js +60 -19
  41. package/dist/core/clipfinder.js +118 -0
  42. package/dist/core/config.js +42 -2
  43. package/dist/core/connector.js +56 -8
  44. package/dist/core/excerpt.js +219 -0
  45. package/dist/core/frontier.js +30 -5
  46. package/dist/core/git.js +1 -1
  47. package/dist/core/kin.js +90 -0
  48. package/dist/core/live.js +22 -13
  49. package/dist/core/media.js +37 -3
  50. package/dist/core/migrate.js +33 -0
  51. package/dist/core/model.js +7 -0
  52. package/dist/core/net.js +93 -19
  53. package/dist/core/overview.js +259 -0
  54. package/dist/core/phrases.js +27 -0
  55. package/dist/core/places.js +43 -0
  56. package/dist/core/queue.js +16 -3
  57. package/dist/core/review.js +369 -0
  58. package/dist/core/schema.js +4 -0
  59. package/dist/core/session.js +5 -1
  60. package/dist/core/shortcut.js +71 -11
  61. package/dist/core/stories.js +3 -4
  62. package/dist/core/stromapp.js +86 -25
  63. package/dist/core/transcriber.js +99 -0
  64. package/dist/core/tree.js +3 -1
  65. package/dist/core/uninstall.js +4 -2
  66. package/dist/core/update.js +48 -22
  67. package/dist/core/validate.js +3 -1
  68. package/dist/core/workers.js +13 -0
  69. package/dist/gedcom/export.js +58 -30
  70. package/dist/gedcom/lines.js +8 -0
  71. package/dist/gedcom/validate.js +2 -2
  72. package/dist/runners/claude.js +2 -0
  73. package/package.json +1 -1
@@ -14,9 +14,10 @@ import { ui } from "../cli/ui.js";
14
14
  import { register } from "../cli/registry.js";
15
15
  import { lines, moreLine, paginate, runs, shellArg, table } from "../cli/format.js";
16
16
  import { StromError, UsageError } from "../core/errors.js";
17
+ import { isAgent } from "../core/which.js";
17
18
  import { requireRecord } from "../core/records.js";
18
19
  import { readable } from "../core/text.js";
19
- import { findImage, regionText, sameRegion } from "../core/media.js";
20
+ import { findImage, isWhole, regionText, sameRegion } from "../core/media.js";
20
21
  import { partRegion } from "../core/views.js";
21
22
  import { loadLogins, loginOf, loginsFile, removeLogin, saveLogin, VISIBLE_FIELDS } from "../core/logins.js";
22
23
  import { readAsset } from "../core/assets.js";
@@ -24,9 +25,9 @@ import { runGit } from "../core/git.js";
24
25
  import { imageSizeOfFile } from "../image/index.js";
25
26
  import { Tree } from "../core/tree.js";
26
27
  import { syncAgentFiles } from "../agents/files.js";
27
- import { clearBlock, CookieJar, DEFAULT_PACE, hostAllowed, hostState, knownHosts, NetError, paceOf, politeRequest, refusedBy, reserveSlots } from "../core/net.js";
28
+ import { clearBlock, CookieJar, DEFAULT_PACE, hostAllowed, hostPace, hostState, knownHosts, MIN_INTERVAL_MS, NetError, paceText, politeRequest, refusedBy, reserveSlots, setOwnPace } from "../core/net.js";
28
29
  import { botCheck, fileBase, findDownload, loadPage, loadPlans, pageKey, pageScript, parseResult, PLAN_TTL_MS, planScript, readSavedPage, removePlan, savePage, savePlan, SCRIPT_MS, } from "../core/browser.js";
29
- import { bareHost, changedSinceConsent, codeWarning, consentRequired, connectorHash, connectorsDir, directNetwork, ensurePluginsDir, findConnector, hostWarning, imageProblem, INTERFACE, listConnectors, loadConsents, MANIFEST, missingConsents, NAME_RE, readManifest, routeOf, routesOf, ROUTES, runConnector, saveConsents, scanConnectors, suggestName, } from "../core/connector.js";
30
+ import { bareHost, changedSinceConsent, codeWarning, consentRequired, connectorHash, connectorsDir, directNetwork, ensurePluginsDir, findConnector, hostWarning, imageProblem, INTERFACE, listConnectors, loadConsents, MANIFEST, missingConsents, NAME_RE, readManifest, routeOf, treeBrowserConnectors, routesOf, ROUTES, runConnector, saveConsents, scanConnectors, suggestName, } from "../core/connector.js";
30
31
  import { registerImages } from "./media.js";
31
32
  function shared(ctx) {
32
33
  const s = ctx.settings.shared();
@@ -36,6 +37,15 @@ function shared(ctx) {
36
37
  }
37
38
  const netDir = (ctx) => path.join(shared(ctx), "net");
38
39
  const when = (ms) => new Date(ms).toISOString().slice(0, 16).replace("T", " ");
40
+ /** The pace for a host now: the user's own for it, else its connector's service's, else strom's. */
41
+ function paceAt(ctx, host, c) {
42
+ const by = c ?? listConnectors(ctx.settings.shared()?.value).find((x) => x.manifest.hosts.some((h) => hostAllowed(host, [h])));
43
+ return hostPace(hostState(netDir(ctx), host), by?.manifest.policy.pace);
44
+ }
45
+ /** A connector's pace: that of its first host. */
46
+ function connectorPace(ctx, c) {
47
+ return paceAt(ctx, bareHost(c.manifest.hosts[0] ?? ""), c);
48
+ }
39
49
  function hostLine(ctx, host) {
40
50
  const s = hostState(netDir(ctx), host);
41
51
  const hour = s.recent.filter((t) => t > Date.now() - 3600_000).length;
@@ -44,6 +54,8 @@ function hostLine(ctx, host) {
44
54
  s.blockedUntil && s.blockedUntil > Date.now() ? `⛔ refused us — left alone until ${when(s.blockedUntil)} (${s.reason ?? ""})` : "",
45
55
  hour ? `${hour} request(s) in the last hour` : "",
46
56
  (s.slowdown ?? 1) > 1 ? `slowed ×${(s.slowdown ?? 1).toFixed(1)}` : "",
57
+ s.waitUntil && s.waitUntil > Date.now() ? `its limit used up until ${when(s.waitUntil)}` : "",
58
+ `${paceText(paceAt(ctx, host))}${s.own ? " (yours)" : ""}`,
47
59
  ]
48
60
  .filter(Boolean)
49
61
  .join(" · ");
@@ -51,7 +63,7 @@ function hostLine(ctx, host) {
51
63
  /** Ask the user, on a terminal, to allow automated access to a host. */
52
64
  async function askHost(ctx, host, c, byWindow = false) {
53
65
  if (!byWindow) {
54
- ctx.io.stdout("\n" + hostWarning(host, c) + "\n");
66
+ ctx.io.stdout("\n" + hostWarning(host, c, paceAt(ctx, host, c)) + "\n");
55
67
  if (!(await ctx.confirm(`Allow automated access to ${host}?`, false)))
56
68
  return false;
57
69
  }
@@ -131,7 +143,9 @@ function describeRun(r, what) {
131
143
  /** Up to this many books, each comes with the command to register it; more, one line each. */
132
144
  const BOOKS_IN_FULL = 3;
133
145
  const BOOKS_SHOWN = 30;
134
- function bookLines(r, connector, listOne) {
146
+ function bookLines(r, c, listOne) {
147
+ // a connector that fetches nothing (the terms forbid it, or it only finds): the images come by hand
148
+ const fetches = c.manifest.policy.automation !== "manual" && c.manifest.can.some((x) => x === "fetch" || x === "locate");
135
149
  const line = (b) => `${b.title}${b.callNumber ? ` (${b.callNumber})` : ""}${b.years ? ` · ${b.years}` : ""}${b.kinds?.length ? ` · ${b.kinds.join(",")}` : ""}${b.images ? ` · ${b.images} images` : ""}`;
136
150
  if (r.books.length > BOOKS_IN_FULL)
137
151
  return [
@@ -143,7 +157,11 @@ function bookLines(r, connector, listOne) {
143
157
  ` ${line(b)}`,
144
158
  b.url ? ` ${b.url}` : undefined,
145
159
  ` strom recordset add ${shellArg(b.title)}${b.callNumber ? ` --call-number ${shellArg(b.callNumber)}` : ""}${b.kinds?.length ? ` --kinds ${b.kinds.join(",")}` : ""}${b.places?.length ? ` --places ${shellArg(b.places.join(","))}` : ""}${b.years ? ` --years ${b.years}` : ""}${b.url ? ` --url ${shellArg(b.url)}` : ""} --access online-free`,
146
- b.id ? ` then: strom fetch ${connector} ${shellArg(b.id)} --images <from-to> --recordset B…` : undefined,
160
+ !fetches
161
+ ? ` then: the user saves the images by hand — strom task wait T… --images B…:<from-to> --on "<the book, its link, which images>"`
162
+ : b.id
163
+ ? ` then: strom fetch ${c.name} ${shellArg(b.id)} --images <from-to> --recordset B…`
164
+ : undefined,
147
165
  ]
148
166
  .filter(Boolean)
149
167
  .join("\n"));
@@ -161,7 +179,7 @@ function parseImages(v, max = 1000) {
161
179
  if (z < a)
162
180
  throw new UsageError("--images: from before to");
163
181
  if (z - a >= max)
164
- throw new UsageError(`--images: at most ${max} at a time`, { hint: "the hourly cap of an archive: the rest in the next hour" });
182
+ throw new UsageError(`--images: at most ${max} at a time`, { hint: "the rest in the next run (an archive's hourly cap: in the next hour)" });
165
183
  return Array.from({ length: z - a + 1 }, (_, i) => a + i);
166
184
  }
167
185
  /** An archive the tree says must not be automated (forbidden, or browser only). */
@@ -184,8 +202,8 @@ function forbiddenBy(ctx, c) {
184
202
  });
185
203
  }
186
204
  /** How long a fetch takes at least, at the connector's pace (one request per image). */
187
- function estimate(c, images) {
188
- const pace = paceOf(c.manifest.policy.pace);
205
+ function estimate(ctx, c, images) {
206
+ const pace = connectorPace(ctx, c);
189
207
  const secs = Math.round(((images - 1) * pace.minIntervalMs) / 1000);
190
208
  const took = secs < 90 ? `${secs} s` : `${Math.round(secs / 60)} min`;
191
209
  return `${images} image(s) through ${c.name}: at least ${took} at its pace (one request per image, ≥${pace.minIntervalMs / 1000} s apart)`;
@@ -235,9 +253,10 @@ register({
235
253
  const c = findConnector(ctx.settings.shared()?.value, args[0]);
236
254
  const m = c.manifest;
237
255
  const direct = directNetwork(c);
238
- const pace = paceOf(m.policy.pace);
256
+ const pace = connectorPace(ctx, c);
257
+ const own = hostState(netDir(ctx), bareHost(m.hosts[0] ?? "")).own;
239
258
  return {
240
- text: lines(`${c.name}${m.version ? ` ${m.version}` : ""} — ${m.title} (${ctx.display(c.dir)})`, `can ${m.can.join(", ")}`, `runs ${m.run.join(" ")}`, `automation ${m.policy.automation}${m.policy.terms ? ` · terms ${m.policy.terms}` : " · terms not recorded"}`, m.policy.termsSummary ? ` ${m.policy.termsSummary}` : undefined, m.policy.officialExport ? `official ${m.policy.officialExport}` : undefined, `pace ≥${pace.minIntervalMs / 1000} s apart, ≤${pace.perHour}/h`, m.can.includes("fetch") || m.can.includes("locate") ? `images ${routeText(ctx, c)}` : undefined, `consent ${consentState(ctx, c)}`, m.login ? `login ${loginState(ctx, c)} — ${m.login.about}${m.login.url ? ` (${m.login.url})` : ""}` : undefined, direct.length ? `⚠ reaches the network directly (strom cannot pace that):\n${direct.map((d) => ` ${d}`).join("\n")}` : "network only through strom (checked)", "hosts", ...m.hosts.map((h) => ` ${hostLine(ctx, bareHost(h))}`)),
259
+ text: lines(`${c.name}${m.version ? ` ${m.version}` : ""} — ${m.title} (${ctx.display(c.dir)})`, `can ${m.can.join(", ")}`, m.newer ? `newer ${m.newer.join(", ")} — from a newer contract, left out here (strom update runs them)` : undefined, `runs ${m.run.join(" ")}`, `automation ${m.policy.automation}${m.policy.terms ? ` · terms ${m.policy.terms}` : " · terms not recorded"}`, m.policy.termsSummary ? ` ${m.policy.termsSummary}` : undefined, m.policy.officialExport ? `official ${m.policy.officialExport}` : undefined, `pace ${paceText(pace)}${own ? " — yours for the host (strom allow host … --pace auto: the service's again)" : m.policy.pace?.source ? ` — the service's: ${m.policy.pace.source}` : ""}`, m.can.includes("fetch") || m.can.includes("locate") ? `images ${routeText(ctx, c)}` : undefined, `consent ${consentState(ctx, c)}`, m.login ? `login ${loginState(ctx, c)} — ${m.login.about}${m.login.url ? ` (${m.login.url})` : ""}` : undefined, direct.length ? `⚠ reaches the network directly (strom cannot pace that):\n${direct.map((d) => ` ${d}`).join("\n")}` : "network only through strom (checked)", "hosts", ...m.hosts.map((h) => ` ${hostLine(ctx, bareHost(h))}`)),
241
260
  data: { name: c.name, connector: m, dir: c.dir, consent: consentState(ctx, c), ...(m.login ? { login: !!loginOf(ctx.env, c) } : {}), direct, pace, routes: routesOf(c), via: routeOf(ctx.env, c).via },
242
261
  };
243
262
  },
@@ -247,7 +266,8 @@ register({
247
266
  group: "sources",
248
267
  description: "Through the browser, the agent fetches them in your own browser (Claude in Chrome), where your login to the\n" +
249
268
  "portal lives: strom still plans every request, paces it, and takes the files over from your downloads\n" +
250
- "folder, checked and registered. The agent gets browser tools for the connector's sites only, from its next\n" +
269
+ "folder, checked and registered. The agent gets browser tools for the connector's sites only, in the trees that\n" +
270
+ "work with the archive (images fetched through it, or an archive, book or record on its site), from its next\n" +
251
271
  "session on. Your agent may switch it when you ask it to.",
252
272
  args: [{ name: "connector", description: "its name", required: true }],
253
273
  options: [{ name: "via", type: "string", value: "<direct|browser>", description: "direct: strom fetches them · browser: your browser does" }],
@@ -284,7 +304,7 @@ register({
284
304
  const hosts = c.manifest.hosts.map(bareHost);
285
305
  return {
286
306
  text: via === "browser"
287
- ? lines(`${c.name}: images through your browser from now on (${hosts.join(", ")})`, ` · the agent gets browser tools (Claude in Chrome) for these sites only, from its next session${synced.length ? ` (permissions of ${synced.join(", ")} updated)` : ""}. Chrome with the`, " Claude extension must be running then, signed in to the account Claude Code uses; browser tools need a model", " that can review its actions (Sonnet or Opus, not Haiku)", ` · strom plans and paces every request, and takes the files over from ${ctx.display(s.downloads())}`, " (another folder: strom config set browser.downloads <folder>)", ` · the first time, Chrome asks whether ${hosts[0]} may download several files: allow it once`, ` · back: strom connector use ${c.name} --via direct`)
307
+ ? lines(`${c.name}: images through your browser from now on (${hosts.join(", ")})`, ` · the agent gets browser tools (Claude in Chrome) for these sites only, in the family trees that work with this`, ` archive (images fetched through it, or an archive, book or record of theirs on its site), from its next session${synced.length ? ` (permissions of ${synced.join(", ")} updated)` : ""}. Chrome with the`, " Claude extension must be running then, signed in to the account Claude Code uses; browser tools need a model", " that can review its actions (Sonnet or Opus, not Haiku)", ` · strom plans and paces every request, and takes the files over from ${ctx.display(s.downloads())}`, " (another folder: strom config set browser.downloads <folder>)", ` · the first time, Chrome asks whether ${hosts[0]} may download several files: allow it once`, ` · back: strom connector use ${c.name} --via direct`)
288
308
  : lines(`${c.name}: images directly through strom from now on, paced (${hosts.join(", ")})`, routesOf(c).includes("browser") ? ` · through your browser again: strom connector use ${c.name} --via browser` : undefined),
289
309
  data: { connector: c.name, via, hosts, synced },
290
310
  };
@@ -663,7 +683,14 @@ register({
663
683
  const c = findConnector(ctx.settings.shared()?.value, args[0]);
664
684
  if (opts.take)
665
685
  return takeOver(ctx, c, opts.result === undefined ? undefined : String(opts.result));
666
- const perHour = paceOf(c.manifest.policy.pace).perHour;
686
+ // Through the browser, an agent needs browser tools — only a tree that works with this archive gives them.
687
+ if (isAgent(ctx.env) && routeOf(ctx.env, c).via === "browser" && !treeBrowserConnectors(tree, ctx.settings.shared()?.value).some((x) => x.name === c.name)) {
688
+ const site = c.manifest.hosts[0];
689
+ throw new StromError(`this family tree does not work with ${c.manifest.title} yet, so you have no browser tools for ${site}`, {
690
+ hint: `record the archive: strom repo add "${c.manifest.title}" --url https://${site} — the browser tools come with the next conversation or run (tell the user)`,
691
+ });
692
+ }
693
+ const perHour = Math.min(1000, connectorPace(ctx, c).perHour);
667
694
  const wantsPart = opts.crop !== undefined || opts.half !== undefined;
668
695
  if (wantsPart && !c.manifest.can.includes("part"))
669
696
  throw new UsageError(`connector ${c.name} fetches whole images only (it cannot: part)`, {
@@ -726,11 +753,44 @@ register({
726
753
  options: [
727
754
  { name: "revoke", type: "boolean", description: "take the consent back" },
728
755
  { name: "unblock", type: "boolean", description: "lift a refusal (401/403) early — only after the archive said it is fine" },
756
+ { name: "pace", type: "string", value: "<seconds|auto>", description: `your own pause between two requests to the host (at least ${MIN_INTERVAL_MS / 1000} s); auto: the service's again` },
757
+ { name: "per-hour", type: "string", value: "<n|none|auto>", description: "your own hourly cap for the host; none: no cap; auto: the service's again (strom sets none by itself)" },
729
758
  ],
730
- examples: ["strom allow host archive.example.org", "strom allow host archive.example.org --revoke"],
759
+ examples: ["strom allow host archive.example.org", "strom allow host archive.example.org --pace 1 --per-hour none", "strom allow host archive.example.org --pace auto --per-hour auto", "strom allow host archive.example.org --revoke"],
731
760
  run: async (ctx, { args, opts }) => {
732
761
  const host = args[0].toLowerCase().replace(/^https?:\/\//, "").replace(/\/.*$/, "");
733
762
  const lang = ctx.uiLang();
763
+ // The user's own pace for the host: theirs to set — asked in a window when an agent runs it.
764
+ if (opts.pace !== undefined || opts["per-hour"] !== undefined) {
765
+ const was = hostState(netDir(ctx), host).own;
766
+ const own = { ...(was?.minIntervalMs !== undefined ? { minIntervalMs: was.minIntervalMs } : {}), ...(was?.perHour !== undefined ? { perHour: was.perHour } : {}) };
767
+ if (opts.pace !== undefined) {
768
+ const v = String(opts.pace).trim();
769
+ if (v === "auto")
770
+ delete own.minIntervalMs;
771
+ else if (Number(v) * 1000 >= MIN_INTERVAL_MS)
772
+ own.minIntervalMs = Math.round(Number(v) * 1000);
773
+ else
774
+ throw new UsageError(`--pace must be seconds, at least ${MIN_INTERVAL_MS / 1000}, or auto — not "${v}"`);
775
+ }
776
+ if (opts["per-hour"] !== undefined) {
777
+ const v = String(opts["per-hour"]).trim();
778
+ if (v === "auto")
779
+ delete own.perHour;
780
+ else if (v === "none")
781
+ own.perHour = 0;
782
+ else if (Number.isInteger(Number(v)) && Number(v) > 0)
783
+ own.perHour = Number(v);
784
+ else
785
+ throw new UsageError(`--per-hour must be a number, none or auto — not "${v}"`);
786
+ }
787
+ const c = listConnectors(ctx.settings.shared()?.value).find((x) => x.manifest.hosts.some((h) => hostAllowed(host, [h])));
788
+ const next = hostPace({ recent: [], own: { ...own, at: "" } }, c?.manifest.policy.pace);
789
+ ctx.requireHuman(`Set the pace of ${host}: ${paceText(next)}?`, `strom allow host ${host}${opts.pace !== undefined ? ` --pace ${opts.pace}` : ""}${opts["per-hour"] !== undefined ? ` --per-hour ${opts["per-hour"]}` : ""}`, `pace:${host}`, ui(lang, "ui.consent.host.pace", { host, seconds: next.minIntervalMs / 1000, cap: Number.isFinite(next.perHour) ? ui(lang, "ui.pace.cap", { n: next.perHour }) : ui(lang, "ui.pace.nocap") }));
790
+ const s = setOwnPace(netDir(ctx), host, own);
791
+ const now = paceAt(ctx, host, c);
792
+ return { text: `${host}: ${paceText(now)}${s.own ? " — yours" : " — the service's again"}`, data: { host, pace: { minIntervalMs: now.minIntervalMs, perHour: Number.isFinite(now.perHour) ? now.perHour : null }, own: s.own ?? null } };
793
+ }
734
794
  const how = ctx.requireHuman(`${opts.revoke ? "Take back" : opts.unblock ? "Lift the refusal of" : "Allow"} automated access to ${host}?`, `strom allow host ${host}${opts.revoke ? " --revoke" : opts.unblock ? " --unblock" : ""}`, `host:${host}`, ui(lang, opts.revoke ? "ui.consent.host.revoke" : opts.unblock ? "ui.consent.host.unblock" : "ui.consent.host", { host }));
735
795
  const all = loadConsents(ctx.env);
736
796
  if (opts.revoke) {
@@ -759,7 +819,7 @@ register({
759
819
  return {
760
820
  text: lines(consentRequired(ctx.env)
761
821
  ? "consents: on — a connector runs only once you allow it, and each archive host it contacts"
762
- : "consents: off — connectors run without asking (paced by strom, only to their hosts); code that goes round strom needs your yes · to be asked first: strom config set connectors.consent on", "connectors", ...names.map((n) => ` ${n} ${ctx.display(all.connectors[n].dir)} since ${all.connectors[n].at.slice(0, 10)} hosts ${all.connectors[n].hosts.join(", ")}`), names.length ? undefined : " (none)", "hosts (automated access)", ...hosts.map((h) => ` ${hostLine(ctx, h)}${all.hosts[h] ? ` · allowed since ${all.hosts[h].at.slice(0, 10)}` : ""}`), hosts.length ? undefined : " (none)", `pace: one request at a time per host, ≥${DEFAULT_PACE.minIntervalMs / 1000} s apart, ≤${DEFAULT_PACE.perHour}/h; a refusal stops for a day`, Object.keys(loadLogins(ctx.env)).length ? `logins saved (strom login): ${Object.keys(loadLogins(ctx.env)).sort().join(", ")}` : undefined, "take back: strom allow connector <name> --revoke · strom allow host <host> --revoke"),
822
+ : "consents: off — connectors run without asking (paced by strom, only to their hosts); code that goes round strom needs your yes · to be asked first: strom config set connectors.consent on", "connectors", ...names.map((n) => ` ${n} ${ctx.display(all.connectors[n].dir)} since ${all.connectors[n].at.slice(0, 10)} hosts ${all.connectors[n].hosts.join(", ")}`), names.length ? undefined : " (none)", "hosts (automated access)", ...hosts.map((h) => ` ${hostLine(ctx, h)}${all.hosts[h] ? ` · allowed since ${all.hosts[h].at.slice(0, 10)}` : ""}`), hosts.length ? undefined : " (none)", `pace: one request at a time per host — the service's pace (its connector says it), else ≥${DEFAULT_PACE.minIntervalMs / 1000} s apart; an hourly cap only where the service or you set one; slower while a host answers slowly, a wait when it says its limit is used up; a refusal stops for a day · yours for a host: strom allow host <host> --pace <seconds> --per-hour <n>`, Object.keys(loadLogins(ctx.env)).length ? `logins saved (strom login): ${Object.keys(loadLogins(ctx.env)).sort().join(", ")}` : undefined, "take back: strom allow connector <name> --revoke · strom allow host <host> --revoke"),
763
823
  data: { consents: consentRequired(ctx.env) ? "on" : "off", ...all, logins: Object.keys(loadLogins(ctx.env)).sort(), hostState: Object.fromEntries(hosts.map((h) => [h, hostState(netDir(ctx), h)])) },
764
824
  };
765
825
  },
@@ -860,7 +920,7 @@ async function testWith(ctx, c, request, max) {
860
920
  return planPages(ctx, c, r.needs, { cmd: "test", request: { ...request }, max }, `strom connector test ${c.name} (${request.cmd})`, describeRun(r, `test ${request.cmd}`));
861
921
  const direct = directNetwork(c);
862
922
  return {
863
- text: lines(describeRun(r, `test ${request.cmd}`), r.books.length ? `books (${r.books.length}):\n${bookLines(r, c.name, `strom connector test ${c.name} --list <id>`).join("\n")}` : undefined, r.images.length ? `images (${r.images.length}): ${r.images.map((i) => `${i.n ?? "?"}${i.region ? ` part ${regionText(i.region)}` : ""}=${path.basename(i.file)} ${fs.statSync(i.file).size} B${pixels(i.file)}`).join(", ")} in ${ctx.display(workDir)}` : undefined, r.located.length ? `located (${r.located.length}):\n${r.located.map((l) => ` ${l.n}${l.region ? ` part ${regionText(l.region)}` : ""} ${l.src}${l.url && l.url !== l.src ? `\n page ${l.url}` : ""}`).join("\n")}` : undefined, c.manifest.login ? (loginOf(ctx.env, c) ? "with the user's login" : `without a login${c.manifest.login.required ? "" : " (the user may save one: strom login " + c.name + ")"}`) : undefined, direct.length ? `⚠ the code reaches the network directly — strom cannot pace that:\n${direct.map((d) => ` ${d}`).join("\n")}` : undefined, c.manifest.policy.automation === "unknown" ? "⚠ policy.automation is unknown — find out what the portal's terms say (DISCOVERY.md, step 1)" : undefined),
923
+ text: lines(describeRun(r, `test ${request.cmd}`), r.books.length ? `books (${r.books.length}):\n${bookLines(r, c, `strom connector test ${c.name} --list <id>`).join("\n")}` : undefined, r.images.length ? `images (${r.images.length}): ${r.images.map((i) => `${i.n ?? "?"}${i.region ? ` part ${regionText(i.region)}` : ""}=${path.basename(i.file)} ${fs.statSync(i.file).size} B${pixels(i.file)}`).join(", ")} in ${ctx.display(workDir)}` : undefined, r.located.length ? `located (${r.located.length}):\n${r.located.map((l) => ` ${l.n}${l.region ? ` part ${regionText(l.region)}` : ""} ${l.src}${l.url && l.url !== l.src ? `\n page ${l.url}` : ""}`).join("\n")}` : undefined, c.manifest.login ? (loginOf(ctx.env, c) ? "with the user's login" : `without a login${c.manifest.login.required ? "" : " (the user may save one: strom login " + c.name + ")"}`) : undefined, direct.length ? `⚠ the code reaches the network directly — strom cannot pace that:\n${direct.map((d) => ` ${d}`).join("\n")}` : undefined, c.manifest.policy.automation === "unknown" ? "⚠ policy.automation is unknown — find out what the portal's terms say (DISCOVERY.md, step 1)" : undefined),
864
924
  data: { ...r, direct },
865
925
  ...(r.stopped && !r.books.length && !r.images.length && !r.located.length ? { exitCode: 1 } : {}),
866
926
  };
@@ -901,7 +961,7 @@ function pagesOf(ctx, c) {
901
961
  function planPages(ctx, c, want, resume, what, before) {
902
962
  const tree = ctx.tree();
903
963
  const u = new URL(want.url);
904
- const pace = paceOf(c.manifest.policy.pace);
964
+ const pace = paceAt(ctx, u.hostname, c);
905
965
  let times;
906
966
  try {
907
967
  // one pause ahead: time to open the tab
@@ -1063,7 +1123,7 @@ async function fetchWith(ctx, c, request, recordset) {
1063
1123
  if (!(await ensureLogin(ctx, c)))
1064
1124
  return { text: "no login — nothing was fetched", exitCode: 1 };
1065
1125
  if (request.cmd === "fetch")
1066
- ctx.io.stderr(estimate(c, request.images.length) + "\n");
1126
+ ctx.io.stderr(estimate(ctx, c, request.images.length) + "\n");
1067
1127
  if (request.cmd === "fetch" && c.manifest.policy.automation === "unknown")
1068
1128
  ctx.io.stderr(`note: what the terms of ${c.manifest.title} say about automated download is not known yet — strom connector show ${c.name}\n`);
1069
1129
  const workDir = path.join(tree.root, ".strom", "fetch", `${c.name}-${Date.now()}`);
@@ -1078,7 +1138,7 @@ async function fetchWith(ctx, c, request, recordset) {
1078
1138
  if (request.cmd === "find" || request.cmd === "list") {
1079
1139
  fs.rmSync(workDir, { recursive: true, force: true });
1080
1140
  return {
1081
- text: lines(describeRun(r, `${request.cmd} (${took})`), r.books.length ? [`books (${r.books.length}):`, ...bookLines(r, c.name, `strom fetch ${c.name} <id> --list`)].join("\n") : "no books found"),
1141
+ text: lines(describeRun(r, `${request.cmd} (${took})`), r.books.length ? [`books (${r.books.length}):`, ...bookLines(r, c, `strom fetch ${c.name} <id> --list`)].join("\n") : "no books found"),
1082
1142
  data: r,
1083
1143
  ...(r.stopped && !r.books.length ? { exitCode: 1 } : {}),
1084
1144
  };
@@ -1095,9 +1155,11 @@ async function fetchWith(ctx, c, request, recordset) {
1095
1155
  const gain = m?.width && m.part && whole?.width ? m.width / m.part.w / whole.width : undefined;
1096
1156
  text = m
1097
1157
  ? lines(`part ${regionText(m.part)} of image ${request.image} of ${recordset} fetched and registered: ${m.id}${m.width ? ` · ${m.width}×${m.height} px` : ""}${gain ? ` · ${gain.toFixed(1)}× the detail of the whole image` : ""} · ${took}`, gain !== undefined && gain < 1.2 ? " no sharper than the whole image: the portal gives no more detail than that" : undefined, `look at it: strom media view ${recordset}:${request.image} --crop ${regionText(m.part)} (a view of the image uses the part by itself)`)
1098
- : res.again.length
1099
- ? lines(`that part is registered already: ${res.again.join(" ")}`, ...res.clashes.map((x) => `⚠ ${x}`))
1100
- : "no part fetched";
1158
+ : res.again.length && whole && res.again.includes(whole.id) && r.images.every((i) => isWhole(i.region))
1159
+ ? lines(`the portal's sharpest of image ${request.image} is the whole scan, registered already: ${whole.id}${whole.width ? ` · ${whole.width}×${whole.height} px` : ""} — no part needed`, `look at it: strom media view ${recordset}:${request.image} --crop ${regionText(request.region)}`)
1160
+ : res.again.length
1161
+ ? lines(`that part is registered already: ${res.again.join(" ")}`, ...res.clashes.map((x) => `⚠ ${x}`))
1162
+ : "no part fetched";
1101
1163
  data = { ...data, added: res.added.map((x) => ({ id: x.id, image: x.image, part: x.part })), again: res.again };
1102
1164
  }
1103
1165
  else if (recordset && r.images.length) {
@@ -1130,7 +1192,7 @@ async function planBrowser(ctx, c, request, recordset) {
1130
1192
  throw new UsageError(`connector ${c.name} cannot say where its images are (can: locate) — the browser has no addresses to fetch`, { hint: `strom connector use ${c.name} --via direct` });
1131
1193
  if (!(await ensureAllowed(ctx, c)))
1132
1194
  return { text: "not allowed — nothing was planned", exitCode: 1 };
1133
- const pace = paceOf(c.manifest.policy.pace);
1195
+ const pace = connectorPace(ctx, c);
1134
1196
  const part = request.cmd === "part";
1135
1197
  const asked = part ? [request.image] : request.images;
1136
1198
  // one script waits at most SCRIPT_MS: as many images as fit in it, the rest next time
@@ -1187,15 +1249,18 @@ async function planBrowser(ctx, c, request, recordset) {
1187
1249
  async function takeOver(ctx, c, result) {
1188
1250
  const tree = ctx.tree();
1189
1251
  const all = loadPlans(tree.root, c.name);
1190
- // pages for the connector first: what they were planned for goes on
1252
+ const dir = ctx.settings.downloads();
1253
+ const got = parseResult(result ?? "");
1254
+ // pages for the connector first: what they were planned for goes on — unless the browser saved images
1255
+ // and none of those pages (a page planned in a run given up must not hold back what did arrive)
1191
1256
  const pagePlans = all.filter((p) => p.pages?.length);
1192
- if (pagePlans.length && !tree.dryRun)
1257
+ const pagesHere = got.some((g) => g.page) || pagePlans.some((p) => p.pages.some((pg) => findDownload(dir, pg.file).file));
1258
+ const imagesHere = all.some((p) => p.items.some((i) => findDownload(dir, i.file).file));
1259
+ if (pagePlans.length && !tree.dryRun && (pagesHere || !imagesHere))
1193
1260
  return takePages(ctx, c, pagePlans, result);
1194
1261
  const plans = all.filter((p) => p.items.length);
1195
1262
  if (!plans.length)
1196
1263
  return { text: `nothing of ${c.name} waits to be taken over — plan it first: strom fetch ${c.name} <book> --images <from-to> --recordset B…`, data: { taken: [] } };
1197
- const dir = ctx.settings.downloads();
1198
- const got = parseResult(result ?? "");
1199
1264
  if (tree.dryRun) {
1200
1265
  const found = plans.flatMap((p) => p.items.filter((i) => findDownload(dir, i.file).file));
1201
1266
  return { text: `dry run: would take over ${found.length} of ${plans.reduce((n, p) => n + p.items.length, 0)} planned image(s) from ${ctx.display(dir)} — nothing was changed`, data: { dryRun: true, found: found.map((i) => i.n) } };
@@ -53,7 +53,7 @@ EVERY SESSION
53
53
 
54
54
  RECORDING
55
55
  A record (register entry, certificate) — the source first, then its facts:
56
- strom source add "Křest Jana Nováka 1885" --kind baptism --recordset B0001 --locator "fol. 12, č. 3" --form original --information primary --transcript @zapis.txt
56
+ strom source add "Křest Jana Nováka 1885" --kind baptism --recordset B0001 --clip B0001:12@0.05,0.40,0.45,0.18 --locator "fol. 12, č. 3" --form original --information primary --transcript @zapis.txt
57
57
  strom person add "Jan /Novák/" --sex M --born "24 JUN 1885" --born-place "Kamenice nad Lipou" --cite S0001
58
58
  strom family add --partner P0002 --partner P0003 --child P0001 the baptism naming the parents proves the link
59
59
  strom event add P0001 CHR --date "25 JUN 1885" --cite S0001 --locator "fol. 12" --with "godparent:Marie Dvořáková" --status proven
@@ -102,6 +102,10 @@ SEARCHING
102
102
  strom search add "Křty Novák 1880-1890" --recordset B0001 --years 1880-1890 --surname Novák --method page-by-page --result negative
103
103
  strom search edit Q0001 --recordset B0002 --reason "…" a search recorded wrongly is corrected, never added again
104
104
  strom repo add … · strom recordset add … · strom place jurisdiction … where the records are
105
+ strom place add "<village>" --kind village --lat … --lon … the places of the facts, on the Strom app's map —
106
+ identify the place first (district, parish, today's name: of several
107
+ villages of a name, the one the records mean); not sure: --unlocated "<why>".
108
+ session close and strom place list --off-map say which are still off it
105
109
  strom connector list connectors: they find books and fetch scans
106
110
  strom fetch <connector> --find "<place>" --years 1780-1850
107
111
  strom fetch <connector> <book> --images 40-69 --recordset B0001 registered at once
@@ -112,12 +116,23 @@ SEARCHING
112
116
  FINDING YOUR WAY
113
117
  strom research show G0001 · strom person show P0001 · strom family show F0001
114
118
  strom gaps · strom find <text> [<text>…] · strom frontier · strom task list
119
+ for the user, in the research language: strom stats (how far the research got) · strom recent
120
+ (what came in lately) · strom plan (what you do next) · strom pedigree [P…] (the ancestors as a
121
+ tree) · strom person card P… (one life)
122
+ ONE PERSON, LOOKED AT AGAIN (the user asks you to check or complete someone; a new model reads better)
123
+ strom review P… [--scope family|line] [--reread] — strom proposes tasks: what records, notes and the
124
+ diary say of them outside their data, entries whose images are here but were not read whole, facts
125
+ resting on one reading, conflicts left open (--reread: a second reading with the model the user reads
126
+ with now). Each task's brief lists its items. Tell the user how many and what they cost; they decide.
115
127
  STORIES OF THE ANCESTORS (the setting stories — on by default; strom shows it)
116
128
  Once records tell a person's life (a baptism and more facts from records), strom proposes a narrate
117
129
  task: write the story for the family book — plain words, the research language, every statement on a
118
130
  recorded fact (strom story set P… --text @notes/story-P….md --fact E… …). More facts later: it
119
- proposes adding to it. A story is a draft until the user approves it (--final). Not told yet: when the
120
- research starts, tell the user in a sentence, and that they may say no (strom config set stories no).
131
+ proposes adding to it. Stories wait behind all the research; working alone
132
+ (strom run), every few sessions one comes first. In a conversation the user leads: write one when they
133
+ want it, or offer it when the research has a pause. A story is a draft until the user approves it (--final). When strom says to tell
134
+ the user (once, not in every conversation): in a sentence, and that they may say no (strom config set
135
+ stories no).
121
136
 
122
137
  The result: strom export gedcom writes output/tree.ged (standard GEDCOM for any program) and
123
138
  output/tree-strom.ged (for the Strom app) — both from the same evidence, also on session close.
@@ -126,8 +141,8 @@ Changes are committed automatically; you never run git yourself.
126
141
  THE STROM APP — where the user sees the result
127
142
  The Strom app (https://stromapp.info) is strom's companion: a free family tree app, no account,
128
143
  the family's data stay on their computer; it shows the tree, the sources, a map, a family book.
129
- When the user wants to see the results (or once, when the first ones are there), suggest it
130
- gently, in a sentence or two — best installed as an app from the browser, then it works offline:
144
+ When the user wants to see the results, or when strom says to offer it (once, not in every
145
+ conversation), suggest it gently, in a sentence or two — best installed as an app from the browser, then it works offline:
131
146
  https://stromapp.info/run/ the app itself: opened in the browser, installed from there (the
132
147
  install icon at the end of the address bar; Safari: File → Add to Dock)
133
148
  strom app install opens it there and says where to click
@@ -135,9 +150,15 @@ THE STROM APP — where the user sees the result
135
150
  run by you, the app follows the research live: what you record shows there
136
151
  by itself — the best way to watch it grow; otherwise in the app: Import,
137
152
  and output/tree-strom.ged
153
+ Each entry comes with its image, cut out of its scan (the sources' clips —
154
+ give --clip when you record an entry); sources without a clip (read earlier,
155
+ or from an older research) — when the user wants them too: strom clips
156
+ --dry-run (how many), then strom clips (readers find and check them); their
157
+ words, when a clip has none: strom transcripts (read, then checked)
138
158
  Without the app, the user can simply ask you about anyone in the tree, a family, a line, what is
139
- proven and by which record: answer from strom (person show, family show, research show, find,
140
- source show, story show, gaps, frontier) in plain words — names, dates, places, no IDs.
159
+ proven and by which record, what is new or next: answer from strom (person card, pedigree, stats,
160
+ recent, plan, person show, family show, research show, find, source show, story show, gaps,
161
+ frontier) in plain words — names, dates, places, no IDs.
141
162
  strom (no arguments) says whether it is installed here (results): not yet — offer to install it; installed —
142
163
  just open it. A program they already use is fine too: output/tree.ged. They do not want it:
143
164
  never again.
@@ -11,7 +11,10 @@ import "./sources.js";
11
11
  import "./media.js";
12
12
  import "./connectors.js";
13
13
  import "./read.js";
14
+ import "./clips.js";
15
+ import "./transcripts.js";
14
16
  import "./analysis.js";
17
+ import "./browse.js";
15
18
  import "./story.js";
16
19
  import "./output.js";
17
20
  import "./session.js";
@@ -6,22 +6,24 @@
6
6
  import fs from "node:fs";
7
7
  import path from "node:path";
8
8
  import { register } from "../cli/registry.js";
9
- import { lines, moreLine, paginate, runs, shellArg, table } from "../cli/format.js";
9
+ import { lines, moreLine, paginate, runs, shellArg, table, truncate } from "../cli/format.js";
10
10
  import { UsageError } from "../core/errors.js";
11
11
  import { makeNote } from "../core/actions.js";
12
- import { collectFiles, fileSha256, findImage, imageNumbers, imageOfRef, inboxFolders, inputPath, mimeOf, otherCopies, parseImageList, regionText, sharperPart, storeShared } from "../core/media.js";
12
+ import { clipText, collectFiles, fileSha256, findImage, imageNumbers, imageOfRef, inboxFolders, inputPath, isWhole, mimeOf, otherCopies, parseImageList, regionText, sharperPart, storeShared } from "../core/media.js";
13
13
  import { create, normId, requireRecord, update } from "../core/records.js";
14
14
  import { imageSizeOfFile } from "../image/index.js";
15
15
  import { imageOf, pageOf } from "../core/calibration.js";
16
16
  import { describeView, makeView, partRegion, viewRegion, VIEW_MAX } from "../core/views.js";
17
17
  import { listConnectors, missingConsents } from "../core/connector.js";
18
+ import { now } from "../core/tree.js";
18
19
  const IMAGE_EXT = new Set([".jpg", ".jpeg", ".png", ".tif", ".tiff", ".gif", ".webp", ".heic", ".jp2", ".bmp"]);
19
20
  /**
20
21
  * Register images of a record set: stored once by content, each with its number
21
22
  * and provenance. Tasks that waited for images of the book go back into the queue.
22
23
  */
23
24
  export function registerImages(tree, shared, items, recordset) {
24
- const known = new Map(tree.list("media").map((m) => [m.sha, m]));
25
+ // a withdrawn image is no longer known: its file can be registered again, where it belongs
26
+ const known = new Map(tree.list("media").filter((m) => !m.retracted).map((m) => [m.sha, m]));
25
27
  const added = [];
26
28
  const again = [];
27
29
  const clashes = [];
@@ -34,7 +36,9 @@ export function registerImages(tree, shared, items, recordset) {
34
36
  again.push(k.id);
35
37
  // the same scan as another image: a portal serving another book's images, or a file given the wrong number
36
38
  const as = (m) => `${m.recordset ?? "no record set"}${m.image !== undefined ? `:${m.image}` : ""}${m.part ? ` part ${regionText(m.part)}` : ""}`;
37
- if (as(k) !== as({ recordset, image: it.image, part: it.part }))
39
+ // a portal whose sharpest part is the whole scan gives the image again: that is no clash
40
+ const sameScan = k.recordset === recordset && k.image === it.image && isWhole(k.part) && isWhole(it.part);
41
+ if (!sameScan && as(k) !== as({ recordset, image: it.image, part: it.part }))
38
42
  clashes.push(`${as({ recordset, image: it.image, part: it.part })} is the same file as ${k.id} (${as(k)}) — not registered again: the portal may have given another book's image, or a number is wrong; check before citing either`);
39
43
  continue;
40
44
  }
@@ -322,16 +326,40 @@ register({
322
326
  const sources = tree.list("source").filter((s) => s.media?.includes(m.id));
323
327
  const b = m.recordset ? tree.get(m.recordset) : undefined;
324
328
  const all = tree.list("media");
325
- const parts = !m.part && m.recordset && m.image !== undefined ? all.filter((x) => x.part && x.recordset === m.recordset && x.image === m.image) : [];
329
+ const parts = !m.part && m.recordset && m.image !== undefined ? all.filter((x) => x.part && !x.retracted && x.recordset === m.recordset && x.image === m.image) : [];
326
330
  const copies = otherCopies(all, m);
327
331
  const image = copies.length && m.recordset && m.image !== undefined ? findImage(all, m.recordset, m.image) : undefined;
328
332
  return {
329
- text: lines(`${m.id} ${m.part ? `part ${regionText(m.part)} of ` : ""}${b ? `image ${m.image ?? "?"} of ${b.id} ${b.title}` : "image"}${pageLabel(tree, m) ? ` · page ${pageLabel(tree, m)}` : ""}`, `${m.width ? `${m.width}×${m.height} px · ` : ""}${Math.round(m.size / 1024)} kB · ${m.mime}${m.from ? ` · from ${m.from}` : ""}${m.url ? ` · ${m.url}` : ""}`, parts.length ? `parts of it, sharper (a view of the image uses them by itself): ${parts.map((p) => `${p.id} ${regionText(p.part)}${p.width ? ` ${p.width}×${p.height}` : ""}`).join(" · ")}` : undefined, copies.length
333
+ text: lines(`${m.id} ${m.part ? `part ${regionText(m.part)} of ` : ""}${b ? `image ${m.image ?? "?"} of ${b.id} ${b.title}` : "image"}${pageLabel(tree, m) ? ` · page ${pageLabel(tree, m)}` : ""}`, `${m.width ? `${m.width}×${m.height} px · ` : ""}${Math.round(m.size / 1024)} kB · ${m.mime}${m.from ? ` · from ${m.from}` : ""}${m.url ? ` · ${m.url}` : ""}`, m.retracted ? `RETRACTED ${m.retracted.at.slice(0, 10)}: ${m.retracted.reason} — no view, part or excerpt uses it` : undefined, parts.length ? `parts of it, sharper (a view of the image uses them by itself): ${parts.map((p) => `${p.id} ${regionText(p.part)}${p.width ? ` ${p.width}×${p.height}` : ""}`).join(" · ")}` : undefined, copies.length
330
334
  ? `other copies of the image: ${copies.map((c) => `${c.id}${c.width ? ` ${c.width}×${c.height}` : ""}`).join(" · ")} — the image is the sharpest (${image.id}); a view of any copy uses it`
331
335
  : undefined, sources.length ? `sources on it: ${sources.map((s) => `${s.id} ${s.title}`).join(" · ")}` : "no source cites it yet", `view: strom media view ${m.id} [--half left|right] [--crop x,y,w,h] [--grid]`),
332
336
  data: { media: m, parts: parts.map((p) => p.id), copies: copies.map((c) => c.id), sources: sources.map((s) => s.id), page: pageLabel(tree, m) || undefined },
333
337
  };
334
338
  },
339
+ }, {
340
+ path: ["media", "retract"],
341
+ summary: "Withdraw an image registered wrongly — another book's, a wrong number, a part put in the wrong place (kept, marked retracted)",
342
+ group: "sources",
343
+ tree: true,
344
+ writes: true,
345
+ description: "The image stays in the research with the reason, but no view, sharper part or excerpt uses it any more, and its\n" +
346
+ "file can be registered again where it belongs. An image a source still cites is not withdrawn: move the\n" +
347
+ "source to the right image first (strom source edit S… --clip …, --clip-remove).",
348
+ args: [{ name: "image", description: "M0012", required: true }],
349
+ examples: ['strom media retract M0001 --reason "a part of another book\'s image 340, not of this one"'],
350
+ run(ctx, { args, opts }) {
351
+ const tree = ctx.tree();
352
+ if (!opts.reason)
353
+ throw new UsageError("--reason is required", { hint: 'say why: --reason "another book\'s image, same number"' });
354
+ const m = requireRecord(tree, normId(args[0]), "media");
355
+ if (m.retracted)
356
+ return { text: `${m.id} is already retracted: ${m.retracted.reason}`, data: { media: m } };
357
+ const citing = tree.list("source").filter((s) => !s.retracted && (s.media?.includes(m.id) || s.clips?.some((c) => c.media === m.id)));
358
+ if (citing.length)
359
+ throw new UsageError(`${m.id} is cited by ${citing.map((s) => s.id).join(", ")}`, { hint: `move them to the right image first: strom source show ${citing[0].id}` });
360
+ const out = update(tree, m.id, "media", (x) => ({ ...x, retracted: { at: now(), reason: String(opts.reason) } }), { op: "media.retract", summary: `${m.id} retracted: ${truncate(String(opts.reason), 80)}` });
361
+ return { text: lines(...tree.written.map((o) => o.summary)), data: { media: out } };
362
+ },
335
363
  }, {
336
364
  path: ["media", "view"],
337
365
  summary: "Make a view of an image to look at: a crop, a half page, enlarged, more contrast, a grid",
@@ -396,11 +424,17 @@ register({
396
424
  const fetchPart = canPart && v.scale > 1.25 && inWhole.w * inWhole.h < 0.9
397
425
  ? `strom fetch ${from.name} --recordset ${whole.recordset} --images ${whole.image} --crop ${regionText(inWhole)}`
398
426
  : undefined;
427
+ // The same part as a clip of the source read in it: of the whole image where it is registered.
428
+ const clip = !t.media || (!spec.crop && !spec.half)
429
+ ? undefined
430
+ : whole && !whole.part
431
+ ? clipText({ media: whole.id, region: inWhole })
432
+ : clipText({ media: t.key, region: { x: v.region.x / v.original.width, y: v.region.y / v.original.height, w: v.region.w / v.original.width, h: v.region.h / v.original.height } });
399
433
  return {
400
- text: lines(describeView(v, (p) => ctx.display(p), fetchPart), sharper
434
+ text: lines(describeView(v, (p) => ctx.display(p), fetchPart), clip ? `the source of an entry read here: --clip ${clip}` : undefined, sharper
401
435
  ? `from ${sharper.part.id}, ${sharper.part.part ? "a part of the image fetched sharper" : "a sharper copy of the image"} (${sharper.gain.toFixed(1)}× the detail of ${sharper.part.part ? "the whole image" : t.media.id})`
402
436
  : undefined, page ? `page ${page}` : undefined),
403
- data: { view: v.file, width: v.width, height: v.height, scale: v.scale, region: v.region, original: v.original, image: t.key, ...(sharper ? { from: sharper.part.id } : {}), page: page || undefined },
437
+ data: { view: v.file, width: v.width, height: v.height, scale: v.scale, region: v.region, original: v.original, image: t.key, ...(sharper ? { from: sharper.part.id } : {}), page: page || undefined, ...(clip ? { clip } : {}) },
404
438
  };
405
439
  },
406
440
  });
@@ -17,10 +17,11 @@ import { gitVersion } from "../core/git.js";
17
17
  import { linuxGitCommand } from "../core/deps.js";
18
18
  import { verifyFast } from "../core/integrity.js";
19
19
  import { currentSession, openSessions } from "../core/session.js";
20
+ import { readByOtherModels } from "../core/review.js";
20
21
  import { liveHolder } from "../core/lock.js";
21
22
  import { isAgent } from "../core/which.js";
22
23
  import { taskQueue, waitingForUser, waitingLines } from "./tasks.js";
23
- import { liveWorkers } from "../core/workers.js";
24
+ import { liveWorkers, runAlive } from "../core/workers.js";
24
25
  import { label } from "../core/people.js";
25
26
  function orientation(ctx) {
26
27
  const lang = ctx.uiLang();
@@ -81,10 +82,33 @@ function orientation(ctx) {
81
82
  const out = path.join(tree.root, "output");
82
83
  const wanted = ctx.settings.config.stromApp !== "no";
83
84
  const file = path.join(out, wanted ? "tree-strom.ged" : "tree.ged");
85
+ // What a conversation tells the user once, not again after it starts afresh (a new conversation, /clear): the Strom app, the stories, asking here.
86
+ const told = ctx.settings.config.told ?? {};
84
87
  if (fs.existsSync(file))
85
- base.results = { file, app: !wanted ? "not-wanted" : installedStromApp(ctx.env) ? "installed" : "offer" };
88
+ base.results = { file, app: !wanted ? "not-wanted" : installedStromApp(ctx.env, process.platform, stromAppUrl(ctx.settings)) ? "installed" : "offer", ...(told.app ? { told: true } : {}) };
89
+ const vision = ctx.settings.models(ctx.settings.agent(tree.config).value, tree.config).vision;
90
+ if (vision) {
91
+ const other = readByOtherModels(tree, vision);
92
+ if (other.facts)
93
+ base.reread = { ...other, model: vision };
94
+ }
86
95
  const stories = ctx.settings.stories(tree.config);
87
- base.stories = !stories.on ? "off" : stories.said ? "on" : "tell";
96
+ base.stories = !stories.on ? "off" : stories.said || told.stories ? "on" : "tell";
97
+ if (!told.ask && base.tree.persons > 0)
98
+ base.ask = true;
99
+ // Given to an agent in a conversation (not a run nobody watches, not a person): told now.
100
+ if (isAgent(ctx.env) && !ctx.env.STROM_SESSION) {
101
+ const now = new Date().toISOString();
102
+ const fresh = {
103
+ ...(base.stories === "tell" ? { stories: now } : {}),
104
+ ...(base.results && base.results.app !== "not-wanted" && !base.results.told ? { app: now } : {}),
105
+ ...(base.ask ? { ask: now } : {}),
106
+ };
107
+ if (Object.keys(fresh).length) {
108
+ ctx.settings.config.told = { ...told, ...fresh };
109
+ ctx.settings.save();
110
+ }
111
+ }
88
112
  // The session this agent works in; else one nobody is working in any more.
89
113
  const busy = (s) => (s.worker && workers.some((w) => w.id === s.worker)) || (s.runner && workers.some((w) => w.id === "run"));
90
114
  const open = currentSession(tree, ctx.env) ?? sessions.find((s) => !busy(s) && !(s.worker && s.worker !== ctx.env.STROM_WORKER && !human));
@@ -98,7 +122,7 @@ function orientation(ctx) {
98
122
  base.next = { why: t("ui.why.check"), command: "strom check" };
99
123
  else if (elsewhere)
100
124
  base.next = { why: t("ui.why.handover"), command: "strom chat" };
101
- else if (open && human && open.runner && !liveHolder(path.join(tree.root, ".strom", "run.lock")))
125
+ else if (open && human && open.runner && !runAlive(tree.root, open))
102
126
  base.next = { why: t("ui.why.runleft", { id: open.id }), command: "strom run" };
103
127
  else if (open && human && open.runner)
104
128
  base.next = { why: t("ui.why.running", { id: open.id }), command: "strom status" };
@@ -132,18 +156,18 @@ register({
132
156
  const lang = ctx.uiLang();
133
157
  const t = (key, values = {}) => ui(lang, key, values);
134
158
  // Labels in one column, whatever their length in the user's language.
135
- const labels = [t("ui.o.home"), t("ui.o.tree"), t("ui.o.lang"), t("ui.o.data"), t("ui.o.results"), t("ui.o.stories"), t("ui.o.update"), t("ui.o.next")];
159
+ const labels = [t("ui.o.home"), t("ui.o.tree"), t("ui.o.lang"), t("ui.o.data"), t("ui.o.results"), t("ui.o.stories"), t("ui.o.ask"), t("ui.o.reread"), t("ui.o.update"), t("ui.o.next")];
136
160
  const width = Math.max(...labels.map((l) => l.length)) + 2;
137
161
  const row = (label, value) => `${label.padEnd(width)}${value}`;
138
162
  const check = o.tree ? `${o.tree.errors ? t("ui.o.errors", { n: o.tree.errors }) : t("ui.o.ok")}${o.tree.warnings ? `, ${t("ui.o.warnings", { n: o.tree.warnings })}` : ""}` : "";
139
163
  const text = lines(t("ui.o.title", { version: o.version }), t("ui.o.rule"), "", row(t("ui.o.home"), o.home ? ctx.display(o.home) : t("ui.o.notsetup")), o.update ? row(t("ui.o.update"), t("ui.o.update.new", { version: o.update, current: o.version })) : undefined, o.tree
140
164
  ? lines(row(t("ui.o.tree"), `${o.tree.name} (${ctx.display(o.tree.root)})`), row(t("ui.o.lang"), `${langName(o.tree.lang, lang)} (${o.tree.lang}) — ${t("ui.o.langnote")}`), row(t("ui.o.data"), t("ui.o.counts", { persons: o.tree.persons, families: o.tree.families, check })), o.results
141
165
  ? row(t("ui.o.results"), t(o.results.app === "installed"
142
- ? appOpensResearch(ctx.env) ? "ui.o.results.installed.live" : "ui.o.results.installed"
166
+ ? appOpensResearch(ctx.settings) && !o.results.told ? "ui.o.results.installed.live" : "ui.o.results.installed"
143
167
  : o.results.app === "offer"
144
- ? appOpensResearch(ctx.env) ? "ui.o.results.offer.live" : "ui.o.results.offer"
145
- : "ui.o.results.plain", { file: ctx.display(o.results.file), url: stromAppUrl(ctx.env) }))
146
- : undefined, o.stories ? row(t("ui.o.stories"), t(`ui.o.stories.${o.stories}`)) : undefined, o.researches?.length
168
+ ? o.results.told ? "ui.o.results.offered" : appOpensResearch(ctx.settings) ? "ui.o.results.offer.live" : "ui.o.results.offer"
169
+ : "ui.o.results.plain", { file: ctx.display(o.results.file), url: stromAppUrl(ctx.settings) }))
170
+ : undefined, o.stories ? row(t("ui.o.stories"), t(`ui.o.stories.${o.stories}`)) : undefined, o.ask ? row(t("ui.o.ask"), t("ui.o.ask.tell")) : undefined, o.reread ? row(t("ui.o.reread"), t("ui.o.reread.offer", { n: o.reread.facts, models: o.reread.models.join(", "), model: o.reread.model })) : undefined, o.researches?.length
147
171
  ? t("ui.o.research") + "\n" + table(o.researches.map((r) => [` ${r.id}`, r.name, `[${r.state}]`, r.focus]))
148
172
  : t("ui.o.research.none"), o.working?.length
149
173
  ? t("ui.o.working") + "\n" + table(o.working.map((w) => [` ${w.who}`, t("ui.o.since", { time: w.since.slice(11, 16) }), w.task ? t("ui.o.on", { task: w.task }) : ""]))