viewable-one 0.0.0-stage → 0.12.1

package/lib/digest.mjs

@@ -0,0 +1,290 @@
+// Shaping the mirror digest before a session pays for it.
+//
+// WHY THIS EXISTS (2026-08-31, from a spend audit over 201 transcripts and
+// 26,841 API requests): `get_board` returned the whole 48KB digest - 201
+// nodes, roughly 12k tokens - and that text then rode along in context for
+// every remaining request of the session. Measured cost: $0.50 per call,
+// $52-71 over nine days, about 60% of everything the board bridge cost. The
+// call frequency was already fine (107 of 137 board-using sessions called it
+// exactly ONCE), so the fix is not "call it less". It is "return less".
+//
+// The digest's own format is the input here. `src/mirror.ts` writes the file
+// and `src/digest/boardDigest.ts` renders its node lines; this module only
+// reads what they wrote. One kind carries the work: `[idea]`. An idea's open
+// and answered questions are NOTES on it, so they appear twice: as indented
+// `asked:` lines under the idea (kept with `detail:true`) and as one row each
+// in the QUESTIONS block, `- <noteId> open|answered ... on=<ideaId> "..."`,
+// which the slim view always keeps whole (the note id is what check_answer
+// reads, the idea id says where it sits). That makes it a PARSER OF SOMEONE ELSE'S OUTPUT, so every
+// rule below fails toward passing the line through untouched: a format change
+// upstream costs the saving, never the content. If a line does not look like
+// what we expect, it survives whole.
+//
+// What the slim view drops, and why each is safe to drop:
+//   - geometry, `(x,y) WxH` on every line. Added 2026-08-25 because a model
+//     asked to TIDY the board needs extents to see overlap - true, and that
+//     is why `view:"full"` and `geometry:true` still carry it. But the other
+//     ~200 sessions were reading the board to find out what was decided, and
+//     paying for pixel coordinates to do it.
+//   - the body preview after `"Title":`. boardView truncates content to 150
+//     chars, and the title is usually the first words of that same content
+//     ("Structured data formats" then "Structured data formats - JSON, XML,
+//     …"). Near-duplicate text at roughly 2/3 of the node line's length.
+//     `get_node` still returns any idea in full.
+//   - the AGES block, which repeats every node id a second time to attach a
+//     number. Slim folds the age onto the node's own line instead, so the
+//     information survives and the second copy of 201 ids does not.
+//   - group `members: [...]` lists, a third copy of the same ids - every node
+//     line already says `group=`.
+//
+// Nothing is ever dropped silently. Every response ends with a line saying
+// what was omitted and which argument brings it back.
+
+/** Section markers, exactly as src/mirror.ts and boardView.ts emit them. */
+const IS_GROUPS = (line) => line === "GROUPS:";
+const IS_NODES = (line) => line === "NODES:";
+const IS_EDGES = (line) => line.startsWith("EDGES: ");
+const IS_QUESTIONS = (line) => line.startsWith("QUESTIONS (");
+// "BOARD FLAGS" since 2026-10-05, when the Subconscious row left and the
+// Board Agent took the scan over; the old spelling still matches, because a
+// digest written before that day is still on disk in linked folders.
+const IS_FLAGS = (line) => line.startsWith("BOARD FLAGS") || line.startsWith("SUBCONSCIOUS FLAGS");
+const IS_AGES = (line) => line.startsWith("AGES (");
+
+/** ` (x,y)` and its optional ` WxH`, wherever boardView put them. */
+const GEOMETRY = /\s\(-?\d+,-?\d+\)(?:\s\d+x\d+)?/;
+
+/** The trailing facts on a node line that are NOT body text. When the body
+ *  preview goes, these come back - they are state a session acts on. */
+const META =
+  /\s+comments=\d+|\s+notes=\d+|\s+tags=\[[^\]]*\]|\s+hidden\b|\s+(?:result|summary|error)="(?:[^"\\]|\\.)*"|\s+PROPOSED-[A-Z]+\([^)]*\)/g;
+
+/** `- <id> [<kind>] …` - the first token of a node or group entry. */
+const ENTRY_ID = /^-\s+(\S+)/;
+const ENTRY_KIND = /^-\s+\S+\s+\[([^\]]+)\]/;
+const ENTRY_DRAWN = /^-\s+\S+\s+\[[^\]]+\] \[drawn\]/;
+const ENTRY_GROUP = /\sgroup=(\S+)/;
+const ENTRY_TAGS = /\stags=\[([^\]]*)\]/;
+
+/** Everything after the FIRST `"quoted title": ` is body preview. A line with
+ *  no such shape (a picture box or a brief line, say) is left exactly as it was. */
+function stripBody(line) {
+  const match = line.match(/^(.*?"(?:[^"\\]|\\.)*"):\s/);
+  if (!match) return line;
+  const tail = line.slice(match[0].length);
+  const meta = tail.match(META) ?? [];
+  return match[1] + meta.join("");
+}
+
+/** `id=3d, id=0d, …` from the AGES line into a map. */
+function parseAges(line) {
+  const ages = new Map();
+  const body = line.slice(line.indexOf("):") + 2);
+  for (const pair of body.split(",")) {
+    const match = pair.trim().match(/^(\S+)=(\d+)d$/);
+    if (match) ages.set(match[1], Number(match[2]));
+  }
+  return ages;
+}
+
+/** Split the raw digest into the parts we shape separately. Anything we do
+ *  not recognise lands in `rest` and is reprinted verbatim, in order. */
+function parse(text) {
+  const out = {
+    preamble: [],
+    groups: [],
+    nodes: [],
+    edges: [],
+    questions: [],
+    flags: [],
+    ages: null,
+    rest: [],
+    hadNodes: false,
+  };
+  let section = "preamble";
+  for (const line of text.split("\n")) {
+    if (IS_GROUPS(line)) {
+      section = "groups";
+      continue;
+    }
+    if (IS_NODES(line)) {
+      section = "nodes";
+      out.hadNodes = true;
+      continue;
+    }
+    if (IS_EDGES(line)) {
+      out.edges.push(line);
+      section = "after";
+      continue;
+    }
+    if (IS_QUESTIONS(line)) {
+      out.questions.push(line);
+      section = "questions";
+      continue;
+    }
+    if (IS_FLAGS(line)) {
+      out.flags.push(line);
+      section = "flags";
+      continue;
+    }
+    if (IS_AGES(line)) {
+      out.ages = line;
+      section = "after";
+      continue;
+    }
+
+    if (section === "groups" || section === "nodes") {
+      const list = section === "groups" ? out.groups : out.nodes;
+      if (line.startsWith("- ")) {
+        list.push({ head: line, extra: [] });
+      } else if (line.trim() === "") {
+        // Layout. The renderer below puts the blanks back where it wants them.
+      } else if (/^\s/.test(line) && list.length > 0) {
+        // INDENTED: boardView's multi-line kinds - one-at-a-time items,
+        // comparison rows, lesson steps - hang off the entry above them.
+        list[list.length - 1].extra.push(line);
+      } else {
+        // Neither an entry nor an indented continuation, so it is something
+        // this parser has never seen: a section the renderer grew after this
+        // module was written. Reprint it whole. Guessing at it would cost
+        // content; passing it through only costs the saving.
+        out.rest.push(line);
+      }
+      continue;
+    }
+    if (section === "questions") out.questions.push(line);
+    else if (section === "flags") out.flags.push(line);
+    else if (section === "preamble") out.preamble.push(line);
+    else out.questions.push(line); // trailing blanks after EDGES/AGES
+  }
+  return out;
+}
+
+const entryId = (entry) => entry.head.match(ENTRY_ID)?.[1] ?? "";
+const entryKind = (entry) => entry.head.match(ENTRY_KIND)?.[1] ?? "";
+const entryGroup = (entry) => entry.head.match(ENTRY_GROUP)?.[1] ?? "";
+const entryTags = (entry) =>
+  (entry.head.match(ENTRY_TAGS)?.[1] ?? "")
+    .split(",")
+    .map((t) => t.trim())
+    .filter(Boolean);
+
+/**
+ * Shape one raw digest for one call.
+ *
+ * @param {string} text  the mirror file, exactly as the app wrote it
+ * @param {object} opts
+ *   view       "slim" (default) | "full"  - "full" returns `text` untouched
+ *              except for filters, so the old behaviour is one argument away
+ *   geometry   true to keep (x,y) WxH in the slim view (for tidying work)
+ *   detail     true to keep body previews and multi-line node contents
+ *   tag        keep only nodes carrying this tag
+ *   group      keep only nodes in this group id
+ *   kind       keep only nodes of this kind ("idea", "doc", "note", …)
+ *   since_days keep only nodes touched within N days
+ *   drawn      keep only nodes carrying a drawing - the writer marks them
+ *              "[drawn]" after the kind (brief 32 Move 1, 2026-09-12)
+ *   match      keep only nodes whose line contains this (case-insensitive)
+ *   limit      cap the node lines returned
+ * @returns {{text: string, stats: object}}
+ */
+export function shapeDigest(text, opts = {}) {
+  const full = opts.view === "full";
+  const keepGeometry = full || opts.geometry === true;
+  const keepDetail = full || opts.detail === true;
+
+  const parsed = parse(text);
+  const ages = parsed.ages ? parseAges(parsed.ages) : new Map();
+  const total = parsed.nodes.length;
+
+  const wanted = [];
+  for (const entry of parsed.nodes) {
+    const id = entryId(entry);
+    if (opts.tag && !entryTags(entry).includes(opts.tag)) continue;
+    if (opts.group && entryGroup(entry) !== opts.group) continue;
+    if (opts.kind && entryKind(entry) !== opts.kind) continue;
+    if (opts.drawn === true && !ENTRY_DRAWN.test(entry.head)) continue;
+    if (opts.match && !entry.head.toLowerCase().includes(String(opts.match).toLowerCase())) continue;
+    if (typeof opts.since_days === "number") {
+      const age = ages.get(id);
+      // A node with no age predates the attention log. Age filters are asked
+      // for to find RECENT work, so an unknown age is not recent.
+      if (age === undefined || age > opts.since_days) continue;
+    }
+    wanted.push(entry);
+  }
+
+  const limit = typeof opts.limit === "number" && opts.limit > 0 ? opts.limit : undefined;
+  const shown = limit === undefined ? wanted : wanted.slice(0, limit);
+  const droppedByLimit = wanted.length - shown.length;
+
+  const renderNode = (entry) => {
+    let head = entry.head;
+    if (!keepDetail) head = stripBody(head);
+    if (!keepGeometry) head = head.replace(GEOMETRY, "");
+    if (!full) {
+      const age = ages.get(entryId(entry));
+      if (age !== undefined) head += ` ${age}d`;
+    }
+    return keepDetail ? [head, ...entry.extra] : [head];
+  };
+
+  const renderGroup = (entry) => {
+    let head = entry.head;
+    if (!keepGeometry) head = head.replace(GEOMETRY, "");
+    if (!keepDetail) head = head.replace(/\smembers:\s\[[^\]]*\]/, "");
+    if (!full) {
+      const age = ages.get(entryId(entry));
+      if (age !== undefined) head += ` ${age}d`;
+    }
+    return head;
+  };
+
+  const omitted = [];
+  if (!keepGeometry) omitted.push("positions and sizes (geometry:true)");
+  if (!keepDetail) omitted.push("body previews and group member lists (detail:true)");
+  if (!full && parsed.ages) omitted.push("the AGES block, folded onto each line as Nd");
+
+  const lines = [...parsed.preamble];
+  if (parsed.groups.length > 0) {
+    lines.push("GROUPS:");
+    for (const entry of parsed.groups) lines.push(renderGroup(entry));
+  }
+  // Only if the digest had one. A mirror with no NODES section (an empty
+  // board, or a test fixture) must not grow a header it never wrote.
+  if (parsed.hadNodes) lines.push("NODES:");
+  for (const entry of shown) lines.push(...renderNode(entry));
+  lines.push(...parsed.rest);
+  lines.push(...parsed.edges);
+  if (parsed.questions.length > 0) lines.push("", ...parsed.questions);
+  if (parsed.flags.length > 0) lines.push("", ...parsed.flags);
+  if (full && parsed.ages) lines.push(parsed.ages);
+
+  // The footer is not decoration. A filtered board that does not say it is
+  // filtered is a board that lies to the next session about what is on it.
+  const filters = [];
+  if (opts.tag) filters.push(`tag=${opts.tag}`);
+  if (opts.group) filters.push(`group=${opts.group}`);
+  if (opts.kind) filters.push(`kind=${opts.kind}`);
+  if (opts.drawn === true) filters.push("drawn");
+  if (typeof opts.since_days === "number") filters.push(`since_days=${opts.since_days}`);
+  if (opts.match) filters.push(`match=${JSON.stringify(opts.match)}`);
+  if (limit !== undefined) filters.push(`limit=${limit}`);
+
+  const note = [
+    `[${full ? "full" : "slim"} view: ${shown.length} of ${total} nodes`,
+    filters.length > 0 ? ` (${filters.join(", ")})` : "",
+    droppedByLimit > 0
+      ? `; ${droppedByLimit} more matched and were NOT shown - raise limit or narrow the filter`
+      : "",
+    omitted.length > 0 ? `. Omitted: ${omitted.join("; ")}` : "",
+    `. get_node(id) returns any idea in full; view:"full" returns the whole digest.]`,
+  ].join("");
+  lines.push("", note);
+
+  const out = lines.join("\n");
+  return {
+    text: out,
+    stats: { total, shown: shown.length, bytesIn: text.length, bytesOut: out.length },
+  };
+}