toolroll 0.9.62 → 0.9.65

package/dist/telegram.js

@@ -46,6 +46,10 @@ import { LEAD_MESSAGE_MAX_CHARS } from "./lead.js";
 import { envValue } from "./names.js";
 import { applyProposalTap, processTelegramConversations, phoneLinkButton, replyContextFor, telegramConversationRepos, telegramRequestId, tooLongText, whichTaskText, } from "./telegram-lead.js";
 import { askOf, parseTelegramUpdate, pickOf, readTelegramButtonData, readTelegramUpdate, telegramButton } from "./contracts/telegram-callback.js";
+/** Read the enrolled project list on demand. No callback means no task data,
+ * never an implicit all-database ceiling. Shared by pass and embedded follower. */
+/** A delivery its person began watching after it was claimed: held or settled at the final fence (cockpit §3). */
+const WATCHED_SINCE_CLAIM = "Held: its person is watching this agent now";
 /** The environment name — and therefore the name the builder strips from agents (both names; see names.ts). */
 export { TELEGRAM_TOKEN_ENV as TOKEN_ENV, TELEGRAM_TOKEN_ENVS as TOKEN_ENVS } from "./names.js";
 /** BotFather's shape: numeric bot id, colon, secret. */
@@ -732,6 +736,9 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
         const blocked = [];
         const preflight = (await readAccess()) ?? channelProblem();
         for (const row of group) {
+            // Watched since the claim (cockpit §3): held for later or settled quietly, never sent now, never a failure.
+            if (preflight === null && store.telegramWatchFence(row, binding, owner, clock()) !== "send")
+                continue;
             const problem = preflight ?? store.telegramDeliveryProblem(row, binding, owner, projects, clock(), [...rows.map(one => one.id), row.id]);
             if (problem === null)
                 rows.push(row);
@@ -747,7 +754,18 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
         if (rows.length === 0)
             continue;
         const ids = rows.map(row => row.id);
-        const fence = () => {
+        // Once any part of this delivery has reached Telegram it is committed: a watch opening between its parts never
+        // releases it (that would send the parts already accepted again). Every transport call below goes through `wire`.
+        let sentAny = false;
+        const wire = async (...args) => {
+            const answer = await transport(...args);
+            if (answer.ok === true)
+                sentAny = true;
+            return answer;
+        };
+        // `beforeSend`: the fence before a transport call also asks the watch — until something of this delivery went out;
+        // the one after a confirmed send never does, so a message Telegram accepted is recorded as delivered.
+        const fence = (beforeSend = !sentAny) => {
             const channel = channelProblem();
             if (channel !== null)
                 return channel;
@@ -755,6 +773,10 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
                 const problem = store.telegramDeliveryProblem(row, binding, owner, projects, clock(), ids);
                 if (problem !== null)
                     return problem;
+                // The watch, at the last moment before the network: a held row's claim is released (its finalize is then a
+                // no-op), so nothing is spent on it.
+                if (beforeSend && store.telegramWatchFence(row, binding, owner, clock()) !== "send")
+                    return WATCHED_SINCE_CLAIM;
             }
             return null;
         };
@@ -763,7 +785,7 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
             const problem = (await readAccess()) ?? fence();
             if (problem !== null)
                 return { ok: false, error: problem };
-            const sent = await send(transport, binding.chatId, text, keyboard, entities, silent);
+            const sent = await send(wire, binding.chatId, text, keyboard, entities, silent);
             if (!sent.ok) {
                 if (sent.retryAfter !== undefined)
                     store.deferTelegram(botId, new Date(clock().getTime() + sent.retryAfter * 1_000).toISOString());
@@ -774,7 +796,7 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
             for (const row of messageRows)
                 store.recordTelegramMessage(row, binding, sent.messageId, clock());
             // Keep the old message's history but never acknowledge a replaced pairing.
-            const after = (await readAccess()) ?? fence();
+            const after = (await readAccess()) ?? fence(false);
             return after === null ? sent : { ok: false, error: after };
         };
         const imageSender = async (row) => {
@@ -803,7 +825,7 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
             // No await between the access fence, file validation and upload.
             let answer;
             try {
-                answer = await transport("sendDocument", { chat_id: binding.chatId, caption: `${resultTaskLabel(row.taskId)} · result #${row.run} · screenshot for acceptance` }, undefined, {
+                answer = await wire("sendDocument", { chat_id: binding.chatId, caption: `${resultTaskLabel(row.taskId)} · result #${row.run} · screenshot for acceptance` }, undefined, {
                     field: "document", bytes: verified.bytes, contentType: verified.format === "png" ? "image/png" : "image/jpeg", fileName: resultImageFileName(row.taskId, row.run, artifact, verified.format),
                 });
             }
@@ -819,7 +841,7 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
                 return { ok: false, error: answer.uncertain || (answer.ok && messageId == null) ? "Screenshot delivery is unconfirmed; retry may duplicate it" : "Telegram did not accept the screenshot" };
             }
             store.recordTelegramMessage(row, binding, String(messageId), clock());
-            const after = (await readAccess()) ?? fence();
+            const after = (await readAccess()) ?? fence(false);
             return after === null ? { ok: true, receipt: receiptFor(botId, binding.chatId, String(messageId)) } : { ok: false, error: after };
         };
         /** Screenshots with a result (result-shots.ts): photos, several as one album, re-proved before every call. Each is
@@ -846,10 +868,10 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
                 let answer;
                 try {
                     answer = batch.length > 1
-                        ? await transport("sendMediaGroup", { chat_id: binding.chatId, media: batch.map((one, index) => ({ type: "photo", media: `attach://shot${index}`, ...(index === 0 || one.numbered === true ? { caption: one.caption } : {}) })) }, undefined, upload(batch[0], "shot0"), batch.slice(1).map((one, index) => upload(one, `shot${index + 1}`)))
+                        ? await wire("sendMediaGroup", { chat_id: binding.chatId, media: batch.map((one, index) => ({ type: "photo", media: `attach://shot${index}`, ...(index === 0 || one.numbered === true ? { caption: one.caption } : {}) })) }, undefined, upload(batch[0], "shot0"), batch.slice(1).map((one, index) => upload(one, `shot${index + 1}`)))
                         : batch[0].photo
-                            ? await transport("sendPhoto", { chat_id: binding.chatId, caption: batch[0].caption }, undefined, upload(batch[0], "photo"))
-                            : await transport("sendDocument", { chat_id: binding.chatId, caption: batch[0].caption }, undefined, upload(batch[0], "document"));
+                            ? await wire("sendPhoto", { chat_id: binding.chatId, caption: batch[0].caption }, undefined, upload(batch[0], "photo"))
+                            : await wire("sendDocument", { chat_id: binding.chatId, caption: batch[0].caption }, undefined, upload(batch[0], "document"));
                 }
                 catch {
                     answer = { ok: false, uncertain: true };
@@ -893,7 +915,7 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
             const keyboard = button === null ? [] : [button];
             if (messageId === null)
                 return sender(card.text, keyboard, rows, card.entities);
-            const edited = await editProgress(transport, binding.chatId, messageId, card.text, keyboard, card.entities);
+            const edited = await editProgress(wire, binding.chatId, messageId, card.text, keyboard, card.entities);
             if (!edited.ok) {
                 if (edited.retryAfter !== undefined)
                     store.deferTelegram(botId, new Date(clock().getTime() + edited.retryAfter * 1000).toISOString());
@@ -905,7 +927,7 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
             }
             if (!onlyExisting)
                 store.recordTelegramMessage(row, binding, messageId, clock());
-            const after = (await readAccess()) ?? fence();
+            const after = (await readAccess()) ?? fence(false);
             return after === null ? edited : { ok: false, error: after };
         };
         /** Quiet mode: the task's one card (or its group's), created silently and then only ever edited. */
@@ -929,11 +951,11 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
                 return { ok: true, messageId: card.message };
             }
             if (card.message !== null) {
-                const edited = await editProgress(transport, binding.chatId, card.message, view.text, keyboard, view.entities);
+                const edited = await editProgress(wire, binding.chatId, card.message, view.text, keyboard, view.entities);
                 if (edited.ok) {
                     store.setChatCardMessage(card.id, card.message, shown);
                     store.recordTelegramMessage(fact, binding, card.message, clock());
-                    const after = (await readAccess()) ?? fence();
+                    const after = (await readAccess()) ?? fence(false);
                     return after === null ? edited : { ok: false, error: after };
                 }
                 if (edited.retryAfter !== undefined)
@@ -1003,14 +1025,14 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
                     if (problem !== null)
                         return { ok: false, error: problem };
                     mintFor();
-                    const edited = await editProgress(transport, binding.chatId, batch.message, text, keyboard);
+                    const edited = await editProgress(wire, binding.chatId, batch.message, text, keyboard);
                     if (edited.ok) {
                         // The repaint landed: the old words' buttons retire and the new ones ride the message.
                         retireDecideTokens(store, "telegram", binding.chatId, batch.message, clock());
                         placeDecideTokens(store, tokens, batch.message);
                         store.setChatBatchMessage(batch.id, batch.message, shown);
                         store.recordTelegramMessage(fact, binding, batch.message, clock());
-                        const after = (await readAccess()) ?? fence();
+                        const after = (await readAccess()) ?? fence(false);
                         return after === null ? { ok: true, receipt: receiptFor(botId, binding.chatId, batch.message) } : { ok: false, error: after };
                     }
                     // A failed edit: the buttons it carried never act, and the card's old ones keep working.
@@ -1049,10 +1071,10 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
                 const problem = (await readAccess()) ?? fence();
                 if (problem !== null)
                     return { ok: false, error: problem };
-                const edited = await editProgress(transport, binding.chatId, earlier, words, keyboard);
+                const edited = await editProgress(wire, binding.chatId, earlier, words, keyboard);
                 if (edited.ok) {
                     store.recordTelegramMessage(fact, binding, earlier, clock());
-                    const after = (await readAccess()) ?? fence();
+                    const after = (await readAccess()) ?? fence(false);
                     return after === null ? { ok: true, receipt: receiptFor(botId, binding.chatId, earlier) } : { ok: false, error: after };
                 }
                 if (edited.retryAfter !== undefined)
@@ -1082,7 +1104,7 @@ async function deliverOutboxTo(store, botId, binding, transport, owner, clock, r
                     ? await deliverDigest(botId, binding, sender, rows, digest.lastSentAt, clock, taskId => chatTitle(store, taskId))
                     : await deliverOne(store, botId, binding, sender, rows[0], clock, phoneOrigin, evidenceRoot, projects);
         }
-        const finalProblem = outcome.ok ? (await readAccess()) ?? fence() : null;
+        const finalProblem = outcome.ok ? (await readAccess()) ?? fence(false) : null;
         const settled = finalProblem === null ? outcome : { ok: false, error: finalProblem };
         // A skipped screenshot settles its row but nothing reached the phone.
         const skipped = settled.ok && settled.receipt !== null && settled.receipt.startsWith("skipped:");
@@ -1748,6 +1770,8 @@ function applyMessage(context, update, effects) {
  * message that carried several tasks (never guessed). Everything hostile is
  * silence, as for every other inbound shape.
  */
+/** At most this many of a chat's messages wait for the lead at once (the personal lead queue's bound). */
+export const TELEGRAM_MAX_WAITING = 10;
 function applyConversation(context, update, effects) {
     const { store, botId, transport, clock, report } = context;
     const message = update.message;
@@ -1812,6 +1836,12 @@ function applyConversation(context, update, effects) {
             context_ = focusContextFor(focused);
         }
     }
+    // The lead queue's bound, per chat: ten messages wait at most; the next is answered at once instead of queued.
+    if (store.telegramMessagesWaiting(binding.id) >= TELEGRAM_MAX_WAITING) {
+        say(`${TELEGRAM_MAX_WAITING} messages are already waiting for the lead. Wait for a reply, then send this again.`);
+        report.ignored++;
+        return;
+    }
     store.enqueueTelegramConversation({
         binding, updateId: update.update_id, messageId: String(message.message_id), replyTo,
         request: telegramRequestId(botId, binding.id, update.update_id), text, context: context_, taskId, sourceRun,
@@ -2248,6 +2278,10 @@ function applyCallback(context, update, effects) {
             return;
         }
         const focused = store.chatFocus("telegram", binding.id);
+        if (store.telegramMessagesWaiting(binding.id) >= TELEGRAM_MAX_WAITING) {
+            ack(`${TELEGRAM_MAX_WAITING} messages are already waiting. Wait for a reply.`);
+            return;
+        }
         store.enqueueTelegramConversation({
             binding, updateId: update.update_id, messageId: String(message.message_id), replyTo: null,
             request: telegramRequestId(botId, binding.id, update.update_id), text: option,

package/dist/terminal-catalog.d.ts

@@ -0,0 +1,11 @@
+import type { Database } from "./store.js";
+import type { TerminalLedger } from "./live-terminal.js";
+export declare function terminalCatalogPath(orders: Database): string | null;
+/** The console's ledger: creates the catalog, drops rows of consoles that are gone, and writes its own. */
+export declare function openTerminalLedger(orders: Database, alive?: (pid: number) => boolean): TerminalLedger | null;
+export declare function installTerminalUpdateGate(orders: Database, id: string): void;
+export declare function removeTerminalUpdateGate(orders: Database, id: string): void;
+/** Terminals starting or running in a console that is still alive: active work an update waits for. */
+export declare function activeTerminalUpdateWork(orders: Database, alive?: (pid: number) => boolean): {
+    terminals: number;
+};

package/dist/terminal-catalog.js

@@ -0,0 +1,129 @@
+/**
+ * Live terminals, written down for the updater (docs/plans/live-terminal.md, review round 2 finding 7). Terminals live
+ * in the console's memory, but an update is decided by another process, so each one is a row in a small catalog
+ * beside the database (`<db>.terminals.sqlite`), as the coding workspace keeps its own:
+ *
+ * - a start inserts its row before any work, and an update's gate (a trigger) refuses that insert while the update
+ *   owns admission: no terminal starts during a drain;
+ * - a row is `starting` or `running` until the terminal's sweep ends, then deleted — or kept as `uncertain` when the
+ *   sweep couldn't prove its processes gone, until the terminal is removed; the updater counts rows whose console is
+ *   still alive as active work, so "update when idle" waits for running and unproved terminals;
+ * - rows of a console that is gone are stale (its programs went with it, or are swept at its next start).
+ */
+import { chmodSync, lstatSync } from "node:fs";
+import { processMayBeAlive } from "./process-liveness.js";
+import { sqliteRuntime } from "./sqlite-runtime.js";
+const gatePrefix = "so_terminal_update_";
+const PAUSED = "Toolroll is updating. Running terminals keep going; new ones can start after the update.";
+const SCHEMA = "CREATE TABLE IF NOT EXISTS live_terminal (id TEXT PRIMARY KEY, console_pid INTEGER NOT NULL, state TEXT NOT NULL CHECK (state IN ('starting','running','uncertain')), at TEXT NOT NULL)";
+export function terminalCatalogPath(orders) {
+    const main = orders.prepare("PRAGMA database_list").all().find(row => row["name"] === "main")?.["file"];
+    return typeof main === "string" && main !== "" ? `${main}.terminals.sqlite` : null;
+}
+function exists(file) {
+    try {
+        return lstatSync(file).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+function open(file, readOnly) {
+    if (exists(file) && lstatSync(file).isSymbolicLink())
+        throw Error("The terminal catalog is linked, not a regular database.");
+    const db = new (sqliteRuntime().DatabaseSync)(file, { readOnly });
+    db.exec("PRAGMA busy_timeout=1000");
+    return db;
+}
+/** The updater's read: never creates the catalog. */
+function withCatalog(orders, absent, readOnly, action) {
+    const file = terminalCatalogPath(orders);
+    if (file === null || !exists(file))
+        return absent;
+    const db = open(file, readOnly);
+    try {
+        return action(db);
+    }
+    finally {
+        db.close();
+    }
+}
+/** The console's ledger: creates the catalog, drops rows of consoles that are gone, and writes its own. */
+export function openTerminalLedger(orders, alive = pid => processMayBeAlive(pid, false)) {
+    const file = terminalCatalogPath(orders);
+    if (file === null)
+        return null;
+    const db = open(file, false);
+    chmodSync(file, 0o600);
+    db.exec(SCHEMA);
+    for (const row of db.prepare("SELECT DISTINCT console_pid FROM live_terminal").all()) {
+        const pid = Number(row["console_pid"]);
+        if (pid !== process.pid && !alive(pid))
+            db.prepare("DELETE FROM live_terminal WHERE console_pid=?").run(pid);
+    }
+    return {
+        reserve(id) {
+            // Fail closed even where this catalog had no gate to install (it didn't exist yet when the update began): the main
+            // database's own admission pause refuses too.
+            if (orders.prepare("SELECT 1 FROM sqlite_master WHERE type='trigger' AND name GLOB 'so_desktop_update_*' LIMIT 1").get() !== undefined)
+                throw new Error(PAUSED);
+            try {
+                db.prepare("INSERT INTO live_terminal (id, console_pid, state, at) VALUES (?, ?, 'starting', ?)").run(id, process.pid, new Date().toISOString());
+            }
+            catch (error) {
+                throw new Error(error instanceof Error && error.message.includes(PAUSED) ? PAUSED : "No terminal can start right now (its record couldn't be written).");
+            }
+        },
+        running(id) { db.prepare("UPDATE live_terminal SET state='running', at=? WHERE id=?").run(new Date().toISOString(), id); },
+        // A sweep that couldn't prove its processes gone stays written down: the updater counts it while this console lives.
+        release(id, cleanup) {
+            try {
+                if (cleanup === "uncertain")
+                    db.prepare("UPDATE live_terminal SET state='uncertain', at=? WHERE id=?").run(new Date().toISOString(), id);
+                else
+                    db.prepare("DELETE FROM live_terminal WHERE id=?").run(id);
+            }
+            catch { /* the next start drops it as stale */ }
+        },
+        forget(id) { try {
+            db.prepare("DELETE FROM live_terminal WHERE id=?").run(id);
+        }
+        catch { /* stale later */ } },
+    };
+}
+function gateStatement(id) {
+    if (!/^[a-f0-9-]{36}$/.test(id))
+        throw Error("Invalid update identity.");
+    return `CREATE TRIGGER ${gatePrefix}start BEFORE INSERT ON live_terminal BEGIN SELECT RAISE(ABORT, '${PAUSED} [${id}]'); END`;
+}
+function changeGate(db, id, install) {
+    const wanted = gateStatement(id);
+    db.exec("BEGIN IMMEDIATE");
+    try {
+        db.exec(SCHEMA);
+        const rows = db.prepare("SELECT name, sql FROM sqlite_master WHERE type='trigger' AND name GLOB 'so_terminal_update_*'").all();
+        if (rows.some(row => String(row["sql"]) !== wanted))
+            throw Error("Another or unrecognized update owns terminal admission. Its gate was preserved.");
+        if (install && rows.length === 0)
+            db.exec(wanted);
+        if (!install)
+            for (const row of rows)
+                db.exec(`DROP TRIGGER "${String(row["name"])}"`);
+        db.exec("COMMIT");
+    }
+    catch (error) {
+        db.exec("ROLLBACK");
+        throw error;
+    }
+}
+export function installTerminalUpdateGate(orders, id) { withCatalog(orders, undefined, false, db => changeGate(db, id, true)); }
+export function removeTerminalUpdateGate(orders, id) { withCatalog(orders, undefined, false, db => changeGate(db, id, false)); }
+/** Terminals starting or running in a console that is still alive: active work an update waits for. */
+export function activeTerminalUpdateWork(orders, alive = pid => processMayBeAlive(pid, false)) {
+    return withCatalog(orders, { terminals: 0 }, true, db => {
+        if (!db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='live_terminal'").get())
+            return { terminals: 0 };
+        const rows = db.prepare("SELECT console_pid FROM live_terminal").all();
+        return { terminals: rows.filter(row => alive(Number(row["console_pid"]))).length };
+    });
+}

package/dist/text-limits.d.ts

@@ -68,6 +68,8 @@ export declare const TEXT_LIMITS: {
     readonly triggerState: 40;
     readonly triggerWebhookTitle: 120;
     readonly triggerWebhookField: 80;
+    /** v126: a Sentry or Vercel project (or Vercel team) a signed webhook is scoped to. */
+    readonly triggerWebhookScope: 120;
     readonly triggerFolder: 100;
     readonly triggerSender: 300;
     readonly triggerSubject: 100;

package/dist/text-limits.js

@@ -68,6 +68,8 @@ export const TEXT_LIMITS = {
     triggerState: 40,
     triggerWebhookTitle: 120,
     triggerWebhookField: 80,
+    /** v126: a Sentry or Vercel project (or Vercel team) a signed webhook is scoped to. */
+    triggerWebhookScope: 120,
     triggerFolder: 100,
     triggerSender: 300,
     triggerSubject: 100,

package/dist/tool-broker.d.ts

@@ -0,0 +1,39 @@
+import type { ToolSpec } from "./contracts/integration-metadata.js";
+/** One tool a launch may use: its values (never leaving this process) and, for research, the only actions it offers. */
+export type BrokeredTool = {
+    spec: ToolSpec;
+    values: Record<string, string>;
+    actions?: readonly string[];
+};
+export type BrokerOptions = {
+    /** Where a local server's environment comes from (the launch's): only SERVER_ENV's names are taken from it. */
+    env?: NodeJS.ProcessEnv;
+    omitEnv?: readonly string[];
+    /** Toolroll's own settings for a local server (the run's isolated database), passed as they are. */
+    pass?: Readonly<Record<string, string>>;
+    cwd?: string;
+    /** How a local server is started inside its run's fence, read when it starts (the fence is fixed after the tools). */
+    wrap?: () => ((file: string, args: readonly string[]) => {
+        file: string;
+        args: string[];
+    }) | null;
+};
+export type ToolBroker = {
+    /** The launch's token: the agent's only key to its tools. */
+    token: string;
+    url: (name: string) => string;
+    /** Its loopback port: the one a confined network leaves reachable for its agent (agent-fence.ts NetworkFence). */
+    port: number;
+    /** The local servers running now (the fence tests read them). */
+    pids: () => number[];
+    /** Revoke the token, stop every server, close the port. */
+    close: () => void;
+};
+/** What a local server inherits from the launch: enough to run and find its tools, and nothing that could be
+ * another tool's or a provider's secret. Its own configured values are added after. */
+export declare const SERVER_ENV: readonly string[];
+/** A local server's environment: the allowlisted names from `base` (and its LC_* locale), less `omit`, then `pass`,
+ * then the tool's own values. */
+export declare function serverEnv(base: NodeJS.ProcessEnv, omit: readonly string[], pass: Readonly<Record<string, string>>, values: Readonly<Record<string, string>>): NodeJS.ProcessEnv;
+/** Start one launch's broker on a loopback port. */
+export declare function startToolBroker(tools: readonly BrokeredTool[], options?: BrokerOptions): Promise<ToolBroker>;