teapot-coding-agent 0.29.4 → 0.29.6

package/dist/agent/agent.js

@@ -628,9 +628,25 @@ export class Agent {
         this.goal = stored ?? { text: "", status: "active", updatedAt: new Date().toISOString() };
         // operator-maintained task list lives beside goal.md
         this.todo = await fs.readFile(this.todoFile, "utf8").catch(() => "");
-        await this.refreshSkills();
-        // the conversation is NOT restored here: boot cost stays O(agents), not
-        // O(history). It is rebuilt lazily by ensureReady() on first interaction.
+        // #143: NO skill scan here. It was the single largest boot cost — 493 agents
+        // each walking their roots (readdir + stat, recursively) measured 2120 ms of
+        // a 4.7 s boot. Every one of those walks was redundant:
+        //
+        //   - the turn loop calls `refreshSkills()` at the TOP of every turn, before
+        //     the system prompt (which embeds the catalogue) is built, so the scan is
+        //     always current before any LLM request
+        //   - the `list_skills` tool calls it itself before reading the cache
+        //   - `/api/agents/:id/skills` (what the panel uses) does its own fresh
+        //     `currentSkills()` walk and never reads this cache
+        //
+        // Caching the walk was the other option and was WORSE: the fingerprint exists
+        // to notice a NESTED skill's add/edit, which does not change its root's mtime,
+        // so any TTL/mtime key reintroduces the staleness the walk was written to fix
+        // (the skills-fingerprint tests pin exactly that). Skipping boot's copy costs
+        // no freshness at all.
+        //
+        // The conversation is likewise not restored here: boot cost stays O(agents),
+        // not O(history). Both are rebuilt lazily by ensureReady()/the turn loop.
         if (this.opts.restoreSession) {
             this.status = "stopped";
             this.statusReason = "session not loaded — select it or send a prompt";

package/dist/log/events.js

@@ -57,11 +57,51 @@ export class EventLog {
         this.filePath = filePath;
         this.agentId = agentId;
     }
+    /**
+     * Bytes read from the END of the log to rebuild the in-memory head.
+     *
+     * `load()` used to `readFile()` the ENTIRE log and `JSON.parse` every event,
+     * only to learn two things: the highest `seq`, and the last event id per
+     * branch. MEASURED on a real 813-session data dir (369 MB of logs): that read
+     * was the single largest boot cost — 1603 ms of `load()` against 493 agents,
+     * plus the GC pressure of holding one whole log as a string and its `split`
+     * array (~50 MB of temporary garbage for a 23 MB log).
+     *
+     * Both facts live at the END of an append-only file:
+     *   - `seq` is assigned monotonically, so the last complete line holds the max
+     *   - `lastEventId()` is only ever called for the CURRENT branch (fork/spawn),
+     *     which is by definition the most recently written one
+     *
+     * So the tail is both sufficient and exact. A branch that has been dormant for
+     * more than WINDOW bytes loses its in-memory head; that only affects
+     * `parent:` on the next append to that stale branch, and the paths that matter
+     * (`fork`, child spawn) re-seed explicitly through `seedBranch()`.
+     */
+    static LOAD_TAIL_BYTES = 512 * 1024;
     async load() {
         mkdirSync(path.dirname(this.filePath), { recursive: true });
         try {
-            const { readFile } = await import("node:fs/promises");
-            const text = await readFile(this.filePath, "utf8");
+            const { readFile, stat, open } = await import("node:fs/promises");
+            const st = await stat(this.filePath);
+            let text;
+            if (st.size <= EventLog.LOAD_TAIL_BYTES) {
+                text = await readFile(this.filePath, "utf8"); // small log: read it all
+            }
+            else {
+                const start = st.size - EventLog.LOAD_TAIL_BYTES;
+                const fh = await open(this.filePath, "r");
+                try {
+                    const buf = Buffer.alloc(EventLog.LOAD_TAIL_BYTES);
+                    const { bytesRead } = await fh.read(buf, 0, buf.length, start);
+                    text = buf.subarray(0, bytesRead).toString("utf8");
+                }
+                finally {
+                    await fh.close();
+                }
+                // the window starts mid-line: drop the partial first line
+                const nl = text.indexOf("\n");
+                text = nl === -1 ? "" : text.slice(nl + 1);
+            }
             for (const line of text.split("\n")) {
                 if (!line.trim())
                     continue;

package/dist/master.js

@@ -13,9 +13,19 @@ import { spawn } from "node:child_process";
 import { Agent } from "./agent/agent.js";
 import { parseSchedule, matches, nextFireAt } from "./scheduler/cron.js";
 import { bus } from "./bus.js";
-import { fetchModelList, contextLengthFor, isReasoningEffort, } from "./model-meta.js";
+import { fetchModelList, peekModelList, contextLengthFor, isReasoningEffort, } from "./model-meta.js";
 /** how deep sub-agent spawning may nest (parent=0, its subs=1, …) */
 const MAX_SPAWN_DEPTH = 3;
+/**
+ * How long boot waits for an endpoint's model catalogue (#143).
+ *
+ * The window and the reasoning-effort gate both come from `/models`, so a boot
+ * that waits on it is waiting for metadata, not for anything it needs to start.
+ * The lookup is fired concurrently for every distinct endpoint before the agent
+ * loop (see prefetchModelLists), so this bound is paid once per ENDPOINT rather
+ * than once per agent — and only when the endpoint is slow or dead.
+ */
+const MODEL_META_BOOT_MS = 1_500;
 /** statuses that mean a child has finished working, for #130 */
 const SETTLED = new Set(["idle", "stopped", "waiting", "error"]);
 /**
@@ -362,6 +372,16 @@ export class Master {
      */
     deferConfigWrites = false;
     configDirty = false;
+    /**
+     * Endpoints whose catalogue boot has already WAITED for (#143).
+     *
+     * The wait must be paid once per endpoint, not once per agent. `Promise.race`
+     * against a timeout is per-CALL: with a dead endpoint every agent on it paid
+     * the full timeout again, because the shared request was still pending. 493
+     * agents over one unreachable provider measured 4.4s of boot spent purely
+     * idle, for a single request.
+     */
+    metaWaited = new Set();
     /** write once and clear the dirty flag — used at the end of a batched section */
     flushConfig() {
         this.configDirty = false;
@@ -495,6 +515,14 @@ export class Master {
         // boot, which is its own silent data loss.
         this.deferConfigWrites = true;
         this.configDirty = false;
+        // #143: start every endpoint's model-catalogue lookup NOW, concurrently.
+        //
+        // `addAgent` awaits this per agent, and the endpoints are shared by many
+        // agents (493 agents over 6 providers here). Fired up front, the requests
+        // overlap and each subsequent agent finds the promise already in flight or
+        // resolved — instead of the requests being serialised behind the agent loop,
+        // where a slow endpoint's timeout is paid again and again.
+        this.prefetchModelLists();
         try {
             for (const ac of this.config.agents) {
                 try {
@@ -596,6 +624,38 @@ export class Master {
      * Restarts reuse the latest existing session; fresh creations never touch
      * an older one's history.
      */
+    /**
+     * Fire the model-catalogue lookup for every distinct endpoint, concurrently.
+     *
+     * `fetchModelList` already shares an in-flight request per baseUrl, so this is
+     * about STARTING them together: without it the requests begin one at a time, as
+     * the agent loop reaches each provider, and a slow endpoint's timeout is paid
+     * serially by every agent on it.
+     *
+     * Best-effort by design — nothing awaits this. A failure is cached briefly by
+     * `fetchModelList`, and an agent whose lookup is not ready simply gets no
+     * inferred window, which is the same outcome as today's timeout.
+     */
+    prefetchModelLists() {
+        const endpoints = new Set();
+        const add = (baseUrl, apiKey) => {
+            if (!baseUrl)
+                return;
+            const key = baseUrl.replace(/\/+$/, "");
+            if (endpoints.has(key))
+                return;
+            endpoints.add(key);
+            void fetchModelList(baseUrl, apiKey).catch(() => []);
+        };
+        for (const ac of this.config.agents) {
+            const prov = this.config.providers?.[ac.provider ?? this.config.defaultProvider ?? ""];
+            add(ac.baseUrl ?? prov?.baseUrl ?? this.config.llm.baseUrl, ac.apiKey ?? prov?.apiKey ?? this.config.llm.apiKey);
+        }
+        // no configured agents yet (first-run wizard) — still warm the default so the
+        // first created agent does not wait
+        if (endpoints.size === 0)
+            add(this.config.llm.baseUrl, this.config.llm.apiKey);
+    }
     async addAgent(ac, opts = {}) {
         // #72: an agent id becomes a DIRECTORY name (sessions/<id>-<hash>), so it
         // must be a safe path segment. Every caller sanitised its own id — the HTTP
@@ -631,24 +691,33 @@ export class Master {
         // spawn-tree depth: computed from the config chain (ac may not be
         // registered yet when this runs — spawnChildFor persists AFTER creation)
         const myDepth = ac.parent ? this.depthOf(ac.parent) + 1 : 0;
-        // learn the model's real context window from provider metadata (once per
-        // endpoint, cached) unless the config already pins one explicitly
-        let inferredWindow;
-        if (!ac.contextWindowTokens && !this.config.contextWindowTokens) {
-            // don't block the event loop on provider metadata: fire the lookup but
-            // keep boot moving. The agent's gauge fills in when it resolves.
-            const listPromise = fetchModelList(llm.baseUrl, llm.apiKey);
-            inferredWindow = await Promise.race([
-                listPromise.then((l) => contextLengthFor(l, llm.model)),
-                new Promise((r) => setTimeout(() => r(undefined), 1_500)),
+        // Model metadata: the context window (for the compaction budget) and the
+        // catalogue entry that gates reasoning_effort support (#47) come from the
+        // SAME `/models` list.
+        //
+        // #143: boot must not WAIT for this. It only tunes the compaction budget and
+        // the effort controls, so a missing list costs nothing that matters; what it
+        // cost was boot time — measured at 4.4s of pure idle on 493 agents pointed at
+        // one unreachable endpoint, because the `Promise.race` timeout below is
+        // per-CALL and the shared request stayed pending for every agent.
+        //
+        // So: use the catalogue if `prefetchModelLists()` has already produced it
+        // (the common case — the lookups were fired when boot started), and wait only
+        // ONCE per endpoint for the stragglers, where the wait overlaps a request
+        // that is already in flight.
+        const needsWindow = !ac.contextWindowTokens && !this.config.contextWindowTokens;
+        const root = llm.baseUrl.replace(/\/+$/, "");
+        let list = peekModelList(llm.baseUrl);
+        if (list === undefined && !this.metaWaited.has(root)) {
+            this.metaWaited.add(root);
+            list = await Promise.race([
+                fetchModelList(llm.baseUrl, llm.apiKey),
+                // a placeholder so the race can settle without the list
+                new Promise((r) => setTimeout(() => r(undefined), MODEL_META_BOOT_MS)),
             ]);
-            void listPromise.catch(() => []);
         }
-        // catalogue entry for this model — gates reasoning_effort support (#47)
-        const modelMeta = (await Promise.race([
-            fetchModelList(llm.baseUrl, llm.apiKey),
-            new Promise((r) => setTimeout(() => r(undefined), 1_500)),
-        ]))?.find((m) => m.id === llm.model);
+        const inferredWindow = needsWindow && list ? contextLengthFor(list, llm.model) : undefined;
+        const modelMeta = list?.find((m) => m.id === llm.model);
         const sessionDir = this.resolveSessionDir(ac.id, opts.fresh === true);
         await mkdirSync(sessionDir, { recursive: true });
         // every session dir this master hands out carries an ownership manifest —
@@ -1200,11 +1269,54 @@ When this task is done (or truly blocked), call finish() with a summary for @${p
         }
         return { owner: best?.id ?? null, hadManifest: false };
     }
+    /**
+     * Who does this session directory really belong to, according to its LOG?
+     *
+     * The manifest is a CACHE; the log is the truth. Every event carries the
+     * `agent` that wrote it, and only that agent ever appends — so the first
+     * event's value cannot be stale or wrong.
+     *
+     * This matters because ownership decided from the manifest ALONE is how a
+     * whole data directory gets abandoned. A manifest that is missing, unreadable,
+     * or simply wrong made `inferSessionOwner` reject every directory the agent
+     * owned, `findLatestSessionDir` return null, and the boot mint a fresh empty
+     * session — REWRITING the agent's pointer in config.json. Nothing warned, and
+     * the real history was left on disk, orphaned and unreachable.
+     *
+     * Measured after an accidental in-place `addAgent()` run had rewritten 813
+     * manifests: 493 agents were silently moved to empty sessions, and the
+     * mapping back was only recoverable from these `agent` fields.
+     */
+    logOwner(dirName) {
+        try {
+            const first = readFileSync(path.join(this.sessionsRoot(), dirName, "chat.jsonl"), "utf8")
+                .split("\n")
+                .find((l) => l.trim());
+            if (!first)
+                return null;
+            const parsed = JSON.parse(first);
+            return typeof parsed.agent === "string" && parsed.agent ? parsed.agent : null;
+        }
+        catch {
+            return null; // no log / unreadable head — cannot prove anything
+        }
+    }
     writeSessionManifest(dirPath, agentId) {
         try {
             const file = path.join(dirPath, "session.json");
-            if (existsSync(file))
-                return; // never clobber an existing manifest
+            if (existsSync(file)) {
+                // A WRONG manifest must be repairable, or the mistake is permanent: the
+                // old code returned here unconditionally, so one bad value kept every
+                // later boot from recognising the directory (see logOwner()).
+                const current = this.readSessionOwner(path.basename(dirPath));
+                if (current === agentId)
+                    return;
+                const proven = this.logOwner(path.basename(dirPath));
+                if (proven !== agentId)
+                    return; // not provably ours — leave it alone
+                console.log(`[teapot] repairing session manifest for ${path.basename(dirPath)}: ` +
+                    `${JSON.stringify(current)} → ${JSON.stringify(agentId)} (the log says ${JSON.stringify(agentId)})`);
+            }
             writeFileSync(file, `${JSON.stringify({ v: 1, agentId }, null, 2)}\n`);
         }
         catch {
@@ -1229,8 +1341,15 @@ When this task is done (or truly blocked), call finish() with a summary for @${p
             const empty = [];
             for (const d of readdirSync(this.sessionsRoot())) {
                 const { owner } = this.inferSessionOwner(d, candidateIds);
-                if (owner !== agentId)
+                // A wrong manifest must not hide a directory: consult the LOG before
+                // rejecting. Without this, one bad manifest value made every dir the
+                // agent owned invisible here, so the caller minted a fresh session and
+                // rewrote config — silently orphaning the real history.
+                const mineByLog = owner !== agentId && this.logOwner(d) === agentId;
+                if (owner !== agentId && !mineByLog)
                     continue;
+                if (mineByLog)
+                    this.writeSessionManifest(path.join(this.sessionsRoot(), d), agentId);
                 const dirPath = path.join(this.sessionsRoot(), d);
                 const m = this.sessionMtime(dirPath);
                 // an EMPTY chat.jsonl means the session was never really used — but it
@@ -1285,13 +1404,23 @@ When this task is done (or truly blocked), call finish() with a summary for @${p
         const recorded = this.config.agents.find((a) => a.id === agentId)?.session;
         if (!fresh && recorded) {
             const dir = path.join(root, recorded);
-            // only trust it when it is really THIS agent's dir (ownership comes from
-            // the manifest; a renamed/reused dir must not resurrect a ghost)
+            // Trust it when it exists and is really THIS agent's dir. Ownership comes
+            // from the manifest, but the LOG is consulted too: a wrong/absent manifest
+            // must not make teapot abandon history it can plainly see (see logOwner).
             const owner = this.readSessionOwner(recorded);
-            if (existsSync(dir) && (owner === null || owner === agentId))
+            const mineByManifest = owner === null || owner === agentId;
+            const mineByLog = owner !== agentId && this.logOwner(recorded) === agentId;
+            if (existsSync(dir) && (mineByManifest || mineByLog)) {
+                if (mineByLog)
+                    this.writeSessionManifest(dir, agentId); // repair it
                 return dir;
-            // the recorded dir is gone (deleted) — fall through and mint a new one
-            console.log(`[teapot] session ${recorded} for ${agentId} is gone; starting a new one`);
+            }
+            // the recorded dir is gone, or provably belongs to someone else — fall
+            // through and mint a new one. Say WHICH, and do not do it quietly: this
+            // line is the only trace that an agent changed session.
+            console.log(`[teapot] session ${recorded} for ${agentId} is unusable ` +
+                `(${existsSync(dir) ? `manifest owner ${JSON.stringify(owner)}` : "directory missing"}); ` +
+                `starting a new one`);
         }
         // Prefer the LATEST session directory that actually carries history.
         // This handles the case where a bare-name directory (e.g. from legacy

package/dist/model-meta.js

@@ -8,11 +8,47 @@ import { providerHeaders } from "./agent/llm.js";
 const cache = new Map();
 const inFlight = new Map();
 const TTL = 10 * 60_000;
+/**
+ * How long a FAILED lookup is remembered.
+ *
+ * A failure used to be dropped on the floor (the `catch` returned `[]` without
+ * caching), so a dead, unauthorised, or simply slow endpoint was re-fetched for
+ * EVERY agent pointed at it. `addAgent` awaits this twice per agent and races it
+ * against a 1.5 s timeout, so with 493 agents across a handful of providers the
+ * same failing request — and its timeout — repeated hundreds of times. MEASURED
+ * at boot: 1883 ms of the 4.6 s was idle, waiting on exactly this.
+ *
+ * Negative results expire much sooner than positive ones: a provider coming back
+ * up should be usable within a minute, not locked out for ten.
+ */
+const NEGATIVE_TTL = 30_000;
+/** is this cached entry still valid? (failures expire sooner) */
+function fresh(entry) {
+    return Date.now() - entry.at < (entry.negative ? NEGATIVE_TTL : TTL);
+}
+/**
+ * The catalogue if it is ALREADY known, without starting a fetch (#143).
+ *
+ * Boot uses this so it never waits for metadata it can do without: the context
+ * window only tunes the compaction budget, and `supportedParameters` only gates
+ * the reasoning-effort controls (#47). `prefetchModelLists()` starts the real
+ * lookups in parallel, so an endpoint that answers quickly is usually warm by the
+ * time its first agent is created, and a slow or dead one costs no boot time.
+ */
+export function peekModelList(baseUrl) {
+    const root = baseUrl.replace(/\/+$/, "");
+    const hit = cache.get(root);
+    if (!hit)
+        return undefined;
+    if (!fresh(hit))
+        return undefined;
+    return hit.negative ? [] : hit.list;
+}
 /** Fetch GET /models from an OpenAI-compatible endpoint (cached ~10 min). */
 export async function fetchModelList(baseUrl, apiKey) {
     const root = baseUrl.replace(/\/+$/, "");
     const hit = cache.get(root);
-    if (hit && Date.now() - hit.at < TTL)
+    if (hit && fresh(hit))
         return hit.list;
     // An in-flight lookup is SHARED: without this, two callers racing for the
     // same endpoint each paid the full timeout, and `newSession` in ACP mode
@@ -63,6 +99,9 @@ export async function fetchModelList(baseUrl, apiKey) {
             return list;
         }
         catch {
+            // Remember the FAILURE (briefly). Dropping it meant every agent on this
+            // endpoint paid the fetch — and up to the 1.5 s race timeout — again.
+            cache.set(root, { at: Date.now(), list: [], negative: true });
             return []; // unreachable provider — caller treats as "unknown"
         }
     })();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "teapot-coding-agent",
-  "version": "0.29.4",
+  "version": "0.29.6",
   "description": "A lightweight, always-on multi-agent harness for AI coding agents",
   "type": "module",
   "license": "AGPL-3.0-or-later",