stagent 0.9.5 → 0.10.0

package/src/lib/agents/runtime/types.ts

@@ -3,10 +3,18 @@ import type { ProfileTestReport } from "@/lib/agents/profiles/test-types";
 import type { RuntimeCapabilities, RuntimeCatalogEntry } from "./catalog";
 import type { TaskAssistResponse } from "./task-assist-types";
 import type { ProfileAssistRequest, ProfileAssistResponse } from "./profile-assist-types";
+import type {
+  OpenAIAccountInfo,
+  OpenAIAuthMode,
+  OpenAIRateLimitInfo,
+} from "@/lib/settings/openai-auth";
 
 export interface RuntimeConnectionResult {
   connected: boolean;
   apiKeySource?: ApiKeySource;
+  account?: OpenAIAccountInfo | null;
+  rateLimits?: OpenAIRateLimitInfo | null;
+  authMode?: OpenAIAuthMode;
   error?: string;
 }
 

package/src/lib/book/chapter-mapping.ts

@@ -19,6 +19,7 @@ export const CHAPTER_SLUGS: Record<string, string> = {
   "ch-11": "ch-11-the-machine-that-builds-machines",
   "ch-12": "ch-12-the-road-ahead",
   "ch-13": "ch-13-the-wealth-manager",
+  "ch-14": "ch-14-the-meta-program",
 };
 
 interface ChapterMapping {
@@ -105,6 +106,16 @@ export const CHAPTER_MAPPING: Record<string, ChapterMapping> = {
     sourceFiles: ["src/lib/workflows/engine.ts", "src/lib/schedules/scheduler.ts", "src/lib/agents/profiles/registry.ts"],
     caseStudies: ["making-machine-that-builds-machines"],
   },
+  "ch-14": {
+    docs: ["workflows", "profiles", "schedules", "blueprints"],
+    sourceFiles: [
+      "src/lib/agents/profiles/registry.ts",
+      "src/lib/workflows/blueprints/registry.ts",
+      "src/lib/workflows/engine.ts",
+      "features/instance-bootstrap.md",
+    ],
+    caseStudies: ["making-machine-that-builds-machines"],
+  },
 };
 
 /** Get related Playbook doc slugs for a chapter */

package/src/lib/book/content.ts

@@ -166,6 +166,16 @@ export const CHAPTERS: BookChapter[] = [
     relatedDocs: ["workflows", "profiles", "schedules"],
     sections: [],
   },
+  {
+    id: "ch-14",
+    number: 14,
+    title: "The Meta-Program",
+    subtitle: "When the System You Are Using Is Also the System You Are Building",
+    part: PARTS[3],
+    readingTime: 16,
+    relatedDocs: ["workflows", "profiles", "schedules", "blueprints"],
+    sections: [],
+  },
 ];
 
 /**

package/src/lib/chat/__tests__/active-streams.test.ts

@@ -0,0 +1,49 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import {
+  registerChatStream,
+  unregisterChatStream,
+  getActiveChatStreamCount,
+  isAnyChatStreaming,
+} from "../active-streams";
+
+describe("active chat streams", () => {
+  beforeEach(() => {
+    for (const id of ["a", "b", "c"]) unregisterChatStream(id);
+  });
+
+  it("starts empty", () => {
+    expect(getActiveChatStreamCount()).toBe(0);
+    expect(isAnyChatStreaming()).toBe(false);
+  });
+
+  it("tracks a single registered stream", () => {
+    registerChatStream("a");
+    expect(getActiveChatStreamCount()).toBe(1);
+    expect(isAnyChatStreaming()).toBe(true);
+  });
+
+  it("tracks multiple streams independently", () => {
+    registerChatStream("a");
+    registerChatStream("b");
+    expect(getActiveChatStreamCount()).toBe(2);
+  });
+
+  it("is idempotent — registering the same id twice still counts as one", () => {
+    registerChatStream("a");
+    registerChatStream("a");
+    expect(getActiveChatStreamCount()).toBe(1);
+  });
+
+  it("unregisters by id", () => {
+    registerChatStream("a");
+    registerChatStream("b");
+    unregisterChatStream("a");
+    expect(getActiveChatStreamCount()).toBe(1);
+    expect(isAnyChatStreaming()).toBe(true);
+  });
+
+  it("unregistering a non-existent id is a no-op", () => {
+    expect(() => unregisterChatStream("never-registered")).not.toThrow();
+    expect(getActiveChatStreamCount()).toBe(0);
+  });
+});

package/src/lib/chat/__tests__/finalize-safety-net.test.ts

@@ -0,0 +1,139 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { db } from "@/lib/db";
+import { conversations, chatMessages } from "@/lib/db/schema";
+import { eq } from "drizzle-orm";
+import { randomUUID } from "crypto";
+import { finalizeStreamingMessage } from "../reconcile";
+
+function seedConversation(): string {
+  const id = randomUUID();
+  const now = new Date();
+  db.insert(conversations)
+    .values({
+      id,
+      runtimeId: "test-runtime",
+      status: "active",
+      createdAt: now,
+      updatedAt: now,
+    })
+    .run();
+  return id;
+}
+
+function seedStreaming(convId: string, content: string): string {
+  const id = randomUUID();
+  db.insert(chatMessages)
+    .values({
+      id,
+      conversationId: convId,
+      role: "assistant",
+      content,
+      status: "streaming",
+      createdAt: new Date(),
+    })
+    .run();
+  return id;
+}
+
+describe("finalizeStreamingMessage", () => {
+  beforeEach(() => {
+    db.delete(chatMessages).run();
+    db.delete(conversations).run();
+  });
+
+  it("is a no-op when the message is already complete", async () => {
+    const convId = seedConversation();
+    const id = randomUUID();
+    db.insert(chatMessages)
+      .values({
+        id,
+        conversationId: convId,
+        role: "assistant",
+        content: "Already finished",
+        status: "complete",
+        createdAt: new Date(),
+      })
+      .run();
+
+    await finalizeStreamingMessage(id, "ignored salvage text");
+
+    const row = db
+      .select()
+      .from(chatMessages)
+      .where(eq(chatMessages.id, id))
+      .get();
+    expect(row?.status).toBe("complete");
+    expect(row?.content).toBe("Already finished");
+  });
+
+  it("salvages streaming row with substantial content as complete", async () => {
+    const convId = seedConversation();
+    const id = seedStreaming(convId, "");
+    const partialText =
+      "I searched the web and found three relevant articles about the topic. Here are the highlights of what I learned...";
+
+    await finalizeStreamingMessage(id, partialText);
+
+    const row = db
+      .select()
+      .from(chatMessages)
+      .where(eq(chatMessages.id, id))
+      .get();
+    expect(row?.status).toBe("complete");
+    expect(row?.content).toBe(partialText);
+  });
+
+  it("marks streaming row with no content as error with fallback string", async () => {
+    const convId = seedConversation();
+    const id = seedStreaming(convId, "");
+
+    await finalizeStreamingMessage(id, "");
+
+    const row = db
+      .select()
+      .from(chatMessages)
+      .where(eq(chatMessages.id, id))
+      .get();
+    expect(row?.status).toBe("error");
+    expect(row?.content).toMatch(/interrupted/i);
+    expect(row?.content.length).toBeGreaterThan(0);
+  });
+
+  it("marks streaming row with very short content as error, not complete", async () => {
+    const convId = seedConversation();
+    const id = seedStreaming(convId, "");
+
+    // 20 chars — not substantial enough to call "complete"
+    await finalizeStreamingMessage(id, "Just a short reply.");
+
+    const row = db
+      .select()
+      .from(chatMessages)
+      .where(eq(chatMessages.id, id))
+      .get();
+    expect(row?.status).toBe("error");
+    expect(row?.content).toBe("Just a short reply.");
+  });
+
+  it("marks streaming row with whitespace-only fullText as error with fallback", async () => {
+    const convId = seedConversation();
+    const id = seedStreaming(convId, "");
+
+    await finalizeStreamingMessage(id, "   \n\n  \t ");
+
+    const row = db
+      .select()
+      .from(chatMessages)
+      .where(eq(chatMessages.id, id))
+      .get();
+    expect(row?.status).toBe("error");
+    expect(row?.content).toMatch(/interrupted/i);
+  });
+
+  it("is a no-op when the message does not exist", async () => {
+    // Should not throw — defensive null check
+    await expect(
+      finalizeStreamingMessage("nonexistent-id", "some text"),
+    ).resolves.not.toThrow();
+  });
+});

package/src/lib/chat/__tests__/reconcile.test.ts

@@ -0,0 +1,137 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { db } from "@/lib/db";
+import { conversations, chatMessages } from "@/lib/db/schema";
+import { eq } from "drizzle-orm";
+import { randomUUID } from "crypto";
+import { reconcileStreamingMessages } from "../reconcile";
+
+function seedConversation(): string {
+  const id = randomUUID();
+  const now = new Date();
+  db.insert(conversations)
+    .values({
+      id,
+      runtimeId: "test-runtime",
+      status: "active",
+      createdAt: now,
+      updatedAt: now,
+    })
+    .run();
+  return id;
+}
+
+function seedMessage(opts: {
+  conversationId: string;
+  status: "streaming" | "complete" | "error";
+  content: string;
+  createdAt: Date;
+}): string {
+  const id = randomUUID();
+  db.insert(chatMessages)
+    .values({
+      id,
+      conversationId: opts.conversationId,
+      role: "assistant",
+      content: opts.content,
+      status: opts.status,
+      createdAt: opts.createdAt,
+    })
+    .run();
+  return id;
+}
+
+describe("reconcileStreamingMessages", () => {
+  beforeEach(() => {
+    // Isolate each test
+    db.delete(chatMessages).run();
+    db.delete(conversations).run();
+  });
+
+  it("sweeps a 20-min-old streaming row with empty content to error state with fallback", async () => {
+    const convId = seedConversation();
+    const twentyMinAgo = new Date(Date.now() - 20 * 60 * 1000);
+    const msgId = seedMessage({
+      conversationId: convId,
+      status: "streaming",
+      content: "",
+      createdAt: twentyMinAgo,
+    });
+
+    const swept = await reconcileStreamingMessages();
+
+    expect(swept).toBe(1);
+    const row = db
+      .select()
+      .from(chatMessages)
+      .where(eq(chatMessages.id, msgId))
+      .get();
+    expect(row?.status).toBe("error");
+    expect(row?.content).toMatch(/Interrupted/i);
+    expect(row?.content.length).toBeGreaterThan(0);
+  });
+
+  it("leaves a 30-second-old streaming row untouched", async () => {
+    const convId = seedConversation();
+    const thirtySecAgo = new Date(Date.now() - 30 * 1000);
+    const msgId = seedMessage({
+      conversationId: convId,
+      status: "streaming",
+      content: "",
+      createdAt: thirtySecAgo,
+    });
+
+    const swept = await reconcileStreamingMessages();
+
+    expect(swept).toBe(0);
+    const row = db
+      .select()
+      .from(chatMessages)
+      .where(eq(chatMessages.id, msgId))
+      .get();
+    expect(row?.status).toBe("streaming");
+    expect(row?.content).toBe("");
+  });
+
+  it("preserves partial content when sweeping old streaming row", async () => {
+    const convId = seedConversation();
+    const twentyMinAgo = new Date(Date.now() - 20 * 60 * 1000);
+    const msgId = seedMessage({
+      conversationId: convId,
+      status: "streaming",
+      content: "Here is what I found so",
+      createdAt: twentyMinAgo,
+    });
+
+    await reconcileStreamingMessages();
+
+    const row = db
+      .select()
+      .from(chatMessages)
+      .where(eq(chatMessages.id, msgId))
+      .get();
+    expect(row?.status).toBe("error");
+    expect(row?.content).toBe("Here is what I found so");
+  });
+
+  it("leaves complete messages untouched regardless of age", async () => {
+    const convId = seedConversation();
+    const twentyMinAgo = new Date(Date.now() - 20 * 60 * 1000);
+    const msgId = seedMessage({
+      conversationId: convId,
+      status: "complete",
+      content: "Finished response",
+      createdAt: twentyMinAgo,
+    });
+
+    const swept = await reconcileStreamingMessages();
+
+    expect(swept).toBe(0);
+    const row = db
+      .select()
+      .from(chatMessages)
+      .where(eq(chatMessages.id, msgId))
+      .get();
+    expect(row?.status).toBe("complete");
+    expect(row?.content).toBe("Finished response");
+  });
+});

package/src/lib/chat/__tests__/stream-telemetry.test.ts

@@ -0,0 +1,151 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import {
+  recordTermination,
+  readTerminations,
+  countTerminations,
+  __resetForTesting,
+} from "../stream-telemetry";
+
+describe("stream-telemetry ring buffer", () => {
+  beforeEach(() => {
+    __resetForTesting();
+  });
+
+  it("returns [] before any events are recorded", () => {
+    expect(readTerminations()).toEqual([]);
+  });
+
+  it("records events in chronological order", () => {
+    recordTermination({
+      reason: "stream.completed",
+      conversationId: "c1",
+      messageId: "m1",
+      durationMs: 100,
+    });
+    recordTermination({
+      reason: "stream.aborted.client",
+      conversationId: "c2",
+      messageId: "m2",
+      durationMs: 50,
+    });
+
+    const events = readTerminations();
+    expect(events).toHaveLength(2);
+    expect(events[0].reason).toBe("stream.completed");
+    expect(events[1].reason).toBe("stream.aborted.client");
+    expect(events[0].timestamp).toBeLessThanOrEqual(events[1].timestamp);
+  });
+
+  it("stamps each event with a timestamp", () => {
+    const before = Date.now();
+    recordTermination({
+      reason: "stream.completed",
+      conversationId: "c1",
+      messageId: "m1",
+      durationMs: 0,
+    });
+    const after = Date.now();
+    const events = readTerminations();
+    expect(events[0].timestamp).toBeGreaterThanOrEqual(before);
+    expect(events[0].timestamp).toBeLessThanOrEqual(after);
+  });
+
+  it("wraps around after 500 events, preserving newest-500 in order", () => {
+    // Write 520 events — first 20 should be evicted.
+    for (let i = 0; i < 520; i++) {
+      recordTermination({
+        reason: "stream.completed",
+        conversationId: `c${i}`,
+        messageId: `m${i}`,
+        durationMs: i,
+      });
+    }
+
+    const events = readTerminations();
+    expect(events).toHaveLength(500);
+    // Oldest surviving event should be #20; newest should be #519.
+    expect(events[0].conversationId).toBe("c20");
+    expect(events[0].durationMs).toBe(20);
+    expect(events[499].conversationId).toBe("c519");
+    expect(events[499].durationMs).toBe(519);
+  });
+
+  it("countTerminations groups by reason code across the full buffer", () => {
+    recordTermination({ reason: "stream.completed", conversationId: "c", messageId: "m", durationMs: 1 });
+    recordTermination({ reason: "stream.completed", conversationId: "c", messageId: "m", durationMs: 1 });
+    recordTermination({ reason: "stream.aborted.client", conversationId: "c", messageId: "m", durationMs: 1 });
+    recordTermination({ reason: "stream.finalized.error", conversationId: "c", messageId: "m", durationMs: 1, error: "boom" });
+    recordTermination({ reason: "stream.abandoned", conversationId: "c", messageId: "m", durationMs: 42 });
+
+    const counts = countTerminations();
+    expect(counts["stream.completed"]).toBe(2);
+    expect(counts["stream.aborted.client"]).toBe(1);
+    expect(counts["stream.finalized.error"]).toBe(1);
+    expect(counts["stream.abandoned"]).toBe(1);
+    expect(counts["stream.aborted.signal"]).toBe(0);
+    expect(counts["stream.reconciled.stale"]).toBe(0);
+  });
+
+  it("stream.abandoned is a valid reason code for iterator abandonment", () => {
+    // finalizeStreamingMessage records this when the engine's happy and
+    // catch paths both missed the termination — the canonical "gap"
+    // indicator. Make sure it round-trips through the buffer.
+    recordTermination({
+      reason: "stream.abandoned",
+      conversationId: "c1",
+      messageId: "m1",
+      durationMs: 100,
+      error: "no content streamed before abandonment",
+    });
+    const events = readTerminations();
+    expect(events[0].reason).toBe("stream.abandoned");
+    expect(events[0].error).toBe("no content streamed before abandonment");
+  });
+
+  it("countTerminations honors the windowMs filter", async () => {
+    recordTermination({ reason: "stream.completed", conversationId: "c", messageId: "m", durationMs: 1 });
+    // Wait a few ms so the second event has a strictly later timestamp.
+    await new Promise((r) => setTimeout(r, 10));
+    const midpoint = Date.now();
+    await new Promise((r) => setTimeout(r, 10));
+    recordTermination({ reason: "stream.completed", conversationId: "c", messageId: "m", durationMs: 1 });
+
+    // Use a window that only includes the second event.
+    const windowMs = Date.now() - midpoint + 5;
+    const counts = countTerminations(windowMs);
+    expect(counts["stream.completed"]).toBe(1);
+  });
+
+  it("readTerminations returns a copy, not a live reference", () => {
+    recordTermination({ reason: "stream.completed", conversationId: "c", messageId: "m", durationMs: 1 });
+    const first = readTerminations();
+    recordTermination({ reason: "stream.completed", conversationId: "c2", messageId: "m2", durationMs: 1 });
+    // first snapshot should still have only the initial event.
+    expect(first).toHaveLength(1);
+    expect(readTerminations()).toHaveLength(2);
+  });
+
+  it("records optional error strings on error events", () => {
+    recordTermination({
+      reason: "stream.finalized.error",
+      conversationId: "c",
+      messageId: "m",
+      durationMs: 42,
+      error: "boom",
+    });
+    expect(readTerminations()[0].error).toBe("boom");
+  });
+
+  it("allows null conversationId / messageId / durationMs for edge cases", () => {
+    recordTermination({
+      reason: "stream.reconciled.stale",
+      conversationId: null,
+      messageId: null,
+      durationMs: null,
+    });
+    const events = readTerminations();
+    expect(events[0].conversationId).toBeNull();
+    expect(events[0].messageId).toBeNull();
+    expect(events[0].durationMs).toBeNull();
+  });
+});

package/src/lib/chat/active-streams.ts

@@ -0,0 +1,27 @@
+/**
+ * In-memory tracker for chat conversations that currently have an SSE stream
+ * in flight. Used by the scheduler tick loop to apply a soft pressure signal
+ * — when chat is active, new schedule firings are deferred by N seconds to
+ * keep the Node event loop responsive for the user's conversation.
+ *
+ * Module-level state; single-process (same Node instance as the scheduler).
+ * Must NOT be persisted — crash recovery relies on the set starting empty.
+ */
+
+const activeStreams = new Set<string>();
+
+export function registerChatStream(conversationId: string): void {
+  activeStreams.add(conversationId);
+}
+
+export function unregisterChatStream(conversationId: string): void {
+  activeStreams.delete(conversationId);
+}
+
+export function getActiveChatStreamCount(): number {
+  return activeStreams.size;
+}
+
+export function isAnyChatStreaming(): boolean {
+  return activeStreams.size > 0;
+}

package/src/lib/chat/codex-engine.ts

@@ -2,7 +2,10 @@ import { db } from "@/lib/db";
 import { projects } from "@/lib/db/schema";
 import { eq } from "drizzle-orm";
 import { CodexAppServerClient } from "@/lib/agents/runtime/codex-app-server-client";
-import { getOpenAIApiKey } from "@/lib/settings/openai-auth";
+import {
+  ensureOpenAICodexClientAuthenticated,
+  resolveOpenAICodexAuthContext,
+} from "@/lib/agents/runtime/openai-codex-auth";
 import {
   extractUsageSnapshot,
   mergeUsageSnapshot,
@@ -128,11 +131,17 @@ export async function* sendCodexMessage(
   });
 
   // Get OpenAI API key
-  const { apiKey } = await getOpenAIApiKey();
-  if (!apiKey) {
-    await updateMessageContent(assistantMsg.id, "OpenAI API key is not configured. Add it in Settings → Auth.");
+  let auth;
+  try {
+    auth = await resolveOpenAICodexAuthContext();
+  } catch (error) {
+    const message =
+      error instanceof Error
+        ? error.message
+        : "OpenAI Codex authentication is not configured.";
+    await updateMessageContent(assistantMsg.id, message);
     await updateMessageStatus(assistantMsg.id, "error");
-    yield { type: "error", message: "OpenAI API key is not configured. Add it in Settings → Auth." };
+    yield { type: "error", message };
     return;
   }
 
@@ -164,20 +173,10 @@ export async function* sendCodexMessage(
   }
 
   try {
-    client = await CodexAppServerClient.connect({
-      cwd: workspace.cwd,
-      env: { OPENAI_API_KEY: apiKey },
-    });
+    client = await auth.connect(workspace.cwd);
 
     // Initialize and authenticate
-    await client.request("initialize", {
-      clientInfo: { name: "Stagent", version: "0.1.1" },
-      capabilities: null,
-    });
-    await client.request("account/login/start", {
-      type: "apiKey",
-      apiKey,
-    });
+    await ensureOpenAICodexClientAuthenticated(client, auth);
 
     // Validate model availability against what the user's account supports
     let validatedModel: string | undefined;

package/src/lib/chat/context-builder.ts

@@ -108,7 +108,7 @@ async function buildTier2(projectId?: string | null): Promise<string> {
   if (recentTasks.length > 0) {
     parts.push("\n### Recent Tasks");
     for (const t of recentTasks) {
-      parts.push(`- [${t.status}] ${t.title} (id: ${t.id.slice(0, 8)})`);
+      parts.push(`- [${t.status}] ${t.title} (id: ${t.id})`);
     }
   }
 
@@ -123,7 +123,7 @@ async function buildTier2(projectId?: string | null): Promise<string> {
   if (activeWorkflows.length > 0) {
     parts.push("\n### Workflows");
     for (const w of activeWorkflows) {
-      parts.push(`- [${w.status}] ${w.name} (id: ${w.id.slice(0, 8)})`);
+      parts.push(`- [${w.status}] ${w.name} (id: ${w.id})`);
     }
   }
 
@@ -137,7 +137,7 @@ async function buildTier2(projectId?: string | null): Promise<string> {
   if (docs.length > 0) {
     parts.push(`\n### Documents (${docs.length})`);
     for (const d of docs) {
-      parts.push(`- ${d.filename} (id: ${d.id.slice(0, 8)})`);
+      parts.push(`- ${d.filename} (id: ${d.id})`);
     }
   }
 
@@ -285,6 +285,7 @@ async function buildTier3(mentions: MentionReference[]): Promise<string> {
   return truncateToTokenBudget(text, TIER_3_BUDGET);
 }
 
+
 // ── Public API ─────────────────────────────────────────────────────────
 
 export interface ChatContext {
@@ -312,6 +313,7 @@ export async function buildChatContext(opts: {
   const tier0 = buildTier0(opts.projectName, opts.workspace);
 
   const systemParts = [tier0];
+
   if (tier3) systemParts.push(tier3);
   if (tier2) systemParts.push(tier2);