u-foo 1.9.7 → 2.1.0

package/src/agent/internalRunner.js

@@ -6,11 +6,32 @@ const EventBus = require("../bus");
 const { runCliAgent } = require("./cliRunner");
 const { normalizeCliOutput } = require("./normalizeOutput");
 const { createActivityStatePublisher } = require("./activityStatePublisher");
+const { loadConfig, normalizeCodexInternalThreadMode } = require("../config");
+const {
+  createCodexThreadProvider,
+  defaultCodexTransportStreamFactory,
+} = require("./codexThreadProvider");
+const {
+  createClaudeThreadProvider,
+  defaultClaudeTransportStreamFactory,
+} = require("./claudeThreadProvider");
+const { resolveClaudeUpstreamCredentials } = require("./credentials/claude");
+const { buildUpstreamAuthFromCredential } = require("./credentials");
+const { listToolsForCallerTier, CALLER_TIERS } = require("../tools");
+const { redactToolCallPayload, redactSecrets } = require("../providerapi/redactor");
 
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+function normalizeWorkerThreadToolMode(value = "") {
+  const raw = String(value || "").trim().toLowerCase();
+  if (raw === "worker-tier01" || raw === "tier01" || raw === "enabled" || raw === "1" || raw === "true") {
+    return "worker-tier01";
+  }
+  return "disabled";
+}
+
 function buildEnv(agentType, sessionId, publisher, nickname) {
   const env = { ...process.env };
   env.AI_BUS_PUBLISHER = publisher || env.AI_BUS_PUBLISHER || "";
@@ -63,6 +84,18 @@ function createBusSender(projectRoot, subscriber) {
   return { enqueue, flush };
 }
 
+function shouldFallbackToLegacyThreadProvider(err, provider) {
+  if (provider !== "claude-cli" || !err || typeof err !== "object") {
+    return false;
+  }
+  const code = String(err.code || "").trim().toUpperCase();
+  return (
+    code === "CLAUDE_AUTH_UNAVAILABLE"
+    || code === "CLAUDE_OAUTH_SCHEMA_UNSUPPORTED"
+    || code === "ANTHROPIC_SDK_UNAVAILABLE"
+  );
+}
+
 function drainQueue(queueFile) {
   if (!fs.existsSync(queueFile)) return [];
   const processingFile = `${queueFile}.processing.${process.pid}.${Date.now()}`;
@@ -96,7 +129,19 @@ function drainQueue(queueFile) {
   return content.split(/\r?\n/).filter(Boolean);
 }
 
-async function handleEvent(projectRoot, agentType, provider, model, subscriber, nickname, evt, cliSessionState, busSender) {
+async function handleEvent(
+  projectRoot,
+  agentType,
+  provider,
+  model,
+  subscriber,
+  nickname,
+  evt,
+  cliSessionState,
+  busSender,
+  extraArgs = [],
+  threadRuntime = null
+) {
   if (!evt || !evt.data || !evt.data.message) return;
   const prompt = evt.data.message;
   const publisher = evt.publisher || "unknown";
@@ -111,6 +156,21 @@ async function handleEvent(projectRoot, agentType, provider, model, subscriber,
     busSender.enqueue(publisher, JSON.stringify({ stream: true, delta: text }));
   };
 
+  if (threadRuntime && threadRuntime.enabled && threadRuntime.thread) {
+    const threadedResult = await handleThreadedEvent({
+      agentType,
+      provider,
+      publisher,
+      prompt,
+      busSender,
+      emitStreamDelta,
+      threadRuntime,
+    });
+    if (!threadedResult || !threadedResult.fallbackToLegacy) {
+      return;
+    }
+  }
+
   let res = await runCliAgent({
     provider,
     model,
@@ -118,6 +178,7 @@ async function handleEvent(projectRoot, agentType, provider, model, subscriber,
     sessionId: cliSessionState.cliSessionId,
     sandbox,
     cwd: projectRoot,
+    extraArgs,
     onStreamDelta: emitStreamDelta,
   });
 
@@ -136,6 +197,7 @@ async function handleEvent(projectRoot, agentType, provider, model, subscriber,
         sessionId: null, // Let runCliAgent generate new session
         sandbox,
         cwd: projectRoot,
+        extraArgs,
         onStreamDelta: emitStreamDelta,
       });
     }
@@ -175,7 +237,271 @@ async function handleEvent(projectRoot, agentType, provider, model, subscriber,
   await busSender.flush();
 }
 
-async function runInternalRunner({ projectRoot, agentType = "codex" }) {
+async function handleThreadedEvent({
+  agentType,
+  provider,
+  publisher,
+  prompt,
+  busSender,
+  emitStreamDelta,
+  threadRuntime,
+}) {
+  try {
+    for await (const event of threadRuntime.thread.runStreamed(prompt, {})) {
+      if (!event || typeof event !== "object") continue;
+      if (event.type === "text_delta" && event.delta) {
+        emitStreamDelta(event.delta);
+      } else if (event.type === "turn_failed") {
+        throw new Error(event.error || `thread turn failed for ${agentType}`);
+      }
+    }
+
+    busSender.enqueue(
+      publisher,
+      JSON.stringify({ stream: true, done: true, reason: "complete" })
+    );
+    await busSender.flush();
+  } catch (err) {
+    if (shouldFallbackToLegacyThreadProvider(err, provider)) {
+      return { fallbackToLegacy: true };
+    }
+    if (threadRuntime && typeof threadRuntime.rebuildThread === "function") {
+      await threadRuntime.rebuildThread();
+    }
+    busSender.enqueue(
+      publisher,
+      JSON.stringify({
+        stream: true,
+        delta: `[internal:${agentType}] error: ${err && err.message ? err.message : "unknown error"}`,
+      })
+    );
+    busSender.enqueue(
+      publisher,
+      JSON.stringify({ stream: true, done: true, reason: "error" })
+    );
+    await busSender.flush();
+    return { fallbackToLegacy: false };
+  }
+}
+
+function getCodexThreadMode(projectRoot) {
+  const envValue = process.env.UFOO_CODEX_INTERNAL_THREAD_MODE;
+  if (typeof envValue === "string" && envValue.trim()) {
+    return normalizeCodexInternalThreadMode(envValue);
+  }
+  return loadConfig(projectRoot).codexInternalThreadMode;
+}
+
+function getWorkerThreadToolMode() {
+  return normalizeWorkerThreadToolMode(process.env.UFOO_CODEX_INTERNAL_THREAD_TOOLS);
+}
+
+function buildWorkerThreadToolRuntime({ projectRoot, subscriber, observer }) {
+  const mode = getWorkerThreadToolMode();
+  if (mode !== "worker-tier01") {
+    return {
+      enabled: false,
+      mode,
+      tools: [],
+      executeToolCall: null,
+    };
+  }
+
+  const eventBus = new EventBus(projectRoot);
+  const toolDefinitions = listToolsForCallerTier(CALLER_TIERS.WORKER);
+  const toolsByName = new Map(toolDefinitions.map((tool) => [tool.name, tool]));
+  const emitAudit = (phase, payload) => {
+    if (observer && typeof observer.onToolCall === "function") {
+      try { observer.onToolCall({ phase, payload }); } catch { /* ignore observer errors */ }
+    }
+  };
+
+  return {
+    enabled: toolDefinitions.length > 0,
+    mode,
+    tools: toolDefinitions.map((tool) => ({
+      name: tool.name,
+      description: tool.description,
+      input_schema: tool.input_schema,
+    })),
+    // Keep a shared-handler executor ready for a future continuation-capable SDK path.
+    // The current Codex seam injects tool descriptors only and does not execute live
+    // tool calls inside the SDK stream yet.
+    async executeToolCall(toolCall = {}) {
+      const name = String(toolCall.name || "").trim();
+      const definition = toolsByName.get(name);
+      const rawArgs = toolCall.arguments || toolCall.args || {};
+      // Slice 1 (§10.7 tool pre-call): build a redacted audit envelope before
+      // the handler receives args, so observability consumers never see raw secrets.
+      const redactedPayload = redactToolCallPayload({
+        name,
+        args: rawArgs,
+        tool_call_id: toolCall.tool_call_id || toolCall.toolCallId || "",
+        caller_tier: CALLER_TIERS.WORKER,
+      });
+      emitAudit("pre_call", redactedPayload);
+      if (!definition) {
+        const errorResult = {
+          ok: false,
+          error: {
+            code: "unsupported_tool",
+            message: `worker tool is unavailable: ${name}`,
+          },
+        };
+        emitAudit("post_call", { ...redactedPayload, result: errorResult });
+        return errorResult;
+      }
+
+      try {
+        const result = await definition.handler({
+          caller_tier: CALLER_TIERS.WORKER,
+          projectRoot,
+          subscriber,
+          eventBus,
+        }, rawArgs);
+        const safeResult = redactSecrets(result);
+        emitAudit("post_call", { ...redactedPayload, result: safeResult });
+        return safeResult;
+      } catch (err) {
+        const errorResult = {
+          ok: false,
+          error: {
+            code: err && err.code ? err.code : "tool_execution_failed",
+            message: err && err.message ? err.message : String(err || "tool execution failed"),
+          },
+        };
+        const safeErrorResult = redactSecrets(errorResult);
+        emitAudit("post_call", { ...redactedPayload, result: safeErrorResult });
+        return safeErrorResult;
+      }
+    },
+  };
+}
+
+function getClaudeThreadMode() {
+  const envValue = process.env.UFOO_CLAUDE_INTERNAL_THREAD_MODE;
+  const raw = String(envValue || "").trim().toLowerCase();
+  if (raw === "api") return "api";
+  return "legacy";
+}
+
+function buildClaudeAuthProvider(projectRoot) {
+  const config = loadConfig(projectRoot);
+  return async () => {
+    const credential = await resolveClaudeUpstreamCredentials({
+      profile: config.claudeOauthProfile,
+      tokenPath: config.claudeOauthTokenPath,
+      refreshWindowMs: Number(config.claudeOauthRefreshWindowSec || 300) * 1000,
+    });
+    return buildUpstreamAuthFromCredential(credential);
+  };
+}
+
+function createThreadRuntime({ projectRoot, provider, model, extraArgs = [], subscriber = "" }) {
+  const disabledRuntime = {
+    enabled: false,
+    thread: null,
+    toolRuntime: { enabled: false, mode: "disabled", tools: [] },
+    close: async () => {},
+    rebuildThread: async () => {},
+  };
+
+  if (provider === "codex-cli") {
+    if (getCodexThreadMode(projectRoot) !== "sdk") {
+      return disabledRuntime;
+    }
+
+    try {
+      const toolRuntime = buildWorkerThreadToolRuntime({
+        projectRoot,
+        subscriber,
+      });
+      let providerInstance = createCodexThreadProvider({
+        model,
+        cwd: projectRoot,
+        extraArgs,
+        tools: toolRuntime.tools,
+        streamFactory: defaultCodexTransportStreamFactory,
+      });
+      let thread = providerInstance.startThread();
+
+      return {
+        enabled: true,
+        toolRuntime,
+        get thread() {
+          return thread;
+        },
+        async rebuildThread() {
+          if (thread && typeof thread.close === "function") {
+            await thread.close();
+          }
+          providerInstance = createCodexThreadProvider({
+            model,
+            cwd: projectRoot,
+            extraArgs,
+            tools: toolRuntime.tools,
+            streamFactory: defaultCodexTransportStreamFactory,
+          });
+          thread = providerInstance.startThread();
+        },
+        async close() {
+          if (thread && typeof thread.close === "function") {
+            await thread.close();
+          }
+        },
+      };
+    } catch {
+      return disabledRuntime;
+    }
+  }
+
+  if (provider === "claude-cli") {
+    if (getClaudeThreadMode() !== "api") {
+      return disabledRuntime;
+    }
+    if (typeof createClaudeThreadProvider !== "function" || typeof resolveClaudeUpstreamCredentials !== "function") {
+      return disabledRuntime;
+    }
+
+    try {
+      let providerInstance = createClaudeThreadProvider({
+        model,
+        authProvider: buildClaudeAuthProvider(projectRoot),
+        streamFactory: defaultClaudeTransportStreamFactory,
+      });
+      let thread = providerInstance.startThread();
+
+      return {
+        enabled: true,
+        get thread() {
+          return thread;
+        },
+        async rebuildThread() {
+          if (thread && typeof thread.close === "function") {
+            await thread.close();
+          }
+          providerInstance = createClaudeThreadProvider({
+            model,
+            authProvider: buildClaudeAuthProvider(projectRoot),
+            streamFactory: defaultClaudeTransportStreamFactory,
+          });
+          thread = providerInstance.startThread();
+        },
+        async close() {
+          if (thread && typeof thread.close === "function") {
+            await thread.close();
+          }
+        },
+      };
+    } catch {
+      return disabledRuntime;
+    }
+  }
+
+  return disabledRuntime;
+}
+
+async function runInternalRunner({ projectRoot, agentType = "codex", extraArgs = [] }) {
   // Internal runner 必须由 daemon 启动，UFOO_SUBSCRIBER_ID 应该已经设置
   const { subscriber, agentType: parsedAgentType, sessionId } = parseSubscriberId();
   const nickname = process.env.UFOO_NICKNAME || "";
@@ -189,6 +515,13 @@ async function runInternalRunner({ projectRoot, agentType = "codex" }) {
   const provider = normalizedAgentType === "codex" ? "codex-cli" : "claude-cli";
   const model = process.env.UFOO_AGENT_MODEL || "";
   const busSender = createBusSender(projectRoot, subscriber);
+  const threadRuntime = createThreadRuntime({
+    projectRoot,
+    provider,
+    model,
+    extraArgs,
+    subscriber,
+  });
 
   // Session state management for CLI continuity
   // Use stable path based on nickname (if exists) or agent type, NOT subscriber ID
@@ -280,7 +613,9 @@ async function runInternalRunner({ projectRoot, agentType = "codex" }) {
               nickname,
               evt,
               cliSessionState,
-              busSender
+              busSender,
+              extraArgs,
+              threadRuntime
             );
           }
 
@@ -310,10 +645,20 @@ async function runInternalRunner({ projectRoot, agentType = "codex" }) {
     // eslint-disable-next-line no-await-in-loop
     await sleep(1000);
   }
+
+  await threadRuntime.close();
 }
 
 module.exports = {
   runInternalRunner,
   createBusSender,
   handleEvent,
+  createThreadRuntime,
+  getCodexThreadMode,
+  getWorkerThreadToolMode,
+  buildWorkerThreadToolRuntime,
+  normalizeWorkerThreadToolMode,
+  getClaudeThreadMode,
+  buildClaudeAuthProvider,
+  shouldFallbackToLegacyThreadProvider,
 };

package/src/agent/loopObservability.js

@@ -0,0 +1,190 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { getUfooPaths } = require("../ufoo/paths");
+const { redactSecrets } = require("../providerapi/redactor");
+
+const LOOP_EVENT_SCHEMA_VERSION = 1;
+
+function getLoopObservabilityPaths(projectRoot) {
+  const { agentDir } = getUfooPaths(projectRoot);
+  return {
+    eventsFile: path.join(agentDir, "ufoo-agent.loop-events.jsonl"),
+    auditFile: path.join(agentDir, "ufoo-agent.audit.jsonl"),
+  };
+}
+
+function getShadowObservabilityPaths(projectRoot, now = new Date()) {
+  const { ufooDir } = getUfooPaths(projectRoot);
+  const stamp = now.toISOString().slice(0, 10);
+  return {
+    shadowDir: path.join(ufooDir, "shadow"),
+    diffFile: path.join(ufooDir, "shadow", `diff-${stamp}.jsonl`),
+  };
+}
+
+function appendJsonLine(file, payload) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.appendFileSync(file, `${JSON.stringify(redactSecrets(payload))}\n`, "utf8");
+}
+
+function appendShadowDiff(projectRoot, payload = {}, now = new Date()) {
+  if (!projectRoot) return null;
+  const { diffFile } = getShadowObservabilityPaths(projectRoot, now);
+  appendJsonLine(diffFile, {
+    schema_version: LOOP_EVENT_SCHEMA_VERSION,
+    ts: now.toISOString(),
+    shadow_only: true,
+    ...payload,
+  });
+  return diffFile;
+}
+
+function createLoopObserver({ projectRoot, enabled = true, defaults = {} } = {}) {
+  const paths = projectRoot ? getLoopObservabilityPaths(projectRoot) : null;
+  const baseDefaults = defaults && typeof defaults === "object" ? { ...defaults } : {};
+
+  function emit(event, payload = {}) {
+    if (!enabled || !paths) return;
+    appendJsonLine(paths.eventsFile, {
+      schema_version: LOOP_EVENT_SCHEMA_VERSION,
+      ts: new Date().toISOString(),
+      event: String(event || "").trim() || "unknown",
+      ...baseDefaults,
+      ...payload,
+    });
+  }
+
+  function audit(payload = {}) {
+    if (!enabled || !paths) return;
+    appendJsonLine(paths.auditFile, {
+      schema_version: LOOP_EVENT_SCHEMA_VERSION,
+      ts: new Date().toISOString(),
+      ...baseDefaults,
+      ...payload,
+    });
+  }
+
+  return {
+    emit,
+    audit,
+    paths,
+  };
+}
+
+function readRecentLoopSummary(projectRoot, options = {}) {
+  if (!projectRoot) return null;
+  const { eventsFile } = getLoopObservabilityPaths(projectRoot);
+  if (!fs.existsSync(eventsFile)) return null;
+
+  const maxLines = Number.isFinite(options.maxLines) && options.maxLines > 0
+    ? Math.floor(options.maxLines)
+    : 400;
+
+  let rows = [];
+  try {
+    rows = fs.readFileSync(eventsFile, "utf8")
+      .split(/\r?\n/)
+      .filter(Boolean)
+      .slice(-maxLines)
+      .map((line) => {
+        try {
+          return JSON.parse(line);
+        } catch {
+          return null;
+        }
+      })
+      .filter(Boolean);
+  } catch {
+    return null;
+  }
+
+  if (rows.length === 0) return null;
+
+  let startIndex = 0;
+  let endIndex = rows.length;
+  let terminalIndex = -1;
+  for (let i = rows.length - 1; i >= 0; i -= 1) {
+    if (rows[i] && rows[i].event === "loop_terminal") {
+      terminalIndex = i;
+      endIndex = i + 1;
+      break;
+    }
+  }
+  if (terminalIndex >= 0) {
+    for (let i = terminalIndex - 1; i >= 0; i -= 1) {
+      if (rows[i] && rows[i].event === "loop_terminal") {
+        startIndex = i + 1;
+        break;
+      }
+    }
+  }
+
+  const segment = rows.slice(startIndex, endIndex);
+  if (segment.length === 0) return null;
+
+  const toolCounts = new Map();
+  const summary = {
+    status: terminalIndex >= 0 ? "completed" : "in_progress",
+    event_count: segment.length,
+    model_calls: 0,
+    rounds: 0,
+    tool_calls: 0,
+    input_tokens: 0,
+    output_tokens: 0,
+    cache_read_tokens: 0,
+    cache_creation_tokens: 0,
+    total_tokens: 0,
+    total_latency_ms: 0,
+    first_token_ms: 0,
+    terminal_reason: "",
+    started_at: "",
+    ended_at: "",
+    tools: [],
+  };
+
+  for (const row of segment) {
+    if (!summary.started_at && row.ts) summary.started_at = String(row.ts);
+    if (row.ts) summary.ended_at = String(row.ts);
+    if (row.event === "model_call") {
+      summary.model_calls += 1;
+      summary.rounds = Math.max(summary.rounds, Number(row.round) || 0);
+      summary.input_tokens += Number(row.input_tokens) || 0;
+      summary.output_tokens += Number(row.output_tokens) || 0;
+      summary.cache_read_tokens += Number(row.cache_read_tokens) || 0;
+      summary.cache_creation_tokens += Number(row.cache_creation_tokens) || 0;
+      summary.total_latency_ms += Number(row.latency_ms) || 0;
+      summary.first_token_ms += Number(row.first_token_ms) || 0;
+    } else if (row.event === "tool_call") {
+      summary.tool_calls += 1;
+      const name = String(row.tool_name || "").trim() || "unknown";
+      toolCounts.set(name, (toolCounts.get(name) || 0) + 1);
+    } else if (row.event === "loop_terminal") {
+      summary.terminal_reason = String(row.terminal_reason || "").trim();
+      if ((Number(row.rounds) || 0) > 0) summary.rounds = Number(row.rounds) || summary.rounds;
+      if ((Number(row.tool_calls) || 0) >= 0) summary.tool_calls = Number(row.tool_calls) || summary.tool_calls;
+      if ((Number(row.total_tokens) || 0) > 0) summary.total_tokens = Number(row.total_tokens) || 0;
+      if ((Number(row.total_latency_ms) || 0) > 0) summary.total_latency_ms = Number(row.total_latency_ms) || 0;
+    }
+  }
+
+  if (summary.total_tokens <= 0) {
+    summary.total_tokens = summary.input_tokens + summary.output_tokens;
+  }
+
+  summary.tools = Array.from(toolCounts.entries())
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .map(([name, count]) => ({ name, count }));
+
+  return summary;
+}
+
+module.exports = {
+  LOOP_EVENT_SCHEMA_VERSION,
+  appendShadowDiff,
+  getLoopObservabilityPaths,
+  getShadowObservabilityPaths,
+  createLoopObserver,
+  readRecentLoopSummary,
+};