u-foo 1.9.8 → 2.2.0

package/src/daemon/promptRequest.js

@@ -6,6 +6,29 @@ const {
   consumeControllerInboxEntries,
 } = require("../report/store");
 const { isGlobalControllerProjectRoot } = require("../projects");
+const {
+  resolveGateRouterConfig,
+  shouldUseGateRouter,
+} = require("../controller/gateRouter");
+const {
+  CONTROLLER_MODES,
+  applyControllerModeForMessage,
+  resolveControllerMode,
+} = require("../controller/flags");
+const {
+  resolveLoopRuntimeOptions,
+  runPromptWithControllerLoop,
+} = require("../agent/loopRuntime");
+const {
+  appendShadowDiff,
+  createLoopObserver,
+} = require("../agent/loopObservability");
+const {
+  DEFAULT_SHADOW_SAMPLING_RATE,
+  createShadowBudgetBreaker,
+  createShadowGuard,
+  shouldSampleShadow,
+} = require("../controller/shadowGuard");
 
 function normalizeProjectRoute(route) {
   if (!route || typeof route !== "object") return null;
@@ -63,6 +86,29 @@ function buildPromptWithPrivateReports(prompt = "", reports = [], requestMeta =
   return lines.join("\n");
 }
 
+function normalizeMessageId(req = {}) {
+  return String(req.request_id || req.message_id || req.msg_id || req.id || "").trim();
+}
+
+function summarizeShadowPayload(payload = {}) {
+  if (!payload || typeof payload !== "object") {
+    return {
+      reply_present: false,
+      dispatch_count: 0,
+      ops_count: 0,
+      terminal_reason: "",
+    };
+  }
+  return {
+    reply_present: Boolean(payload.reply),
+    dispatch_count: Array.isArray(payload.dispatch) ? payload.dispatch.length : 0,
+    ops_count: Array.isArray(payload.ops) ? payload.ops.length : 0,
+    terminal_reason: payload.loop && typeof payload.loop === "object"
+      ? String(payload.loop.terminal_reason || "").trim()
+      : "",
+  };
+}
+
 async function handlePromptRequest(options = {}) {
   const {
     projectRoot,
@@ -72,10 +118,12 @@ async function handlePromptRequest(options = {}) {
     model,
     processManager = null,
     runPromptWithAssistant,
+    runPromptWithControllerLoop: injectedLoopRunner = runPromptWithControllerLoop,
     runUfooAgent,
-    runAssistantTask,
+    runUfooRouteAgent,
     dispatchMessages,
     handleOps,
+    ackBus,
     markPending = () => {},
     reportTaskStatus = () => {},
     forwardProjectPrompt = null,
@@ -84,8 +132,40 @@ async function handlePromptRequest(options = {}) {
 
   log(`prompt ${String(req.text || "").slice(0, 200)}`);
   const requestMeta = req.request_meta && typeof req.request_meta === "object" ? req.request_meta : {};
+  const messageId = normalizeMessageId(req);
+  const requestedControllerMode = String(
+    requestMeta.controller_mode || requestMeta.agent_execution_path || ""
+  ).trim();
+  let controllerMode = resolveControllerMode({
+    projectRoot,
+    requestedMode: requestedControllerMode,
+  });
   const isGlobalController = isGlobalControllerProjectRoot(projectRoot);
   const forcedProjectRoot = String(requestMeta.force_project_root || "").trim();
+  const resolvedLoopRuntime = resolveLoopRuntimeOptions();
+  if (controllerMode === CONTROLLER_MODES.LEGACY && resolvedLoopRuntime.enabled) {
+    controllerMode = CONTROLLER_MODES.LOOP;
+  }
+  const controllerObserver = createLoopObserver({
+    projectRoot,
+    enabled: true,
+    defaults: {
+      controller_mode: controllerMode,
+    },
+  });
+  const appliedControllerMode = applyControllerModeForMessage({
+    projectRoot,
+    nextMode: controllerMode,
+    messageId,
+  });
+  if (appliedControllerMode.transition) {
+    controllerObserver.emit("controller.flag.transition", appliedControllerMode.transition);
+  }
+  const loopRuntime = {
+    ...resolvedLoopRuntime,
+    enabled: controllerMode === CONTROLLER_MODES.LOOP,
+  };
+  const shadowEnabled = controllerMode === CONTROLLER_MODES.SHADOW;
 
   if (isGlobalController && forcedProjectRoot) {
     try {
@@ -131,26 +211,166 @@ async function handlePromptRequest(options = {}) {
   }
 
   const privateReports = listControllerInboxEntries(projectRoot, "ufoo-agent", { num: 100 });
-  const promptText = buildPromptWithPrivateReports(req.text || "", privateReports, requestMeta);
   const useGlobalProjectRouter = isGlobalController;
+  const promptRunner = runPromptWithAssistant;
+  const ufooAgentOptions = useGlobalProjectRouter ? { routingMode: "global-router" } : { controllerMode };
+  let nextRequestMeta = requestMeta;
+  if (!Object.prototype.hasOwnProperty.call(nextRequestMeta, "agent_execution_path") && controllerMode !== CONTROLLER_MODES.LEGACY) {
+    nextRequestMeta = {
+      ...nextRequestMeta,
+      agent_execution_path: controllerMode,
+    };
+  }
+
+  const logGateRouterEvent = (event, details = {}) => {
+    controllerObserver.emit(event, details);
+    if (typeof log !== "function") return;
+    log(`event ${JSON.stringify({ event, ...details })}`);
+  };
+
+  const attachGateRouterMeta = (reason, detail = {}) => {
+    nextRequestMeta = {
+      ...nextRequestMeta,
+      gate_router: {
+        attempted: true,
+        reason,
+        ...detail,
+      },
+    };
+  };
+
+  controllerObserver.emit("controller.prompt_path_selected", {
+    applied_from_msg_id: messageId,
+    shadow_enabled: shadowEnabled,
+    loop_enabled: loopRuntime.enabled,
+  });
+
+  const gateRouterEligibility = shouldUseGateRouter({
+    projectRoot,
+    prompt: req.text || "",
+    requestMeta: nextRequestMeta,
+  });
+  const gateRouterConfig = gateRouterEligibility.enabled
+    ? resolveGateRouterConfig({
+      projectRoot,
+      requestMeta,
+    })
+    : null;
+
+  if (!useGlobalProjectRouter && gateRouterEligibility.enabled && typeof runUfooRouteAgent === "function") {
+    logGateRouterEvent("controller.gate_router_attempted", {
+      flag: gateRouterEligibility.executionPath,
+      intent_reason: gateRouterEligibility.intent.reason,
+    });
+
+    const routed = await runUfooRouteAgent({
+      projectRoot,
+      prompt: req.text || "",
+      provider: gateRouterConfig.provider,
+      model: gateRouterConfig.model,
+      timeoutMs: gateRouterConfig.timeoutMs,
+    });
+
+    if (!routed || routed.ok !== true) {
+      attachGateRouterMeta("provider_error", {
+        error: routed && routed.error ? routed.error : "route_agent_failed",
+      });
+      logGateRouterEvent("controller.gate_router_upgraded", {
+        reason: "provider_error",
+        fallback_used: "main_router",
+      });
+    } else {
+      const route = routed.route || {};
+      const canDispatch = route.decision === "direct_dispatch"
+        && route.target && route.target !== "unknown"
+        && route.confidence >= gateRouterConfig.confidenceThreshold;
+
+      if (!canDispatch) {
+        const upgradeReason = route.decision && route.decision !== "direct_dispatch"
+          ? route.decision
+          : "low_confidence";
+        attachGateRouterMeta(upgradeReason, {
+          decision: route.decision || "",
+          target: route.target || "unknown",
+          confidence: Number(route.confidence || 0),
+          route_reason: route.reason || "",
+        });
+        logGateRouterEvent("controller.gate_router_upgraded", {
+          reason: upgradeReason,
+          decision: route.decision || "",
+          target: route.target || "unknown",
+          confidence: Number(route.confidence || 0),
+          fallback_used: "main_router",
+        });
+      } else {
+        const payload = {
+          reply: "",
+          dispatch: [{
+            target: route.target,
+            message: route.message || req.text || "",
+            injection_mode: route.injection_mode || "immediate",
+            source: "ufoo-agent-gate-router",
+          }],
+          ops: [],
+        };
+        try {
+          markPending(route.target);
+          await dispatchMessages(projectRoot, payload.dispatch);
+          consumeControllerInboxEntries(projectRoot, "ufoo-agent", privateReports);
+          logGateRouterEvent("controller.gate_router_completed", {
+            target: route.target,
+            confidence: Number(route.confidence || 0),
+            provider: routed.meta && routed.meta.provider ? routed.meta.provider : "",
+            model: routed.meta && routed.meta.model ? routed.meta.model : "",
+            fallback_used: "none",
+          });
+          socket.write(
+            `${JSON.stringify({
+              type: IPC_RESPONSE_TYPES.RESPONSE,
+              data: payload,
+              opsResults: [],
+            })}\n`,
+          );
+          return true;
+        } catch (err) {
+          attachGateRouterMeta("dispatch_failed", {
+            target: route.target,
+            confidence: Number(route.confidence || 0),
+            route_reason: route.reason || "",
+            error: err && err.message ? err.message : String(err),
+          });
+          logGateRouterEvent("controller.gate_router_upgraded", {
+            reason: "dispatch_failed",
+            target: route.target,
+            confidence: Number(route.confidence || 0),
+            fallback_used: "main_router",
+          });
+        }
+      }
+    }
+  }
+
+  const promptText = buildPromptWithPrivateReports(req.text || "", privateReports, nextRequestMeta);
 
   try {
-    const handled = await runPromptWithAssistant({
+    const handled = await promptRunner({
       projectRoot,
       prompt: promptText,
       provider,
       model,
       processManager,
       runUfooAgent,
-      runAssistantTask,
+      runPromptWithControllerLoop: injectedLoopRunner,
       dispatchMessages,
       handleOps,
+      ackBus,
       markPending,
       reportTaskStatus,
       maxAssistantLoops: 2,
       log,
-      ufooAgentOptions: useGlobalProjectRouter ? { routingMode: "global-router" } : {},
+      ufooAgentOptions,
       finalizeLocally: !useGlobalProjectRouter,
+      loopRuntime,
     });
 
     if (!handled.ok) {
@@ -164,6 +384,124 @@ async function handlePromptRequest(options = {}) {
       return false;
     }
 
+    let shadowResult = null;
+    if (shadowEnabled && !useGlobalProjectRouter && typeof injectedLoopRunner === "function") {
+      const shadowObserver = createLoopObserver({
+        projectRoot,
+        enabled: true,
+        defaults: {
+          controller_mode: controllerMode,
+          shadow_only: true,
+        },
+      });
+
+      const shadowSamplingRate = Number.isFinite(Number(requestMeta.shadow_sampling_rate))
+        ? Math.max(0, Math.min(1, Number(requestMeta.shadow_sampling_rate)))
+        : DEFAULT_SHADOW_SAMPLING_RATE;
+      const sampling = shouldSampleShadow({ messageId, samplingRate: shadowSamplingRate });
+      const budgetBreaker = createShadowBudgetBreaker({ projectRoot });
+      const budgetStatus = budgetBreaker.check();
+
+      if (!sampling.sampled) {
+        shadowObserver.emit("controller.shadow.skipped", {
+          applied_from_msg_id: messageId,
+          reason: "sampling_excluded",
+          sampling_rate: sampling.rate,
+        });
+      } else if (!budgetStatus.allowed) {
+        shadowObserver.emit("controller.shadow.skipped", {
+          applied_from_msg_id: messageId,
+          reason: budgetStatus.reason,
+          sampling_rate: sampling.rate,
+        });
+      } else {
+        const shadowGuard = createShadowGuard({ projectRoot });
+        const noOpExecutors = shadowGuard.buildNoOpExecutors();
+        const beforeSnapshot = shadowGuard.takeSnapshot();
+
+        shadowObserver.emit("controller.shadow.started", {
+          applied_from_msg_id: messageId,
+          primary_mode: CONTROLLER_MODES.LEGACY,
+          candidate_mode: CONTROLLER_MODES.LOOP,
+          sampling_rate: sampling.rate,
+        });
+
+        try {
+          shadowResult = await injectedLoopRunner({
+            projectRoot,
+            prompt: promptText,
+            provider,
+            model,
+            processManager,
+            runUfooAgent,
+            dispatchMessages: noOpExecutors.dispatchMessages,
+            handleOps: noOpExecutors.handleOps,
+            ackBus: noOpExecutors.ackBus,
+            markPending: noOpExecutors.markPending,
+            reportTaskStatus,
+            maxAssistantLoops: 2,
+            log,
+            ufooAgentOptions,
+            finalizeLocally: false,
+            loopRuntime: {
+              ...resolvedLoopRuntime,
+              enabled: true,
+            },
+            observer: shadowObserver,
+            observabilityDefaults: {
+              controller_mode: controllerMode,
+              shadow_only: true,
+            },
+          });
+        } catch (shadowErr) {
+          shadowResult = {
+            ok: false,
+            error: shadowErr && shadowErr.message ? shadowErr.message : String(shadowErr),
+          };
+        }
+
+        const assertion = shadowGuard.assertNoSideEffects(beforeSnapshot);
+        if (!assertion.ok) {
+          shadowObserver.emit("controller.shadow.violation", {
+            applied_from_msg_id: messageId,
+            violations: assertion.violations,
+          });
+        }
+
+        const loopSummary = shadowResult && shadowResult.payload && shadowResult.payload.loop
+          ? shadowResult.payload.loop
+          : null;
+        const totalInputTokens = loopSummary && typeof loopSummary.total_tokens === "number"
+          ? loopSummary.total_tokens
+          : 0;
+        budgetBreaker.record({ inputTokens: totalInputTokens });
+
+        const diffFile = appendShadowDiff(projectRoot, {
+          event: "controller.shadow.diff",
+          request_id: messageId,
+          primary_mode: CONTROLLER_MODES.LEGACY,
+          candidate_mode: CONTROLLER_MODES.LOOP,
+          sampling_rate: sampling.rate,
+          primary: summarizeShadowPayload(handled.payload),
+          shadow: shadowResult && shadowResult.ok
+            ? summarizeShadowPayload(shadowResult.payload)
+            : {
+              ok: false,
+              error: shadowResult && shadowResult.error ? String(shadowResult.error) : "shadow_run_failed",
+            },
+          side_effects_ok: assertion.ok,
+          side_effect_violations: assertion.violations,
+        });
+        shadowObserver.emit("controller.shadow.completed", {
+          applied_from_msg_id: messageId,
+          ok: shadowResult && shadowResult.ok === true,
+          diff_file: diffFile || "",
+          side_effects_ok: assertion.ok,
+          terminal_reason: loopSummary ? String(loopSummary.terminal_reason || "") : "",
+        });
+      }
+    }
+
     if (useGlobalProjectRouter) {
       const route = normalizeProjectRoute(handled.payload && handled.payload.project_route);
       if (route) {
@@ -216,6 +554,17 @@ async function handlePromptRequest(options = {}) {
     }
     const opsResults = handled.opsResults || [];
     log(`ok reply=${Boolean(payload.reply)} dispatch=${(payload.dispatch || []).length} ops=${(payload.ops || []).length}`);
+    controllerObserver.emit("controller.prompt_completed", {
+      applied_from_msg_id: messageId,
+      ok: true,
+      shadow_enabled: shadowEnabled,
+      shadow_ok: shadowResult ? shadowResult.ok === true : false,
+      dispatch_count: Array.isArray(payload.dispatch) ? payload.dispatch.length : 0,
+      ops_count: Array.isArray(payload.ops) ? payload.ops.length : 0,
+      terminal_reason: payload.loop && typeof payload.loop === "object"
+        ? String(payload.loop.terminal_reason || "").trim()
+        : "",
+    });
     socket.write(
       `${JSON.stringify({
         type: IPC_RESPONSE_TYPES.RESPONSE,
@@ -226,6 +575,12 @@ async function handlePromptRequest(options = {}) {
     return true;
   } catch (err) {
     log(`error ${err.message || String(err)}`);
+    controllerObserver.emit("controller.prompt_completed", {
+      applied_from_msg_id: messageId,
+      ok: false,
+      error: err.message || String(err),
+      shadow_enabled: shadowEnabled,
+    });
     socket.write(
       `${JSON.stringify({
         type: IPC_RESPONSE_TYPES.ERROR,

package/src/daemon/status.js

@@ -3,6 +3,7 @@ const path = require("path");
 const { getUfooPaths } = require("../ufoo/paths");
 const { isMetaActive } = require("../bus/utils");
 const { readReportSummary, countControllerInboxEntries } = require("../report/store");
+const { readRecentLoopSummary } = require("../agent/loopObservability");
 
 function readBus(projectRoot) {
   const busPath = getUfooPaths(projectRoot).agentsFile;
@@ -180,6 +181,7 @@ function buildStatus(projectRoot, options = {}) {
     controller: {
       pending_total: controllerPendingTotal,
     },
+    loop: readRecentLoopSummary(projectRoot),
     cron: {
       count: cronTasks.length,
       tasks: cronTasks,

package/src/history/inputTimeline.js

@@ -5,6 +5,7 @@ const os = require("os");
 const path = require("path");
 const { getUfooPaths } = require("../ufoo/paths");
 const { loadAgentsData } = require("../ufoo/agentsStore");
+const { redactSecrets, redactString } = require("../providerapi/redactor");
 
 const HISTORY_DEBUG = process.env.UFOO_HISTORY_DEBUG === "1";
 const debugLog = (...args) => { if (HISTORY_DEBUG) console.error("[history]", ...args); };
@@ -446,10 +447,10 @@ function appendBusEntry(projectRoot, { seq, timestamp, publisher, target, messag
       fromId: publisher,
       to: nicknameMap.get(target) || target,
       toId: target,
-      message,
+      message: redactString(message),
     };
 
-    fs.appendFileSync(timelineFile, JSON.stringify(entry) + "\n", "utf8");
+    fs.appendFileSync(timelineFile, JSON.stringify(redactSecrets(entry)) + "\n", "utf8");
 
     if (seq) {
       const lock = acquireWatermarkLock(projectRoot);
@@ -504,10 +505,14 @@ function buildTimeline(projectRoot, { force = false } = {}) {
   const lock = acquireWatermarkLock(projectRoot);
   try {
     if (force) {
-      const content = newEntries.map((e) => JSON.stringify(e)).join("\n") + (newEntries.length > 0 ? "\n" : "");
+      const content = newEntries.map((e) => JSON.stringify(redactSecrets(e))).join("\n") + (newEntries.length > 0 ? "\n" : "");
       fs.writeFileSync(timelineFile, content, "utf8");
     } else {
-      fs.appendFileSync(timelineFile, newEntries.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf8");
+      fs.appendFileSync(
+        timelineFile,
+        newEntries.map((e) => JSON.stringify(redactSecrets(e))).join("\n") + "\n",
+        "utf8"
+      );
     }
 
     const prevCount = force ? 0 : (watermark.entryCount || 0);

package/src/memory/index.js

@@ -0,0 +1,24 @@
+const { ensureDir, appendJSONL, getTimestamp } = require("../bus/utils");
+const { canonicalProjectRoot } = require("../projects/projectId");
+const { getUfooPaths } = require("../ufoo/paths");
+
+class MemoryManager {
+  constructor(projectRoot) {
+    // Phase 0 scaffolding only: this seam must stay dormant until loop/runtime
+    // wiring passes an explicit projectRoot into the memory tool path.
+    this.projectRoot = canonicalProjectRoot(projectRoot);
+    const paths = getUfooPaths(this.projectRoot);
+    this.memoryDir = paths.memoryDir;
+    this.memoryFile = paths.memoryFile;
+    ensureDir(this.memoryDir);
+  }
+
+  addEntry(entry) {
+    appendJSONL(this.memoryFile, {
+      timestamp: getTimestamp(),
+      ...entry,
+    });
+  }
+}
+
+module.exports = MemoryManager;

package/src/providerapi/redactor.js

@@ -0,0 +1,87 @@
+"use strict";
+
+const REDACTED = "[REDACTED]";
+const SENSITIVE_KEY_PATTERN = /(^|_|-)(authorization|accesstoken|access_token|refreshtoken|refresh_token|apikey|api_key|tokenhash|token_hash)$/i;
+const BEARER_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]+\b/gi;
+
+function isPlainObject(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function isSensitiveKey(key = "") {
+  const normalized = String(key || "").replace(/[\s-]+/g, "_");
+  if (!normalized) return false;
+  if (SENSITIVE_KEY_PATTERN.test(normalized)) return true;
+  return /^token$/i.test(normalized);
+}
+
+function redactString(value) {
+  return String(value || "").replace(BEARER_PATTERN, "Bearer [REDACTED]");
+}
+
+function redactSecrets(value, options = {}) {
+  const seen = options._seen || new WeakMap();
+
+  if (typeof value === "string") {
+    return redactString(value);
+  }
+  if (!value || typeof value !== "object") {
+    return value;
+  }
+  if (seen.has(value)) {
+    return seen.get(value);
+  }
+  if (Array.isArray(value)) {
+    const out = [];
+    seen.set(value, out);
+    for (const item of value) {
+      out.push(redactSecrets(item, { ...options, _seen: seen }));
+    }
+    return out;
+  }
+  if (!isPlainObject(value)) {
+    return value;
+  }
+
+  const out = {};
+  seen.set(value, out);
+  for (const [key, entryValue] of Object.entries(value)) {
+    if (isSensitiveKey(key)) {
+      out[key] = REDACTED;
+      continue;
+    }
+    out[key] = redactSecrets(entryValue, { ...options, _seen: seen });
+  }
+  return out;
+}
+
+function redactJsonLine(value) {
+  return JSON.stringify(redactSecrets(value));
+}
+
+function redactUfooEvent(event) {
+  if (!event || typeof event !== "object") return event;
+  return redactSecrets(event);
+}
+
+function redactToolCallPayload(payload = {}) {
+  const input = payload && typeof payload === "object" ? payload : {};
+  return {
+    name: typeof input.name === "string" ? input.name : String(input.name || ""),
+    args: redactSecrets(input.args || input.arguments || {}),
+    tool_call_id: typeof input.tool_call_id === "string"
+      ? input.tool_call_id
+      : (typeof input.toolCallId === "string" ? input.toolCallId : ""),
+    caller_tier: typeof input.caller_tier === "string" ? input.caller_tier : "",
+  };
+}
+
+module.exports = {
+  REDACTED,
+  redactSecrets,
+  redactJsonLine,
+  redactString,
+  isSensitiveKey,
+  redactUfooEvent,
+  redactToolCallPayload,
+};