totemlayer 0.1.68 → 0.1.74

package/bin/observability.mjs

@@ -0,0 +1,336 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { randomUUID } from "node:crypto";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { readPrivateRuntimeJson } from "./mission-agent-session.mjs";
+
+const context = new AsyncLocalStorage();
+const capabilities = new WeakMap();
+const MAX_TEXT = 24000;
+const failureCode = (error, fallback) =>
+  typeof error?.code === "string" && /^[A-Z0-9_]{1,100}$/.test(error.code) ? error.code : fallback;
+
+/** Export data-plane text only. Never pass request headers or provider configuration here. */
+export function redactObservation(value, depth = 0) {
+  if (depth > 8) return "[depth limit]";
+  if (typeof value === "string")
+    return value
+      .replace(
+        /\b(?:tn_(?:live|test)_[A-Za-z0-9_-]{20,}|sk-(?:ant-)?[A-Za-z0-9_-]{16,}|AKIA[A-Z0-9]{16})\b/g,
+        "[credential redacted]",
+      )
+      .replace(/\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+\/-]+=*/gi, "[authorization redacted]")
+      .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[token redacted]")
+      .replace(
+        /-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?-----END [^-]*PRIVATE KEY-----/g,
+        "[private key redacted]",
+      )
+      .replace(
+        /((?:["']?)(?:[A-Z0-9_]*secret[A-Z0-9_]*|[A-Z0-9_]*password|api[_ -]?(?:key|token)|access[_ -]?token|AWS_SESSION_TOKEN|authorization)(?:["']?)\s*[=:]\s*)(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^\s,;"'}]+)/gi,
+        "$1[redacted]",
+      )
+      .slice(0, MAX_TEXT);
+  if (Array.isArray(value))
+    return value.slice(0, 80).map((item) => redactObservation(item, depth + 1));
+  if (value && typeof value === "object")
+    return Object.fromEntries(
+      Object.entries(value)
+        .slice(0, 80)
+        .map(([key, item]) => [
+          key,
+          /(?:authorization|password|secret|api[_-]?(?:key|token)|access[_-]?token|AWS_SESSION_TOKEN|leaseToken)/i.test(
+            key,
+          )
+            ? "[redacted]"
+            : redactObservation(item, depth + 1),
+        ]),
+    );
+  return value;
+}
+
+export function providerUsage(result) {
+  const u = result?.usage ?? result ?? {};
+  const count = (value) => (Number.isSafeInteger(value) && value >= 0 ? value : undefined);
+  const input = count(u.inputTokens ?? u.input_tokens ?? u.prompt_tokens ?? u.prompt_eval_count);
+  const output = count(u.outputTokens ?? u.output_tokens ?? u.completion_tokens ?? u.eval_count);
+  const cached = count(
+    u.cached_input_tokens ??
+      u.cacheReadInputTokens ??
+      u.cache_read_input_tokens ??
+      u.input_tokens_details?.cached_tokens ??
+      u.prompt_tokens_details?.cached_tokens,
+  );
+  const inclusiveCache =
+    u.cached_input_tokens ??
+    u.input_tokens_details?.cached_tokens ??
+    u.prompt_tokens_details?.cached_tokens;
+  const write = count(u.cacheWriteInputTokens ?? u.cache_creation_input_tokens);
+  return Object.fromEntries(
+    Object.entries({
+      input,
+      output,
+      ...(cached === undefined
+        ? {}
+        : {
+            input: input === undefined ? undefined : Math.max(0, input - (inclusiveCache ?? 0)),
+            cache_read_input_tokens: cached,
+          }),
+      cache_creation_input_tokens: write,
+    }).filter(([, v]) => v !== undefined),
+  );
+}
+
+export function providerCost(result) {
+  const value = result?.total_cost_usd ?? result?.cost_usd ?? result?.usage?.cost;
+  return typeof value === "number" && Number.isFinite(value) && value >= 0
+    ? { total: value }
+    : undefined;
+}
+
+async function telemetryDeadline(operation) {
+  let timer;
+  try {
+    return await Promise.race([
+      operation(),
+      new Promise((_, reject) => {
+        timer = setTimeout(() => reject(Error("TELEMETRY_TIMEOUT")), 1500);
+      }),
+    ]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function enabled(agentId, client) {
+  if (process.env.TOTEM_OBSERVABILITY_ENABLED !== undefined)
+    return process.env.TOTEM_OBSERVABILITY_ENABLED === "true";
+  try {
+    const settings = await readPrivateRuntimeJson(
+      join(process.env.TOTEM_CONFIG_DIR ?? join(homedir(), ".totem"), "observability.json"),
+    );
+    return settings.enabled === true && settings.agentIds?.includes(agentId);
+  } catch (error) {
+    if (error.code !== "ENOENT") {
+      console.error("Totem observability configuration unavailable");
+      return false;
+    }
+    if (!client?.observability?.status) return false;
+    const cached = capabilities.get(client);
+    if (cached?.expires > Date.now()) return cached.enabled;
+    let active = false;
+    try {
+      active = (await telemetryDeadline(() => client.observability.status())).enabled === true;
+    } catch {
+      /* Older collectors and opted-out owners send no content. */
+    }
+    capabilities.set(client, { enabled: active, expires: Date.now() + 60000 });
+    return active;
+  }
+}
+
+export async function submitObservation(client, observation) {
+  try {
+    if (!client?.observability?.record) throw Error("REPORTER_UNAVAILABLE");
+    await telemetryDeadline(() =>
+      client.observability.record({
+        observations: observation.map((record) => {
+          const safe = redactObservation(record);
+          for (const key of ["input", "output"]) {
+            const serialized = JSON.stringify(safe[key]);
+            if (serialized?.length > 20000) {
+              safe[key] = { truncated: true, preview: serialized.slice(0, 20000) };
+              safe.metadata = { ...safe.metadata, contentTruncated: true };
+            }
+          }
+          return safe;
+        }),
+      }),
+    );
+  } catch {
+    // Observability never changes a tool authorization or turns a completed model call into a retry.
+    console.error("Totem observability delivery unavailable");
+  }
+}
+
+export function observationOutcome({ output, error }) {
+  const root = context.getStore();
+  if (!root) return;
+  if (output !== undefined) root.output = output;
+  if (error) root.error = failureCode(error, "AGENT_FAILED");
+}
+
+export async function withAgentObservation(
+  {
+    client,
+    agentId,
+    sessionId,
+    traceId = randomUUID(),
+    input,
+    name = "agent.conversation",
+    metadata = {},
+    output: outputValue,
+  },
+  operation,
+) {
+  const parent = context.getStore();
+  if (parent?.agentId === agentId) {
+    const value = await operation();
+    if (typeof value === "string") parent.output = value;
+    return value;
+  }
+  if (!(await enabled(agentId, client))) return operation();
+  const root = {
+    client,
+    traceId,
+    sessionId: sessionId ?? traceId,
+    id: traceId + ":" + name,
+    agentId,
+    records: [],
+  };
+  const startedAt = new Date().toISOString();
+  return context.run(root, async () => {
+    let output, error;
+    try {
+      output = await operation();
+      return output;
+    } catch (failure) {
+      error = failureCode(failure, "AGENT_FAILED");
+      throw failure;
+    } finally {
+      await submitObservation(client, [
+        {
+          traceId,
+          id: root.id,
+          sessionId: root.sessionId,
+          name,
+          type: "agent",
+          startedAt,
+          endedAt: new Date().toISOString(),
+          input,
+          output: outputValue === undefined ? (root.output ?? output) : outputValue,
+          ...((error ?? root.error) ? { error: error ?? root.error } : {}),
+          metadata,
+        },
+        ...root.records.map((record, index) =>
+          index < 4
+            ? record
+            : {
+                ...record,
+                input: undefined,
+                output: record.type === "guardrail" ? record.output : undefined,
+                metadata: { ...record.metadata, contentTruncated: true },
+              },
+        ),
+      ]);
+    }
+  });
+}
+
+export async function observeGeneration(
+  { name = "model.generate", model, provider, input, metrics },
+  operation,
+) {
+  const root = context.getStore();
+  if (!root) return operation();
+  const startedAt = new Date().toISOString();
+  let value, error;
+  try {
+    value = await operation();
+    return value;
+  } catch (failure) {
+    error = failureCode(failure, "MODEL_FAILED");
+    throw failure;
+  } finally {
+    const measured = value ?? metrics?.();
+    if (root.records.length < 40)
+      root.records.push({
+        traceId: root.traceId,
+        id: randomUUID(),
+        parentId: root.id,
+        sessionId: root.sessionId,
+        type: "generation",
+        name,
+        model,
+        startedAt,
+        endedAt: new Date().toISOString(),
+        input,
+        output: value?.output ?? value?.choices?.[0]?.message ?? value?.message ?? value,
+        usage: providerUsage(measured),
+        cost: providerCost(measured),
+        ...(error ? { error } : {}),
+        metadata: {
+          provider,
+          usageSource: Object.keys(providerUsage(measured)).length ? "provider" : "unavailable",
+          costSource: providerCost(measured)
+            ? (measured?.cost_source ?? "provider")
+            : model && model !== "default"
+              ? "langfuse-model-definition"
+              : "unavailable",
+        },
+      });
+  }
+}
+
+const observedInspectors = new WeakSet();
+export function observeInspector(inspector) {
+  if (observedInspectors.has(inspector)) return inspector;
+  async function inspect(kind, value, operation) {
+    const root = context.getStore();
+    if (!root) return operation(() => {});
+    const startedAt = new Date().toISOString();
+    let recorded = false;
+    function complete(error) {
+      if (recorded) return;
+      recorded = true;
+      if (root.records.length < 40)
+        root.records.push({
+          traceId: root.traceId,
+          id: randomUUID(),
+          parentId: root.id,
+          sessionId: root.sessionId,
+          name: `inspection.${kind}`,
+          type: "guardrail",
+          startedAt,
+          endedAt: new Date().toISOString(),
+          input: value,
+          output: {
+            decision: error === "INSPECTION_BLOCKED" ? "block" : error ? "unavailable" : "allow",
+            code: error ?? "ALLOWED",
+            detector: inspector.mode ?? "rules",
+          },
+          ...(error ? { error } : {}),
+          metadata: { detector: inspector.mode ?? "rules" },
+        });
+    }
+    try {
+      const result = await operation(() => complete());
+      complete();
+      return result;
+    } catch (failure) {
+      complete(failureCode(failure, "INSPECTION_UNAVAILABLE"));
+      throw failure;
+    }
+  }
+  const observed = Object.freeze({
+    ...inspector,
+    assertAllowed: (kind, value, options) =>
+      inspect(kind, value, () => inspector.assertAllowed(kind, value, options)),
+    runAction: (value, operation, options) =>
+      inspect("action", value, (allowed) =>
+        inspector.runAction(
+          value,
+          (...args) => {
+            allowed();
+            return operation(...args);
+          },
+          options,
+        ),
+      ),
+    inspectFile: (name, mediaType, bytes, options) =>
+      inspect("file", { name, mediaType, bytes: bytes.byteLength }, () =>
+        inspector.inspectFile(name, mediaType, bytes, options),
+      ),
+  });
+  observedInspectors.add(observed);
+  return observed;
+}

package/bin/platform-client.mjs

@@ -85,11 +85,14 @@ export function installedClientInvocation(command, args, environment = process.e
   if (platform !== "win32") {
     const isExecutable = options.isExecutable ?? executableFile;
     if (
-      platform === "darwin" && command === "codex" &&
-      !(environment.PATH ?? "/usr/bin:/bin").split(delimiter)
+      platform === "darwin" &&
+      command === "codex" &&
+      !(environment.PATH ?? "/usr/bin:/bin")
+        .split(delimiter)
         .some((directory) => isExecutable(join(directory || ".", command)))
     ) {
-      const bundled = "/Applications/ChatGPT.app/Contents/Resources/codex-cli/CodexCLI.app/Contents/MacOS/codex";
+      const bundled =
+        "/Applications/ChatGPT.app/Contents/Resources/codex-cli/CodexCLI.app/Contents/MacOS/codex";
       if (isExecutable(bundled)) return { command: bundled, args };
     }
     return { command, args };

package/bin/plugin-connect.mjs

@@ -39,6 +39,8 @@ export function pluginConnectionOptions(args = [], environment = process.env) {
       "agent-id": { type: "string" },
       environment: { type: "string" },
       "base-url": { type: "string" },
+      "device-login": { type: "boolean" },
+      "no-browser": { type: "boolean" },
       help: { type: "boolean", short: "h" },
     },
   }).values;
@@ -64,6 +66,8 @@ export function pluginConnectionOptions(args = [], environment = process.env) {
     agentId,
     environment: flags.environment,
     baseUrl,
+    ...(flags["device-login"] ? { deviceLogin: true } : {}),
+    ...(flags["no-browser"] ? { noBrowser: true } : {}),
     configDirectory: join(runtimeRoot(environment), "plugins", flags.provider, selection),
   };
 }
@@ -74,6 +78,7 @@ export function createPluginConnector(selection, options = {}) {
   const env = {
     ...inherited,
     TOTEM_CONFIG_DIR: selection.configDirectory,
+    TOTEM_HOST_LOCK_ROOT: runtimeRoot(inherited),
     TOTEM_BASE_URL: selection.baseUrl,
     TOTEM_PROVIDER: selection.provider,
   };
@@ -355,6 +360,8 @@ export function createPluginConnector(selection, options = {}) {
             "--base-url",
             selection.baseUrl,
             "--json",
+            ...(selection.deviceLogin ? ["--device-login"] : []),
+            ...(selection.noBrowser ? ["--no-browser"] : []),
             ...(selection.agentId ? ["--agent-id", selection.agentId] : []),
           ],
           {
@@ -396,6 +403,9 @@ export function createPluginConnector(selection, options = {}) {
                           ready: false,
                           persistent: false,
                           signInUrl: url.toString(),
+                          ...(selection.deviceLogin && /^[A-Z2-9]{12}$/.test(report.code ?? "")
+                            ? { approvalCode: report.code }
+                            : {}),
                         };
                     }
                   } catch {

package/bin/provider-diagnostics.mjs

@@ -4,14 +4,24 @@ import { runtimeErrorCode } from "./runtime-diagnostics.mjs";
 
 export function providerDiagnostic(error, { environment = process.env, provider, model } = {}) {
   // A stream truncated at its output bound could contain only part of a secret.
-  const captured = error.code === "OUTPUT_LIMIT_EXCEEDED" ? {} : error.privateProviderDiagnostic ?? {};
+  const captured =
+    error.code === "OUTPUT_LIMIT_EXCEEDED" ? {} : (error.privateProviderDiagnostic ?? {});
   const secrets = Object.entries(environment)
-    .filter(([name, value]) => /secret|token|password|api.?key|db.?pass/i.test(name) && typeof value === "string" && value.length >= 4)
+    .filter(
+      ([name, value]) =>
+        /secret|token|password|api.?key|db.?pass/i.test(name) &&
+        typeof value === "string" &&
+        value.length >= 4,
+    )
     .map(([, value]) => value);
-  const redact = value => {
+  const redact = (value) => {
     let text = String(value ?? "");
     for (const secret of secrets) text = text.split(secret).join("[REDACTED]");
-    return text.replace(/\b(?:tn_(?:live|test)_[A-Za-z0-9_-]+|sk-[A-Za-z0-9_-]+|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)\b/g, "[REDACTED]")
+    return text
+      .replace(
+        /\b(?:tn_(?:live|test)_[A-Za-z0-9_-]+|sk-[A-Za-z0-9_-]+|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)\b/g,
+        "[REDACTED]",
+      )
       .replace(/(Bearer\s+)[^\s"']+/gi, "$1[REDACTED]")
       .replace(/(https?:\/\/)[^\s\/@]+@/gi, "$1[REDACTED]@")
       .slice(-6000);
@@ -22,25 +32,44 @@ export function providerDiagnostic(error, { environment = process.env, provider,
       const event = JSON.parse(line);
       if (["error", "turn.failed"].includes(event.type))
         errors.push(redact(event.error?.message ?? event.message ?? event.error));
-    } catch { /* Successful model output and ordinary stdout never become diagnostics. */ }
+    } catch {
+      /* Successful model output and ordinary stdout never become diagnostics. */
+    }
   }
-  return { version: 1, checkedAt: new Date().toISOString(), provider, model,
-    code: runtimeErrorCode(error), command: redact(captured.command),
+  return {
+    version: 1,
+    checkedAt: new Date().toISOString(),
+    provider,
+    model,
+    code: runtimeErrorCode(error),
+    command: redact(captured.command),
     ...(Number.isInteger(captured.exitCode) ? { exitCode: captured.exitCode } : {}),
-    stderr: redact(captured.stderr), errors: errors.slice(-8) };
+    stderr: redact(captured.stderr),
+    errors: errors.slice(-8),
+  };
 }
 
 export async function saveProviderDiagnostic(directory, error, context) {
   if (typeof directory !== "string" || !isAbsolute(directory)) return;
-  await privateWrite(join(directory, "last-provider-failure.json"), providerDiagnostic(error, context));
+  await privateWrite(
+    join(directory, "last-provider-failure.json"),
+    providerDiagnostic(error, context),
+  );
 }
 
 export async function readProviderDiagnostic(profile) {
   try {
     const value = await privateRead(join(profile.stateDirectory, "last-provider-failure.json"));
     const model = profile.client && profile.provider === "qwen-local" ? "default" : profile.model;
-    if (value.version !== 1 || value.provider !== (profile.client ?? profile.provider) || value.model !== model || !Number.isFinite(Date.parse(value.checkedAt)))
+    if (
+      value.version !== 1 ||
+      value.provider !== (profile.client ?? profile.provider) ||
+      value.model !== model ||
+      !Number.isFinite(Date.parse(value.checkedAt))
+    )
       return undefined;
     return value;
-  } catch (error) { if (error.code !== "ENOENT") throw error; }
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error;
+  }
 }

package/bin/runtime-installation.mjs

@@ -128,6 +128,7 @@ async function installRuntimeUnlocked(env) {
     for (const name of ["bin", "package.json", "LICENSE.md", "README.md"])
       await cp(join(packageRoot, name), join(stage, name), { recursive: true, dereference: false });
     const dependencies = [
+      { name: "@modelcontextprotocol/client", from: packageRoot },
       { name: "zod", from: packageRoot },
       { name: "undici", from: packageRoot },
       { name: "@modelcontextprotocol/server", from: packageRoot },
@@ -375,7 +376,8 @@ export async function installRuntimeService(runtime, env = process.env, options
         status.pid > 1 &&
         owner?.pid === status.pid &&
         owner.identity &&
-        (await (options.processIdentity ?? processIdentity)(status.pid)) === owner.identity
+        (await (options.processIdentity ?? processIdentity)(status.pid, owner.identity)) ===
+          owner.identity
       ) {
         const invocation =
           platform === "win32"
@@ -433,6 +435,7 @@ export async function installRuntimeService(runtime, env = process.env, options
     "TOTEM_INSPECTOR_MODEL",
     "TOTEM_INSPECTOR_TIMEOUT_MS",
     "TOTEM_PROVIDER",
+    "TOTEM_HOST_LOCK_ROOT",
     "TOTEM_RUNTIME_HOST_KIND",
     "TOTEM_MODEL",
     "TOTEM_BASE_URL",

package/bin/runtime-lock.mjs

@@ -1,13 +1,13 @@
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
-import { lstat, rename, readdir, unlink, rmdir, rm } from "node:fs/promises";
+import { lstat, rename, readdir, unlink, rmdir, rm, readFile } from "node:fs/promises";
 import { join } from "node:path";
-import { randomUUID } from "node:crypto";
+import { randomUUID, createHash } from "node:crypto";
 import { powershellInvocation } from "./platform-client.mjs";
 import { privateDirectory, privateRead, privateWrite } from "./runtime-installation.mjs";
 
 const execute = promisify(execFile);
-export async function processIdentity(pid) {
+export async function processIdentity(pid, previousIdentity, options = {}) {
   if (!Number.isSafeInteger(pid) || pid <= 1) return null;
   try {
     process.kill(pid, 0);
@@ -17,14 +17,51 @@ export async function processIdentity(pid) {
   }
   // PID reuse across reboot must not make an abandoned lock permanent. A failed
   // process inspection is uncertainty, never evidence that another owner died.
+  const platform = options.platform ?? process.platform;
+  if (
+    platform === "linux" &&
+    (!previousIdentity || previousIdentity.startsWith("linux-proc-v1:"))
+  ) {
+    const read = options.readFile ?? readFile;
+    const statPath = `/proc/${pid}/stat`;
+    let stat;
+    try {
+      stat = await read(statPath, "utf8");
+    } catch (error) {
+      if (error.code === "ENOENT") return null;
+      throw error;
+    }
+    // comm is parenthesized and may contain spaces or closing parentheses.
+    const startTime = (value) => {
+      if (!value.startsWith(`${pid} (`)) throw new Error("Invalid Linux process identity.");
+      const fields = value
+        .slice(value.lastIndexOf(")") + 2)
+        .trim()
+        .split(/\s+/);
+      if (!/^[A-Za-z]$/.test(fields[0] ?? "") || !/^\d{1,20}$/.test(fields[19] ?? ""))
+        throw new Error("Invalid Linux process identity.");
+      return fields[19];
+    };
+    const started = startTime(stat);
+    const boot = (await read("/proc/sys/kernel/random/boot_id", "utf8")).trim();
+    if (!/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(boot))
+      throw new Error("Cannot verify the Linux boot identity.");
+    const command = await read(`/proc/${pid}/cmdline`);
+    if (!command.length || command.length > 2 * 1024 * 1024)
+      throw new Error("Cannot verify the Linux process command.");
+    // Re-read after inspecting the command to reject a PID reused during the read.
+    if (startTime(await read(statPath, "utf8")) !== started)
+      throw new Error("Linux process identity changed during inspection.");
+    return `linux-proc-v1:${boot}:${pid}:${started}:${createHash("sha256").update(command).digest("hex")}`;
+  }
   const invocation =
-    process.platform === "win32"
+    platform === "win32"
       ? powershellInvocation(
           `$p=Get-CimInstance Win32_Process -Filter 'ProcessId=${pid}';if(!$p){exit 1};$p|Select-Object ProcessId,CreationDate,CommandLine|ConvertTo-Json -Compress`,
           process.env,
         )
       : { command: "ps", args: ["-p", String(pid), "-o", "lstart=", "-o", "args="] };
-  const result = await execute(invocation.command, invocation.args, {
+  const result = await (options.execute ?? execute)(invocation.command, invocation.args, {
     encoding: "utf8",
     timeout: 3000,
     maxBuffer: 8192,
@@ -95,7 +132,7 @@ export async function acquireSupervisorLock(directory, options = {}) {
           if (Date.now() - snapshot.mtimeMs < (options.emptyLockGraceMs ?? 30000))
             throw new Error("Another supervisor is starting. Wait before retrying.");
         } else {
-          const actual = await identity(current.owner.pid);
+          const actual = await identity(current.owner.pid, current.owner.identity);
           if (actual && (!current.owner.identity || current.owner.identity === actual))
             throw Object.assign(new Error("The Totem supervisor is already running."), {
               code: "SUPERVISOR_ALREADY_RUNNING",

package/bin/sdk/index.d.ts

@@ -16,6 +16,7 @@ export declare class Totem {
             identityAgentId: string;
         }>;
     };
+    observability: { status: () => Promise<{enabled: boolean}>; record: (observation: Record<string, unknown>) => Promise<{accepted: boolean}> };
     memory: {
         search: (query?: AgentMemorySearch, options?: RequestOptions) => Promise<AgentMemoryHistory>;
         get: (options?: RequestOptions) => Promise<AgentMemory>;