totemlayer 0.1.4 → 0.1.6

package/bin/agent-runner.mjs

@@ -0,0 +1,609 @@
+import { spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { constants } from "node:fs";
+import { lstat, mkdir, open, opendir, rename, unlink } from "node:fs/promises";
+import { dirname, isAbsolute, join, parse, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+
+import { readAgentCredential, readAgentProfile, withAgentProfileLock } from "./agent-profile.mjs";
+import { localClientEnvironment } from "./platform-client.mjs";
+import { Totem } from "./sdk/index.js";
+
+const CLI_PATH = fileURLToPath(new URL("./totem.mjs", import.meta.url));
+const UUID = /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i;
+const PHASES = new Set([
+  "starting",
+  "connected",
+  "reconnecting",
+  "working",
+  "waiting",
+  "blocked",
+  "stopping",
+  "stopped",
+  "failed",
+]);
+const LABELS = {
+  starting: "Connecting your agent…",
+  connected: "Agent connected. Looking for approved Mission work.",
+  reconnecting: "Connection interrupted. Waiting to reconnect from saved progress.",
+  working: "Working on an assigned Mission task.",
+  waiting: "Connected and waiting for approved work.",
+  blocked: "A Mission needs attention. Open Missions to review the blocker.",
+  stopping: "Stopping the agent and saving its progress…",
+  stopped: "Agent stopped. Saved progress will be checked when it starts again.",
+  failed:
+    "The agent stopped with an error. Check its connection and Mission status before restarting.",
+};
+
+export const AGENT_START_HELP = `Run your connected Totem agent
+
+  totem start [--profile default] [--background] [--json]
+  totem stop [--profile default] [--json]
+  totem status [--profile default] [--json]
+
+Run setup once first. Start receives approved Mission assignments automatically
+and keeps checking for new work. Stop preserves the worker's recovery journal.
+Use --background to keep this process running after closing the terminal.
+Your computer and local model must remain running. This does not install a
+login service, start a model, download weights, or grant new permissions.
+`;
+
+function alive(pid) {
+  if (!Number.isSafeInteger(pid) || pid < 1) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code === "EPERM";
+  }
+}
+
+function privateEntry(info, directory = false, opened = false) {
+  if (
+    !(directory ? info.isDirectory() : info.isFile()) ||
+    info.isSymbolicLink() ||
+    (!directory && ((info.nlink !== 1 && !(opened && info.nlink === 0)) || info.size > 4096)) ||
+    (typeof process.getuid === "function" && info.uid !== process.getuid()) ||
+    (process.platform !== "win32" && (info.mode & 0o777) !== (directory ? 0o700 : 0o600))
+  )
+    throw new Error("The agent status storage is not a bounded private file or directory.");
+}
+
+async function directoryTree(path, privateRoot) {
+  if (typeof path !== "string" || !isAbsolute(path) || resolve(path) !== path)
+    throw new Error("The agent status directory is invalid.");
+  let current = parse(path).root;
+  for (const part of path.slice(current.length).split(sep).filter(Boolean)) {
+    current = join(current, part);
+    const info = await lstat(current);
+    if (!info.isDirectory() || info.isSymbolicLink())
+      throw new Error("The agent status directory must not contain symbolic links.");
+    if (current === privateRoot || current.startsWith(`${privateRoot}${sep}`))
+      privateEntry(info, true);
+  }
+}
+
+async function verifyRuntimeDirectory(directory) {
+  await directoryTree(directory, dirname(dirname(dirname(directory))));
+}
+
+async function runtimeDirectory(profile, create = false) {
+  await directoryTree(profile.stateDirectory, dirname(dirname(profile.stateDirectory)));
+  const path = join(profile.stateDirectory, "runner");
+  if (create) {
+    try {
+      await mkdir(path, { mode: 0o700 });
+    } catch (error) {
+      if (error.code !== "EEXIST") throw error;
+    }
+  }
+  await verifyRuntimeDirectory(path);
+  return path;
+}
+
+async function readPrivateJson(path) {
+  await verifyRuntimeDirectory(dirname(path));
+  for (let attempt = 0; ; attempt += 1) {
+    const info = await lstat(path);
+    // macOS may finish a path lookup against the inode an atomic rename just
+    // unlinked. Retry that snapshot only, after all other safety checks pass.
+    privateEntry(info, false, true);
+    if (info.nlink === 1) break;
+    if (attempt === 2)
+      throw new Error("The agent status record kept changing during inspection. Retry shortly.");
+  }
+  const file = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+  try {
+    // Replacement may unlink this already-open snapshot. It is still bounded and
+    // private; multiple hardlinks remain forbidden on both the path and descriptor.
+    privateEntry(await file.stat(), false, true);
+    const buffer = Buffer.alloc(4097);
+    let length = 0;
+    while (length < buffer.length) {
+      const { bytesRead } = await file.read(buffer, length, buffer.length - length, length);
+      if (!bytesRead) break;
+      length += bytesRead;
+    }
+    if (length > 4096) throw new Error("The agent status record exceeds its size limit.");
+    try {
+      return JSON.parse(buffer.subarray(0, length).toString("utf8"));
+    } catch {
+      throw new Error("The saved agent status is invalid.");
+    }
+  } finally {
+    await file.close();
+  }
+}
+
+async function writePrivateJson(path, record) {
+  await verifyRuntimeDirectory(dirname(path));
+  const body = `${JSON.stringify(record)}\n`;
+  if (Buffer.byteLength(body) > 4096) throw new Error("The agent status record is too large.");
+  const file = await open(
+    path,
+    constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | (constants.O_NOFOLLOW ?? 0),
+    0o600,
+  );
+  try {
+    await file.writeFile(body);
+  } finally {
+    await file.close();
+  }
+}
+
+async function saveStatus(directory, status) {
+  const temporary = join(directory, `.${status.runId}.${randomUUID()}.tmp`);
+  try {
+    await writePrivateJson(temporary, status);
+    const destination = join(directory, `${status.runId}.json`);
+    try {
+      privateEntry(await lstat(destination));
+    } catch (error) {
+      if (error.code !== "ENOENT") throw error;
+    }
+    await verifyRuntimeDirectory(directory);
+    await rename(temporary, destination);
+  } finally {
+    await unlink(temporary).catch(() => {});
+  }
+}
+
+export async function readAgentRuntimeStatus(profile, { isAlive = alive, now = Date.now } = {}) {
+  let directory;
+  try {
+    directory = await runtimeDirectory(profile);
+  } catch (error) {
+    if (error?.code === "ENOENT") return { running: false, phase: "not_started" };
+    throw error;
+  }
+  const names = [];
+  let entryCount = 0;
+  for await (const entry of await opendir(directory)) {
+    entryCount += 1;
+    if (UUID.test(entry.name.replace(/\.json$/, "")) && entry.name.endsWith(".json"))
+      names.push(entry.name);
+    if (names.length > 1000 || entryCount > 2000)
+      throw new Error(
+        "Too many saved agent sessions. Inspect the private status directory before restarting.",
+      );
+  }
+  const records = [];
+  for (const name of names) {
+    const record = await readPrivateJson(join(directory, name));
+    if (
+      !record ||
+      typeof record !== "object" ||
+      Array.isArray(record) ||
+      record.version !== 1 ||
+      record.runId !== name.slice(0, -5) ||
+      !PHASES.has(record.phase) ||
+      record.agentId !== profile.agentId ||
+      record.baseUrl !== profile.baseUrl ||
+      !Number.isSafeInteger(record.pid) ||
+      record.pid < 1 ||
+      typeof record.updatedAt !== "string" ||
+      typeof record.startedAt !== "string" ||
+      !Number.isFinite(Date.parse(record.updatedAt)) ||
+      !Number.isFinite(Date.parse(record.startedAt)) ||
+      Date.parse(record.startedAt) > Date.parse(record.updatedAt) ||
+      Date.parse(record.updatedAt) > now() + 5000
+    )
+      throw new Error("The saved agent status is invalid. Inspect it before restarting.");
+    records.push(record);
+  }
+  records.sort((left, right) => Date.parse(right.startedAt) - Date.parse(left.startedAt));
+  const running = records.find(
+    (record) => !["stopped", "failed"].includes(record.phase) && isAlive(record.pid),
+  );
+  const record = running ?? records[0];
+  if (!record) return { running: false, phase: "not_started" };
+  const fresh = now() - Date.parse(record.updatedAt) <= 30_000;
+  return {
+    running: Boolean(running),
+    phase: running
+      ? fresh
+        ? record.phase
+        : "unresponsive"
+      : ["stopped", "failed"].includes(record.phase)
+        ? record.phase
+        : "interrupted",
+    runId: record.runId,
+    updatedAt: record.updatedAt,
+    ...(UUID.test(record.missionId ?? "") ? { missionId: record.missionId } : {}),
+    ...(UUID.test(record.workItemId ?? "") ? { workItemId: record.workItemId } : {}),
+  };
+}
+
+function eventState(event) {
+  const candidate = event?.state ?? event?.status ?? event?.type;
+  const phase =
+    typeof candidate === "string" ? candidate.replace(/^mission_agent_|^agent_/, "") : "";
+  if (
+    ["reconnecting", "runtime_reconnecting"].includes(phase) ||
+    [event?.code, event?.reason].some((code) =>
+      ["service_unavailable", "runtime_reconnecting"].includes(code),
+    )
+  )
+    return { phase: "reconnecting" };
+  if (phase === "completed")
+    return {
+      phase: "waiting",
+      ...(UUID.test(event.missionId ?? "") ? { missionId: event.missionId } : {}),
+    };
+  if (!["connected", "working", "waiting", "blocked"].includes(phase)) return null;
+  return {
+    phase,
+    ...(UUID.test(event.missionId ?? "") ? { missionId: event.missionId } : {}),
+    ...(UUID.test(event.workItemId ?? "") ? { workItemId: event.workItemId } : {}),
+  };
+}
+
+/** Only fixed labels and validated IDs leave the local worker; model output never becomes a log. */
+export async function runAgentSession(profile, options = {}) {
+  if (profile.client)
+    throw new Error(
+      "This profile connects an existing AI app. Automatic Missions currently require a separate local-agent profile.",
+    );
+  const intervalMs = options.statusIntervalMs ?? 1000;
+  if (!Number.isInteger(intervalMs) || intervalMs < 10 || intervalMs > 60_000)
+    throw new Error("Invalid agent status interval.");
+  const apiKey = await readAgentCredential(profile);
+  const client = (
+    options.createClient ?? ((input) => new Totem({ ...input, timeoutMs: 10_000, maxRetries: 0 }))
+  )({ apiKey, baseUrl: profile.baseUrl });
+  const identity = await client.agent({ signal: options.signal });
+  if (
+    identity.id !== profile.agentId ||
+    identity.environment !== profile.environment ||
+    identity.address !== profile.address
+  )
+    throw new Error("This credential no longer matches the saved agent. Run setup again.");
+  const runId = options.runId ?? randomUUID();
+  if (!UUID.test(runId)) throw new Error("Invalid agent run identifier.");
+  const directory = join(profile.stateDirectory, "runner");
+  const controller = new AbortController();
+  const stop = () => controller.abort();
+  options.signal?.addEventListener("abort", stop, { once: true });
+  if (options.signal?.aborted) controller.abort();
+  let status = {
+    version: 1,
+    runId,
+    pid: process.pid,
+    agentId: profile.agentId,
+    baseUrl: profile.baseUrl,
+    startedAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    phase: "starting",
+  };
+  let writes = Promise.resolve();
+  let persistenceFailed = false;
+  const persist = () => {
+    const snapshot = { ...status, updatedAt: new Date().toISOString() };
+    writes = writes
+      .then(() => saveStatus(directory, snapshot))
+      .catch(() => {
+        persistenceFailed = true;
+        controller.abort();
+      });
+    return writes;
+  };
+  const emit = (state) => {
+    status = { ...status, missionId: undefined, workItemId: undefined, ...state };
+    try {
+      options.emit?.({
+        phase: status.phase,
+        message: LABELS[status.phase],
+        ...(status.missionId ? { missionId: status.missionId } : {}),
+      });
+    } catch {
+      persistenceFailed = true;
+      controller.abort();
+    }
+    void persist();
+  };
+  try {
+    await withAgentProfileLock(profile, async () => {
+      const before = await readAgentRuntimeStatus(profile);
+      if (before.running)
+        throw new Error(
+          "This agent is already running. Use totem status or totem stop for this profile.",
+        );
+      await runtimeDirectory(profile, true);
+      try {
+        await lstat(join(directory, `${runId}.json`));
+        throw new Error("An agent run identifier cannot be reused.");
+      } catch (error) {
+        if (error.code !== "ENOENT") throw error;
+      }
+      await persist();
+      if (persistenceFailed) throw new Error("Could not save agent status. No worker was started.");
+    });
+  } catch (error) {
+    options.signal?.removeEventListener("abort", stop);
+    await writes;
+    throw error;
+  }
+  let polling;
+  let stopReadFailed = false;
+  const poll = async () => {
+    try {
+      const request = await readPrivateJson(join(directory, `${runId}.stop`));
+      if (
+        !request ||
+        request.runId !== runId ||
+        Object.keys(request).some((key) => key !== "runId")
+      )
+        throw new Error("The existing stop request is invalid.");
+      controller.abort();
+    } catch (error) {
+      if (error?.code !== "ENOENT") {
+        stopReadFailed = true;
+        controller.abort();
+      }
+    }
+    await persist();
+  };
+  let timer;
+  let result = 0;
+  let acceptingEvents = true;
+  try {
+    await poll();
+    timer = setInterval(() => {
+      if (!polling)
+        polling = poll().finally(() => {
+          polling = undefined;
+        });
+    }, intervalMs);
+    if (!controller.signal.aborted) {
+      const runService =
+        options.runService ?? (await import("./mission-agent-service.mjs")).runMissionAgentService;
+      await runService({
+        client,
+        expectedAgentId: profile.agentId,
+        baseUrl: profile.baseUrl,
+        stateDirectory: profile.stateDirectory,
+        repositoryBindings: profile.repositoryRoots.map(({ repositoryId, root }) => ({
+          repositoryUrl: repositoryId,
+          root,
+        })),
+        provider: profile.provider,
+        qwenUrl: profile.qwenUrl,
+        model: profile.model,
+        signal: controller.signal,
+        emit: (event) => {
+          if (!acceptingEvents) return;
+          const state = eventState(event);
+          if (state) emit(state);
+        },
+      });
+    }
+    if (persistenceFailed || stopReadFailed)
+      throw new Error("Agent status could not be saved; the worker was stopped.");
+  } catch {
+    result = 1;
+    controller.abort();
+  } finally {
+    acceptingEvents = false;
+    clearInterval(timer);
+    await polling;
+    options.signal?.removeEventListener("abort", stop);
+    await writes;
+    if (persistenceFailed || stopReadFailed) result = 1;
+    emit({ phase: result ? "failed" : "stopped" });
+    await writes;
+    if (persistenceFailed) result = 1;
+    await unlink(join(directory, `${runId}.stop`)).catch(() => {});
+  }
+  return result;
+}
+
+function runnerOptions(args) {
+  const parsed = parseArgs({
+    args,
+    strict: true,
+    allowPositionals: false,
+    options: {
+      profile: { type: "string", default: "default" },
+      background: { type: "boolean" },
+      json: { type: "boolean" },
+      help: { type: "boolean", short: "h" },
+      "run-id": { type: "string" },
+    },
+  });
+  return parsed.values;
+}
+
+async function requestStop(profile, runId) {
+  if (!UUID.test(runId)) throw new Error("Invalid agent run identifier.");
+  const directory = await runtimeDirectory(profile, true);
+  const path = join(directory, `${runId}.stop`);
+  const temporary = join(directory, `.${runId}.${randomUUID()}.stop.tmp`);
+  try {
+    // Only publish a closed, complete record. Polling must never see the empty
+    // or partially written file while an ordinary stop request is being saved.
+    await writePrivateJson(temporary, { runId });
+    try {
+      const existing = await readPrivateJson(path);
+      if (
+        !existing ||
+        existing.runId !== runId ||
+        Object.keys(existing).some((key) => key !== "runId")
+      )
+        throw new Error("The existing stop request is invalid.");
+      return;
+    } catch (error) {
+      if (error?.code !== "ENOENT") throw error;
+    }
+    await verifyRuntimeDirectory(directory);
+    await rename(temporary, path);
+  } finally {
+    await unlink(temporary).catch(() => {});
+  }
+}
+
+export async function agentStartMain(args, options = {}) {
+  const environment = options.env ?? process.env;
+  const output = options.stdout ?? process.stdout;
+  const parsed = runnerOptions(args);
+  if (parsed.help) {
+    output.write(AGENT_START_HELP);
+    return 0;
+  }
+  const profile = await readAgentProfile(parsed.profile, { environment });
+  if (profile.client)
+    throw new Error("Existing-client MCP profiles cannot start automatic Missions.");
+  const report = (value) =>
+    output.write(`${parsed.json ? JSON.stringify(value) : value.message}\n`);
+  if (parsed.background) {
+    if (parsed["run-id"])
+      throw new Error("An internal run identifier cannot be combined with background mode.");
+    const existing = await readAgentRuntimeStatus(profile);
+    if (existing.running) {
+      report({
+        ...existing,
+        message:
+          existing.phase === "unresponsive"
+            ? "The existing agent session is unresponsive. Inspect its status before restarting."
+            : "This agent is already running.",
+      });
+      return 0;
+    }
+    const runId = randomUUID();
+    const env = localClientEnvironment(environment, {
+      ...(environment.TOTEM_CONFIG_DIR ? { TOTEM_CONFIG_DIR: environment.TOTEM_CONFIG_DIR } : {}),
+    });
+    delete env.TOTEM_NETWORK_API_KEY_FILE;
+    delete env.TOTEM_BASE_URL;
+    const child = (options.spawnProcess ?? spawn)(
+      process.execPath,
+      [CLI_PATH, "start", "--profile", parsed.profile, "--run-id", runId, "--json"],
+      { env, detached: true, stdio: "ignore", windowsHide: true },
+    );
+    let failed = false;
+    child.once("error", () => {
+      failed = true;
+    });
+    child.once("exit", () => {
+      failed = true;
+    });
+    child.unref();
+    try {
+      const startupTimeoutMs = options.startupTimeoutMs ?? 10_000;
+      if (
+        !Number.isInteger(startupTimeoutMs) ||
+        startupTimeoutMs < 10 ||
+        startupTimeoutMs > 120_000
+      )
+        throw new Error("Invalid background startup deadline.");
+      const deadline = Date.now() + startupTimeoutMs;
+      while (Date.now() < deadline) {
+        await new Promise((resolve) =>
+          setTimeout(resolve, Math.min(250, Math.max(1, deadline - Date.now()))),
+        );
+        if (failed) break;
+        const state = await readAgentRuntimeStatus(profile);
+        if (
+          state.runId === runId &&
+          state.running &&
+          ["connected", "working", "waiting", "blocked"].includes(state.phase)
+        ) {
+          report({
+            ...state,
+            message:
+              "Agent started in the background. Use totem status to check its connection and work.",
+          });
+          return 0;
+        }
+        if (state.runId === runId && ["failed", "stopped", "interrupted"].includes(state.phase))
+          break;
+      }
+    } catch {
+      // The run-bound stop is also requested when a status read fails closed.
+    }
+    await requestStop(profile, runId);
+    throw new Error(
+      "The background agent did not confirm startup. Run totem start without --background to check its setup.",
+    );
+  }
+  const controller = new AbortController();
+  const stop = () => controller.abort();
+  process.once("SIGINT", stop);
+  process.once("SIGTERM", stop);
+  try {
+    return await runAgentSession(profile, {
+      ...options,
+      runId: parsed["run-id"],
+      signal: options.signal
+        ? AbortSignal.any([controller.signal, options.signal])
+        : controller.signal,
+      emit: report,
+    });
+  } finally {
+    process.removeListener("SIGINT", stop);
+    process.removeListener("SIGTERM", stop);
+  }
+}
+
+export async function agentStopMain(args, options = {}) {
+  const parsed = runnerOptions(args);
+  const output = options.stdout ?? process.stdout;
+  if (parsed.help) {
+    output.write(AGENT_START_HELP);
+    return 0;
+  }
+  const profile = await readAgentProfile(parsed.profile, {
+    environment: options.env ?? process.env,
+  });
+  const state = await readAgentRuntimeStatus(profile);
+  if (state.running) await requestStop(profile, state.runId);
+  const result = {
+    requested: state.running,
+    message: state.running
+      ? "Stop requested. The worker will cancel pending work and preserve its recovery journal."
+      : "This agent is not running.",
+  };
+  output.write(`${parsed.json ? JSON.stringify(result) : result.message}\n`);
+  return 0;
+}
+
+export async function agentRunnerMain(command, args, options = {}) {
+  try {
+    return command === "stop"
+      ? await agentStopMain(args, options)
+      : await agentStartMain(args, options);
+  } catch {
+    const output = args.includes("--json")
+      ? (options.stdout ?? process.stdout)
+      : (options.stderr ?? process.stderr);
+    const result = {
+      error: {
+        code: "AGENT_START_FAILED",
+        message:
+          "Could not change the agent's running state. Run totem status for this profile, check its setup, then retry. Existing-client MCP profiles do not run automatic Missions.",
+      },
+    };
+    output.write(`${args.includes("--json") ? JSON.stringify(result) : result.error.message}\n`);
+    return 1;
+  }
+}