totemlayer 0.1.4 → 0.1.6

package/bin/mission-agent-service.mjs

@@ -0,0 +1,441 @@
+import { randomUUID } from "node:crypto";
+import { readdir, realpath, rename, rm, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { isAbsolute, join, resolve } from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
+
+import {
+  acquireMissionAgentSession,
+  ensurePrivateRuntimeDirectory,
+  missionAgentOrigin,
+  readPrivateRuntimeJson,
+} from "./mission-agent-session.mjs";
+import {
+  missionModelEndpoint,
+  preflightMissionRepository,
+  runMissionWorkerCycle,
+} from "./mission-worker.mjs";
+import { repositoryIdFromRemote } from "./repository.mjs";
+
+const UUID = /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i;
+const STAGES = new Set(["proposal", "engineering", "decision", "code", "review", "handoff"]);
+const REPOSITORY_STAGES = new Set(["engineering", "code", "review"]);
+const TERMINAL = new Set(["completed", "canceled", "failed"]);
+const RECOVERABLE = new Set([
+  "ready",
+  "result_settling",
+  "local_test_failure_settling",
+  "claiming",
+]);
+const CHECKPOINT_BYTES = 2 * 1024 * 1024;
+
+function transient(error) {
+  return (
+    ["NETWORK_ERROR", "TIMEOUT", "ECONNRESET", "ECONNREFUSED", "ETIMEDOUT"].includes(error?.code) ||
+    [408, 425, 429, 500, 502, 503, 504].includes(error?.status) ||
+    (error instanceof TypeError && /fetch|network/i.test(error.message))
+  );
+}
+function cursor(value) {
+  return (
+    value === null || (typeof value === "string" && value.length > 0 && value.length <= 16_384)
+  );
+}
+function assertPage(page) {
+  if (!page || !Array.isArray(page.data) || !cursor(page.nextCursor))
+    throw new Error("The Missions API returned an invalid discovery page.");
+  return page;
+}
+async function saveCheckpoint(path, state) {
+  const value = JSON.stringify(state);
+  if (Buffer.byteLength(value) > CHECKPOINT_BYTES)
+    throw new Error("The saved discovery checkpoint is full; inspect it before continuing.");
+  const temporary = `${path}.${randomUUID()}.tmp`;
+  try {
+    await writeFile(temporary, value, { mode: 0o600, flag: "wx" });
+    await rename(temporary, path);
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+async function loadCheckpoint(path, origin, agentId) {
+  const state = await readPrivateRuntimeJson(path, CHECKPOINT_BYTES).catch((error) => {
+    if (error.code === "ENOENT") return null;
+    throw error;
+  });
+  if (!state)
+    return {
+      version: 1,
+      origin,
+      agentId,
+      missionCursor: null,
+      pendingMissions: [],
+      workCursors: {},
+      recoveryAfter: null,
+      preferRecovery: true,
+    };
+  if (
+    state.version !== 1 ||
+    state.origin !== origin ||
+    state.agentId !== agentId ||
+    !cursor(state.missionCursor) ||
+    !Array.isArray(state.pendingMissions) ||
+    !state.pendingMissions.every((id) => UUID.test(id)) ||
+    !state.workCursors ||
+    typeof state.workCursors !== "object" ||
+    Array.isArray(state.workCursors) ||
+    !Object.entries(state.workCursors).every(([id, value]) => UUID.test(id) && cursor(value)) ||
+    !(state.recoveryAfter === null || UUID.test(state.recoveryAfter)) ||
+    typeof state.preferRecovery !== "boolean"
+  )
+    throw new Error("The saved Missions discovery checkpoint needs inspection.");
+  return state;
+}
+async function recoveryMissions(directory) {
+  const found = [];
+  for (const entry of await readdir(directory, { withFileTypes: true })) {
+    if (!entry.isDirectory() || !UUID.test(entry.name)) continue;
+    const missionDirectory = join(directory, entry.name);
+    await ensurePrivateRuntimeDirectory(missionDirectory);
+    for (const file of await readdir(missionDirectory, { withFileTypes: true })) {
+      if (!file.isFile() || !file.name.endsWith(".json") || !UUID.test(file.name.slice(0, -5)))
+        continue;
+      const record = await readPrivateRuntimeJson(
+        join(missionDirectory, file.name),
+        CHECKPOINT_BYTES,
+      );
+      if (
+        RECOVERABLE.has(record.state) ||
+        (record.state === "completion_blocked" && !record.blockerRecorded)
+      ) {
+        found.push(entry.name);
+        break;
+      }
+    }
+  }
+  return found.sort();
+}
+
+/** One authenticated local agent, serial work, durable discovery, explicit repositories. */
+export async function runMissionAgentService(
+  {
+    client,
+    expectedAgentId,
+    baseUrl,
+    repositoryBindings = [],
+    provider = "qwen-local",
+    model,
+    qwenUrl,
+    nimUrl,
+    ollamaUrl,
+    reasoning = false,
+    stateDirectory = join(homedir(), ".totem", "missions"),
+    pollMs = 5000,
+    signal,
+    emit = () => {},
+  },
+  dependencies = {},
+) {
+  const origin = missionAgentOrigin(baseUrl);
+  missionModelEndpoint(provider, { qwenUrl, ollamaUrl, nimUrl });
+  if (!Number.isInteger(pollMs) || pollMs < 10 || pollMs > 60_000)
+    throw new Error("Choose a Missions polling interval between 10 and 60000 milliseconds.");
+  if (!Array.isArray(repositoryBindings))
+    throw new Error("Provide explicit local repository bindings.");
+  const bindings = new Map();
+  for (const binding of repositoryBindings) {
+    if (
+      !binding ||
+      typeof binding.repositoryUrl !== "string" ||
+      repositoryIdFromRemote(`https://${binding.repositoryUrl}`) !== binding.repositoryUrl ||
+      typeof binding.root !== "string" ||
+      !isAbsolute(binding.root)
+    )
+      throw new Error(
+        "Repository bindings require a canonical repository ID and an absolute local root.",
+      );
+    if (bindings.has(binding.repositoryUrl))
+      throw new Error("Configure each repository binding only once.");
+    bindings.set(binding.repositoryUrl, resolve(binding.root));
+  }
+  const authenticate = async () => {
+    const agent = await client.agent({ signal });
+    if (
+      typeof agent?.id !== "string" ||
+      !/^[a-zA-Z0-9-]{1,200}$/.test(agent.id) ||
+      (expectedAgentId && agent.id !== expectedAgentId)
+    )
+      throw Object.assign(new Error("The credential no longer identifies the configured agent."), {
+        code: "AGENT_IDENTITY_MISMATCH",
+      });
+    return agent;
+  };
+  let agent;
+  let failures = 0;
+  let session;
+  try {
+    while (!agent && !signal?.aborted) {
+      try {
+        agent = await authenticate();
+      } catch (error) {
+        if (signal?.aborted || !transient(error)) throw error;
+        emit({ type: "waiting", code: "service_unavailable" });
+        await sleep(
+          Math.min(30_000, Math.max(1000, pollMs) * 2 ** Math.min(failures++, 5)),
+          undefined,
+          { signal },
+        );
+      }
+    }
+    if (!agent || signal?.aborted) {
+      emit({ type: "stopped" });
+      return;
+    }
+    session = await acquireMissionAgentSession(
+      { baseUrl: origin, agentId: agent.id, signal },
+      dependencies.session,
+    );
+  } catch (error) {
+    if (!signal?.aborted) throw error;
+    emit({ type: "stopped" });
+    return;
+  }
+  let lastEvent;
+  const report = (event) => {
+    const encoded = JSON.stringify(event);
+    if (encoded !== lastEvent) {
+      lastEvent = encoded;
+      emit(event);
+    }
+  };
+  const workerStateDirectory = join(resolve(stateDirectory), "mission-agents", session.key);
+  const directory = join(workerStateDirectory, agent.id);
+  try {
+    await ensurePrivateRuntimeDirectory(resolve(stateDirectory));
+    await ensurePrivateRuntimeDirectory(join(resolve(stateDirectory), "mission-agents"));
+    await ensurePrivateRuntimeDirectory(workerStateDirectory);
+    await ensurePrivateRuntimeDirectory(directory);
+    const checkpointPath = join(directory, `discovery-${session.key}.json`);
+    const state = await loadCheckpoint(checkpointPath, origin, agent.id);
+    const persist = () => saveCheckpoint(checkpointPath, state);
+    const revalidate = async (error, missionId) => {
+      if (signal?.aborted) throw error;
+      if (error.status === 401 || error.code === "AGENT_IDENTITY_MISMATCH") throw error;
+      if (error.status === 403) {
+        const current = await authenticate();
+        if (current.id !== agent.id)
+          throw Object.assign(new Error("Agent identity changed."), {
+            code: "AGENT_IDENTITY_MISMATCH",
+          });
+        report({
+          type: "blocked",
+          ...(missionId ? { missionId } : {}),
+          code: "permission_denied",
+        });
+        return true;
+      }
+      if ([404, 409].includes(error.status) && missionId) {
+        report({ type: "waiting", missionId, code: "assignment_unavailable" });
+        return true;
+      }
+      return false;
+    };
+    const bindingFor = async (mission, kind, recovery = false) => {
+      if (!mission.repository && !REPOSITORY_STAGES.has(kind)) return null;
+      const root = mission.repository && bindings.get(mission.repository.url);
+      if (!root) {
+        report({ type: "waiting", missionId: mission.id, code: "repository_unbound" });
+        return undefined;
+      }
+      try {
+        const canonicalRoot = await realpath(root);
+        await preflightMissionRepository(mission, canonicalRoot);
+        return canonicalRoot;
+      } catch {
+        report({
+          type: "blocked",
+          missionId: mission.id,
+          code: "repository_configuration_invalid",
+        });
+        // Saved completion settlement does not read a repository or repeat an effect.
+        return recovery ? null : undefined;
+      }
+    };
+    const runCycle = async (missionId, repositoryRoot, workItem, recoveryOnly = false) => {
+      await (dependencies.runCycle ?? runMissionWorkerCycle)({
+        client,
+        baseUrl: origin,
+        expectedAgentId: agent.id,
+        _session: session,
+        missionId,
+        repositoryRoot,
+        stateDirectory: workerStateDirectory,
+        provider,
+        model,
+        qwenUrl,
+        nimUrl,
+        ollamaUrl,
+        reasoning,
+        signal,
+        candidateWorkItem: workItem,
+        recoveryOnly,
+        emit(event) {
+          const type =
+            event.type === "work_started"
+              ? "working"
+              : ["work_completed", "work_recovered"].includes(event.type)
+                ? "completed"
+                : ["needs_attention", "work_failed", "recovery_waiting"].includes(event.type)
+                  ? "blocked"
+                  : "waiting";
+          report({
+            type,
+            missionId,
+            ...(UUID.test(event.workItemId ?? "") ? { workItemId: event.workItemId } : {}),
+            code: event.type,
+          });
+        },
+      });
+    };
+    const recoverOne = async () => {
+      const ids = await recoveryMissions(directory);
+      if (!ids.length) return false;
+      const missionId =
+        ids.find((id) => !state.recoveryAfter || id > state.recoveryAfter) ?? ids[0];
+      state.recoveryAfter = missionId;
+      await persist();
+      try {
+        const mission = await client.missions.get(missionId, { signal });
+        const root = await bindingFor(mission, null, true);
+        // Missing bindings prevent claim replay; saved completions still settle safely.
+        await runCycle(missionId, root ?? null, undefined, true);
+      } catch (error) {
+        if (!(await revalidate(error, missionId))) throw error;
+      }
+      return true;
+    };
+    const discoverOne = async () => {
+      let loadedPage = false;
+      for (let inspected = 0; inspected < 25; inspected += 1) {
+        if (!state.pendingMissions.length) {
+          if (loadedPage && state.missionCursor === null) return false;
+          let page;
+          try {
+            page = assertPage(
+              await client.missions.listPage(
+                { cursor: state.missionCursor ?? undefined, limit: 25 },
+                { signal },
+              ),
+            );
+          } catch (error) {
+            if (error.status === 422 && error.code === "INVALID_CURSOR") {
+              state.missionCursor = null;
+              await persist();
+              return false;
+            }
+            throw error;
+          }
+          loadedPage = true;
+          if (page.data.some((mission) => !UUID.test(mission?.id)))
+            throw new Error("The Missions API returned an invalid mission identity.");
+          state.pendingMissions.push(...page.data.map((mission) => mission.id));
+          state.missionCursor = page.nextCursor;
+          await persist();
+          if (!state.pendingMissions.length) continue;
+        }
+        const missionId = state.pendingMissions.shift();
+        await persist();
+        let enteredCycle = false;
+        try {
+          const mission = await client.missions.get(missionId, { signal });
+          if (mission.id !== missionId)
+            throw new Error("The Missions API returned the wrong mission.");
+          if (mission.status !== "active") {
+            if (TERMINAL.has(mission.status)) delete state.workCursors[missionId];
+            report({ type: "waiting", missionId, code: "mission_inactive" });
+            continue;
+          }
+          const page = assertPage(
+            await client.missions.workItemsPage(
+              missionId,
+              { cursor: state.workCursors[missionId] ?? undefined, limit: 1 },
+              { signal },
+            ),
+          );
+          state.workCursors[missionId] = page.nextCursor;
+          await persist();
+          if (!page.data.length) continue;
+          if (page.data.length !== 1)
+            throw new Error("The Missions API exceeded the one-item discovery limit.");
+          const workItem = page.data[0];
+          if (!UUID.test(workItem?.id) || workItem.missionId !== missionId)
+            throw new Error("The Missions API returned an invalid work identity.");
+          if (!STAGES.has(workItem.metadata?.kind)) {
+            report({
+              type: "waiting",
+              missionId,
+              workItemId: workItem.id,
+              code: "unsupported_work",
+            });
+            continue;
+          }
+          if (workItem.assignedAgentId && workItem.assignedAgentId !== agent.id) continue;
+          const root = await bindingFor(mission, workItem.metadata.kind);
+          if (root === undefined) continue;
+          enteredCycle = true;
+          await runCycle(missionId, root, workItem);
+          return true;
+        } catch (error) {
+          if (error.status === 422 && error.code === "INVALID_CURSOR") {
+            delete state.workCursors[missionId];
+            await persist();
+            continue;
+          }
+          if (!(await revalidate(error, missionId))) throw error;
+          if (enteredCycle) return true;
+        }
+      }
+      return false;
+    };
+    report({ type: "connected", agentId: agent.id });
+    failures = 0;
+    while (!signal?.aborted) {
+      try {
+        await client.runtime.heartbeat({ label: "missions:local-agent" }, { signal });
+        if (signal?.aborted) break;
+        let handled = false;
+        const recoveryFirst = state.preferRecovery;
+        state.preferRecovery = !state.preferRecovery;
+        await persist();
+        if (recoveryFirst) handled = await recoverOne();
+        if (!handled) handled = await discoverOne();
+        if (!handled && !recoveryFirst) handled = await recoverOne();
+        failures = 0;
+        if (!handled) report({ type: "waiting", code: "no_runnable_work" });
+      } catch (error) {
+        if (signal?.aborted) break;
+        if (await revalidate(error)) {
+          /* Verified identity; a scoped permission can change later. */
+        } else if (!transient(error)) throw error;
+        else {
+          failures += 1;
+          report({ type: "waiting", code: "service_unavailable" });
+        }
+      }
+      try {
+        await sleep(
+          failures
+            ? Math.min(30_000, Math.max(1000, pollMs) * 2 ** Math.min(failures - 1, 5))
+            : pollMs,
+          undefined,
+          { signal },
+        );
+      } catch (error) {
+        if (!signal?.aborted) throw error;
+      }
+    }
+  } finally {
+    await session.release();
+    report({ type: "stopped" });
+  }
+}

package/bin/mission-agent-session.mjs

@@ -0,0 +1,188 @@
+import { createHash, randomUUID } from "node:crypto";
+import { lstat, mkdir, open, rm } from "node:fs/promises";
+import { constants } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
+
+const sessions = new WeakSet();
+
+export function missionAgentOrigin(baseUrl = "https://totemlayer.com") {
+  const url = new URL(baseUrl);
+  if (
+    url.username ||
+    url.password ||
+    url.search ||
+    url.hash ||
+    !["https:", "http:"].includes(url.protocol) ||
+    (url.protocol === "http:" && !["localhost", "127.0.0.1", "[::1]"].includes(url.hostname))
+  )
+    throw new Error("The agent service requires a credential-free HTTPS API URL or loopback HTTP.");
+  return url.origin;
+}
+
+export function missionAgentSessionKey(baseUrl, agentId) {
+  if (typeof agentId !== "string" || !agentId || agentId.length > 200)
+    throw new Error("The agent session needs a verified agent identity.");
+  return createHash("sha256")
+    .update(JSON.stringify([missionAgentOrigin(baseUrl), agentId]))
+    .digest("hex");
+}
+
+function alive(pid) {
+  if (!Number.isSafeInteger(pid) || pid < 1) return true;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    if (error.code === "ESRCH") return false;
+    return true;
+  }
+}
+
+function ownedPrivate(info, directory = false) {
+  return (
+    (directory ? info.isDirectory() : info.isFile()) &&
+    !info.isSymbolicLink() &&
+    (process.platform === "win32" || ((info.mode & 0o077) === 0 && info.uid === process.getuid()))
+  );
+}
+
+export async function ensurePrivateRuntimeDirectory(path) {
+  await mkdir(path, { recursive: true, mode: 0o700 });
+  const info = await lstat(path);
+  if (!ownedPrivate(info, true))
+    throw new Error(
+      "The agent runtime directory must be private, owned by this user, and not a symbolic link.",
+    );
+}
+
+export async function readPrivateRuntimeJson(path, maxBytes = 4096) {
+  const before = await lstat(path);
+  if (!ownedPrivate(before) || before.size > maxBytes)
+    throw new Error("The saved agent runtime record needs inspection.");
+  const file = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+  try {
+    const info = await file.stat();
+    if (
+      !ownedPrivate(info) ||
+      info.size > maxBytes ||
+      info.ino !== before.ino ||
+      info.dev !== before.dev
+    )
+      throw new Error("The saved agent runtime record changed during inspection.");
+    const buffer = Buffer.alloc(maxBytes + 1);
+    const { bytesRead } = await file.read(buffer, 0, buffer.length, 0);
+    if (bytesRead > maxBytes) throw new Error("The saved agent runtime record is too large.");
+    return JSON.parse(buffer.subarray(0, bytesRead).toString("utf8"));
+  } finally {
+    await file.close();
+  }
+}
+
+async function privateRecord(path, origin, agentId) {
+  const record = await readPrivateRuntimeJson(path);
+  if (
+    record.version !== 1 ||
+    record.origin !== origin ||
+    record.agentId !== agentId ||
+    !Number.isSafeInteger(record.pid) ||
+    record.pid < 1 ||
+    typeof record.nonce !== "string" ||
+    !/^[a-f0-9-]{36}$/i.test(record.nonce)
+  )
+    throw new Error("The saved agent lock is incomplete or does not match this agent and API.");
+  return record;
+}
+
+export async function acquireMissionAgentSession(
+  { baseUrl, agentId, signal },
+  { lockDirectory = join(homedir(), ".totem", "agent-runtime-locks") } = {},
+) {
+  const origin = missionAgentOrigin(baseUrl);
+  const key = missionAgentSessionKey(origin, agentId);
+  if (lockDirectory === join(homedir(), ".totem", "agent-runtime-locks"))
+    await ensurePrivateRuntimeDirectory(join(homedir(), ".totem"));
+  await ensurePrivateRuntimeDirectory(lockDirectory);
+  const path = join(lockDirectory, `${key}.lock`);
+  const nonce = randomUUID();
+  for (let attempt = 0; attempt < 10; attempt += 1) {
+    signal?.throwIfAborted();
+    try {
+      const file = await open(path, "wx", 0o600);
+      try {
+        await file.writeFile(
+          JSON.stringify({ version: 1, pid: process.pid, nonce, origin, agentId }),
+        );
+      } finally {
+        await file.close();
+      }
+      const session = { origin, agentId, key };
+      sessions.add(session);
+      let released = false;
+      session.release = async () => {
+        if (released) return;
+        released = true;
+        sessions.delete(session);
+        const current = await privateRecord(path, origin, agentId).catch(() => null);
+        if (current?.nonce === nonce && current.pid === process.pid)
+          await rm(path, { force: true });
+      };
+      return session;
+    } catch (error) {
+      if (error.code !== "EEXIST") throw error;
+    }
+    const record = await privateRecord(path, origin, agentId).catch((error) => {
+      if (error.code === "ENOENT") return null;
+      throw error;
+    });
+    if (!record) {
+      await sleep(25, undefined, { signal });
+      continue;
+    }
+    if (alive(record.pid))
+      throw Object.assign(
+        new Error(
+          "This agent is already running against this Totem API. Stop its existing runtime before starting another profile.",
+        ),
+        { code: "AGENT_ALREADY_RUNNING" },
+      );
+    // Serialize dead-owner cleanup separately, then re-read the main owner.
+    // A stale cleanup guard is kept for inspection rather than risking removal
+    // of another live process's lock.
+    const cleanupPath = `${path}.cleanup`;
+    let cleanup;
+    try {
+      cleanup = await open(cleanupPath, "wx", 0o600);
+    } catch (error) {
+      if (error.code !== "EEXIST") throw error;
+      await sleep(25, undefined, { signal });
+      continue;
+    }
+    try {
+      const latest = await privateRecord(path, origin, agentId).catch((error) => {
+        if (error.code === "ENOENT") return null;
+        throw error;
+      });
+      if (latest && !alive(latest.pid)) await rm(path);
+    } finally {
+      await cleanup.close();
+      await rm(cleanupPath, { force: true });
+    }
+  }
+  throw Object.assign(
+    new Error(
+      "The agent session lock is busy or incomplete; inspect the existing runtime before retrying.",
+    ),
+    { code: "AGENT_SESSION_LOCK_BUSY" },
+  );
+}
+
+export function assertMissionAgentSession(session, baseUrl, agentId) {
+  if (
+    !sessions.has(session) ||
+    session.origin !== missionAgentOrigin(baseUrl) ||
+    session.agentId !== agentId
+  )
+    throw new Error("The worker does not own this verified agent session.");
+}