surf-cli 2.9.0 → 2.11.0

package/native/oracle-jobs.cjs

@@ -0,0 +1,253 @@
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+const {
+  assertNotSymlink,
+  atomicWriteFile,
+  atomicWriteJson,
+  ensurePrivateDir,
+  getPrivateStateRoot,
+  readPrivateFile,
+  readPrivateJson,
+} = require("./private-state.cjs");
+
+const JOB_ID_PATTERN = /^\d{8}-\d{6}-[0-9a-f]{4}$/;
+const TERMINAL_STATES = new Set(["captured", "failed"]);
+const TRANSITIONS = {
+  created: new Set(["dispatched", "failed"]),
+  dispatched: new Set(["awaiting", "failed"]),
+  awaiting: new Set(["captured", "failed"]),
+};
+
+function oracleRoot(root = getPrivateStateRoot()) {
+  return path.join(root, "oracle");
+}
+
+function jobDirectory(id, root = getPrivateStateRoot()) {
+  if (!JOB_ID_PATTERN.test(id)) throw codedError("not_found", `oracle job not found: ${id}`);
+  return path.join(oracleRoot(root), id);
+}
+
+function codedError(code, message, details = {}) {
+  const error = new Error(message);
+  error.code = code;
+  Object.assign(error, details);
+  return error;
+}
+
+function readJobs(root = getPrivateStateRoot()) {
+  const base = oracleRoot(root);
+  if (!fs.existsSync(base)) return [];
+  const stat = assertNotSymlink(base, false);
+  if (!stat.isDirectory()) throw new Error(`oracle state path is not a directory: ${base}`);
+  return fs.readdirSync(base)
+    .filter((id) => JOB_ID_PATTERN.test(id))
+    .sort((a, b) => b.localeCompare(a))
+    .map((id) => readPrivateJson(path.join(base, id, "job.json"), null, { root }))
+    .filter(Boolean);
+}
+
+function createJob({ prompt, contextManifest = {}, model = null, effortRequested = null, follow = null }) {
+  const root = getPrivateStateRoot();
+  const base = oracleRoot(root);
+  ensurePrivateDir(base, root);
+  const inFlight = readJobs(root).find((job) => !TERMINAL_STATES.has(job.state));
+  if (inFlight) {
+    throw codedError(
+      "capacity",
+      `oracle job capacity reached; in-flight job: ${inFlight.id}`,
+      { jobId: inFlight.id },
+    );
+  }
+
+  const now = new Date();
+  const compactTimestamp = now.toISOString().replace(/\D/g, "").slice(0, 14);
+  const timestamp = `${compactTimestamp.slice(0, 8)}-${compactTimestamp.slice(8)}`;
+  let id;
+  let directory;
+  for (;;) {
+    id = `${timestamp}-${crypto.randomBytes(2).toString("hex")}`;
+    directory = path.join(base, id);
+    try {
+      fs.mkdirSync(directory, { mode: 0o700 });
+      break;
+    } catch (error) {
+      if (error?.code !== "EEXIST") throw error;
+    }
+  }
+
+  try {
+    ensurePrivateDir(path.join(directory, "turns"), root);
+    atomicWriteFile(path.join(directory, "request.md"), prompt, { root, encoding: "utf8" });
+    atomicWriteJson(path.join(directory, "context-manifest.json"), contextManifest, { root });
+    const job = {
+      id,
+      state: "created",
+      model,
+      effortRequested,
+      effortVerified: null,
+      createdAt: now.toISOString(),
+      dispatchedAt: null,
+      awaitingAt: null,
+      capturedAt: null,
+      failedAt: null,
+      tabId: null,
+      conversationUrl: null,
+      promptEcho: null,
+      error: null,
+      turns: [],
+      ...(follow ? { follow } : {}),
+    };
+    atomicWriteJson(path.join(directory, "job.json"), job, { root });
+    return job;
+  } catch (error) {
+    fs.rmSync(directory, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+function getJob(id) {
+  const root = getPrivateStateRoot();
+  const job = readPrivateJson(path.join(jobDirectory(id, root), "job.json"), null, { root });
+  if (!job) throw codedError("not_found", `oracle job not found: ${id}`);
+  return job;
+}
+
+function getResponse(id) {
+  const root = getPrivateStateRoot();
+  getJob(id);
+  return readPrivateFile(path.join(jobDirectory(id, root), "response.md"), {
+    root,
+    encoding: "utf8",
+  });
+}
+
+function transition(id, state, updates) {
+  const job = getJob(id);
+  if (!TRANSITIONS[job.state]?.has(state)) {
+    throw codedError(
+      "invalid_transition",
+      `oracle job ${id} cannot transition from ${job.state} to ${state}`,
+    );
+  }
+  const updated = { ...job, state, ...updates };
+  const root = getPrivateStateRoot();
+  atomicWriteJson(path.join(jobDirectory(id, root), "job.json"), updated, { root });
+  return updated;
+}
+
+function markDispatched(id, { tabId, promptEcho, modelVerified, effortVerified }) {
+  return transition(id, "dispatched", {
+    dispatchedAt: new Date().toISOString(),
+    tabId,
+    ...(promptEcho ? { promptEcho } : {}),
+    ...(modelVerified ? { model: modelVerified } : {}),
+    ...(effortVerified ? { effortVerified } : {}),
+  });
+}
+
+function markAwaiting(id, { conversationUrl, promptEcho }) {
+  return transition(id, "awaiting", {
+    awaitingAt: new Date().toISOString(),
+    conversationUrl,
+    promptEcho,
+  });
+}
+
+function markCaptured(id, { response }) {
+  const job = getJob(id);
+  if (!TRANSITIONS[job.state]?.has("captured")) {
+    throw codedError(
+      "invalid_transition",
+      `oracle job ${id} cannot transition from ${job.state} to captured`,
+    );
+  }
+  const root = getPrivateStateRoot();
+  atomicWriteFile(path.join(jobDirectory(id, root), "response.md"), response, {
+    root,
+    encoding: "utf8",
+  });
+  return transition(id, "captured", { capturedAt: new Date().toISOString() });
+}
+
+function markFailed(id, { code, message }) {
+  return transition(id, "failed", {
+    failedAt: new Date().toISOString(),
+    error: { code, message },
+  });
+}
+
+function updateTabId(id, tabId) {
+  const job = getJob(id);
+  if (TERMINAL_STATES.has(job.state)) {
+    throw codedError(
+      "invalid_transition",
+      `oracle job ${id} cannot transition from ${job.state} to update tab`,
+    );
+  }
+  const updated = { ...job, tabId };
+  const root = getPrivateStateRoot();
+  atomicWriteJson(path.join(jobDirectory(id, root), "job.json"), updated, { root });
+  return updated;
+}
+
+function appendTurn(id, turn) {
+  const job = getJob(id);
+  const storedTurn = {
+    prompt: turn.prompt,
+    dispatchedAt: turn.dispatchedAt ?? null,
+    capturedAt: turn.capturedAt ?? null,
+  };
+  const root = getPrivateStateRoot();
+  const directory = jobDirectory(id, root);
+  const turnName = `${String(job.turns.length + 1).padStart(4, "0")}.json`;
+  atomicWriteJson(path.join(directory, "turns", turnName), storedTurn, { root });
+  const updated = { ...job, turns: [...job.turns, storedTurn] };
+  atomicWriteJson(path.join(directory, "job.json"), updated, { root });
+  return updated;
+}
+
+function markTurnCaptured(id, { dispatchedAt, capturedAt }) {
+  const job = getJob(id);
+  const turnIndex = job.turns.findIndex((turn) => turn.dispatchedAt === dispatchedAt);
+  if (turnIndex === -1) {
+    throw codedError(
+      "invalid_transition",
+      `oracle job ${id} has no follow turn dispatched at ${dispatchedAt}`,
+    );
+  }
+  const turns = [...job.turns];
+  turns[turnIndex] = { ...turns[turnIndex], capturedAt };
+  const root = getPrivateStateRoot();
+  const directory = jobDirectory(id, root);
+  const turnName = `${String(turnIndex + 1).padStart(4, "0")}.json`;
+  atomicWriteJson(path.join(directory, "turns", turnName), turns[turnIndex], { root });
+  const updated = { ...job, turns };
+  atomicWriteJson(path.join(directory, "job.json"), updated, { root });
+  return updated;
+}
+
+function listJobs({ limit } = {}) {
+  const jobs = readJobs();
+  return limit === undefined ? jobs : jobs.slice(0, Math.max(0, limit));
+}
+
+function adoptOrphans() {
+  return listJobs({}).filter((job) => !TERMINAL_STATES.has(job.state));
+}
+
+module.exports = {
+  adoptOrphans,
+  appendTurn,
+  createJob,
+  getJob,
+  getResponse,
+  listJobs,
+  markAwaiting,
+  markCaptured,
+  markDispatched,
+  markFailed,
+  markTurnCaptured,
+  oracleRoot,
+  updateTabId,
+};

package/native/playbook-authoring.cjs

@@ -0,0 +1,44 @@
+const fs = require("fs");
+const path = require("path");
+const { readRecent } = require("./activity-journal.cjs");
+const { writeNetworkExport } = require("./network-export.cjs");
+const { getPrivateStateRoot, readPrivateJson } = require("./private-state.cjs");
+const { draftFromRecord, readRecord, recordsRoot } = require("./playbook-records.cjs");
+const { savePlaybook, validateOp } = require("./playbooks.cjs");
+const { commandMetadata, promoteRedactedStepArgs } = require("./workflow-definition.cjs");
+
+function suggestions({ since = "1h", root } = {}) {
+  const events = readRecent({ since, root });
+  const counts = new Map();
+  for (const event of events.filter((entry) => entry.type === "tool.issued")) counts.set(event.command, (counts.get(event.command) || 0) + 1);
+  return [...counts.entries()].map(([command, count]) => ({ command, count })).sort((a, b) => b.count - a.count || a.command.localeCompare(b.command));
+}
+
+function saveFromRecent({ site, op: opId, since = "1h", scope = "user", cwd, home, root }) {
+  const events = readRecent({ since, root }).filter((event) => event.type === "tool.issued");
+  if (events.length === 0) throw new Error("no recent Surf activity to save");
+  if (events.some((event) => ["page-write", "unknown"].includes(commandMetadata(event.command).effect))) {
+    throw new Error("recent activity includes write-capable commands; use an explicit record and review its draft before saving");
+  }
+  const promoted = promoteRedactedStepArgs(events.map((event) => ({ tool: event.command, args: event.argsRedacted || {} })));
+  const op = { id: opId, description: "Drafted from recent Surf activity", effect: "read", args: promoted.args, run: [{ using: "workflow", steps: promoted.steps }], provenance: { recentSince: since } };
+  validateOp(op, { origins: [] });
+  return savePlaybook({ manifest: { id: site, name: site, version: "1.0.0", origins: [] }, op, scope, cwd, home });
+}
+
+function saveFromRecord({ recordId, scope = "user", cwd, home, root = getPrivateStateRoot() }) {
+  const record = readRecord(recordId, root);
+  if (!record) throw new Error(`record not found: ${recordId}`);
+  const draftPath = path.join(recordsRoot(root), recordId, "draft", "op.json");
+  const op = fs.existsSync(draftPath) ? readPrivateJson(draftPath, null, { root }) : draftFromRecord(recordId, root);
+  validateOp(op, { origins: record.origin ? [record.origin] : [] });
+  return savePlaybook({ manifest: { id: record.site, name: record.site, version: "1.0.0", origins: record.origin ? [record.origin] : [] }, op, scope, cwd, home });
+}
+
+function exportRecordHar(recordId, output, root = getPrivateStateRoot()) {
+  const trace = readPrivateJson(path.join(recordsRoot(root), recordId, "network", "trace.json"), null, { root });
+  if (!trace) throw new Error(`record ${recordId} has no network trace`);
+  return writeNetworkExport(path.resolve(output), trace.entries, "har");
+}
+
+module.exports = { exportRecordHar, saveFromRecent, saveFromRecord, suggestions };

package/native/playbook-cli.cjs

@@ -0,0 +1,157 @@
+const path = require("path");
+const { openClientTransport } = require("./client-transport.cjs");
+const { resolveRequestDeadlineMs } = require("./host-sessions.cjs");
+const { exportRecordHar, saveFromRecent, saveFromRecord, suggestions } = require("./playbook-authoring.cjs");
+const { deriveClient, exportClient, verifyClient } = require("./playbook-client.cjs");
+const { exportPlaybookDirectory, importPlaybookDirectory, listPlaybooks, resolvePlaybook } = require("./playbooks.cjs");
+
+function parseCommandArgs(argv) {
+  const positional = [];
+  const options = {};
+  for (let index = 0; index < argv.length; index++) {
+    const value = argv[index];
+    if (!value.startsWith("--")) {
+      positional.push(value);
+      continue;
+    }
+    const name = value.slice(2);
+    const next = argv[index + 1];
+    if (next !== undefined && !next.startsWith("--")) {
+      options[name] = /^-?\d+(?:\.\d+)?$/.test(next) ? Number(next) : next;
+      index++;
+    } else options[name] = true;
+  }
+  return { positional, options };
+}
+
+function unwrapResponse(response) {
+  if (response.error) throw new Error(response.error.content?.[0]?.text || JSON.stringify(response.error));
+  const text = response.result?.content?.[0]?.text;
+  if (text === undefined) return response.result;
+  try { return JSON.parse(text); } catch { return text; }
+}
+
+async function requestHost(endpoint, tool, args, options = {}) {
+  const transport = await openClientTransport(endpoint, { requestTimeoutMs: options.timeoutMs || 11 * 60 * 1000 });
+  try {
+    const request = { type: "tool_request", method: "execute_tool", params: { tool, args }, id: `playbook-${Date.now()}-${Math.random()}` };
+    if (options.tabId) request.tabId = options.tabId;
+    return unwrapResponse(await transport.request(request, options.timeoutMs || 11 * 60 * 1000));
+  } finally {
+    await transport.close();
+  }
+}
+
+function runSpec(argv) {
+  const direct = argv[0] === "use";
+  const offset = direct ? 1 : 2;
+  const parsed = parseCommandArgs(argv.slice(offset));
+  const [playbook, op] = parsed.positional;
+  if (!playbook || !op) throw new Error(direct ? "Usage: surf use <playbook> <op> [--arg value]" : "Usage: surf pb run <playbook> <op> [--arg value]");
+  const reserved = new Set(["json", "no-lock", "tab-id", "write", "repeat", "retry-attempt", "override-in-doubt", "pin-built-in"]);
+  const args = Object.fromEntries(Object.entries(parsed.options).filter(([name]) => !reserved.has(name)));
+  return { playbook, op, args, options: parsed.options };
+}
+
+function resolveRunTimeout(spec, cwd) {
+  const explicit = Number(spec.args.timeout);
+  if (Number.isFinite(explicit) && explicit > 0) return explicit;
+  try {
+    const playbook = resolvePlaybook(spec.playbook, { cwd, pinBuiltIn: spec.options["pin-built-in"] === true });
+    const op = playbook.ops.get(spec.op);
+    const value = Number(op?.args?.timeout?.default);
+    return Number.isFinite(value) && value > 0 ? value : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function playbookCommandNeedsBrowser(argv) {
+  if (argv[0] === "use") return true;
+  const subcommand = argv[1];
+  if (subcommand === "run") return true;
+  return subcommand === "record" && ["start", "stop", "discard"].includes(argv[2]);
+}
+
+async function handlePlaybookCli(argv, { endpoint, cwd = process.cwd() }) {
+  if (!["playbook", "pb", "use"].includes(argv[0])) return { handled: false };
+  if (argv[0] === "use" || argv[1] === "run") {
+    const spec = runSpec(argv);
+    const timeout = resolveRunTimeout(spec, cwd);
+    const args = {
+      playbook: spec.playbook,
+      op: spec.op,
+      args: spec.args,
+      projectDir: cwd,
+      ...(timeout ? { timeout } : {}),
+      write: spec.options.write === true,
+      repeat: spec.options.repeat === true,
+      retryAttempt: spec.options["retry-attempt"],
+      overrideInDoubt: spec.options["override-in-doubt"] === true,
+      pinBuiltIn: spec.options["pin-built-in"] === true,
+    };
+    const value = await requestHost(endpoint, "playbook.run", args, {
+      tabId: spec.options["tab-id"],
+      timeoutMs: resolveRequestDeadlineMs("playbook.run", args),
+    });
+    return { handled: true, value, json: spec.options.json === true };
+  }
+  const command = argv[1];
+  const parsed = parseCommandArgs(argv.slice(2));
+  if (!command || command === "help") return { handled: true, value: "Usage: surf playbook|pb <list|show|ops|run|record|suggest|save|client|trace|export|import>" };
+  if (endpoint?.kind === "remote" && ["list", "show", "ops"].includes(command)) throw new Error(`playbook ${command} is local-only with --remote because runs resolve on the browser host`);
+  if (command === "list") return { handled: true, value: listPlaybooks({ cwd }), json: parsed.options.json === true };
+  if (command === "show") {
+    const playbook = resolvePlaybook(parsed.positional[0], { cwd });
+    return { handled: true, value: { id: playbook.id, name: playbook.name, version: playbook.version, description: playbook.description, origins: playbook.origins, provenance: playbook.provenance, ops: [...playbook.ops.keys()] }, json: parsed.options.json === true };
+  }
+  if (command === "ops") {
+    const playbook = resolvePlaybook(parsed.positional[0], { cwd });
+    return { handled: true, value: [...playbook.ops.values()].map((op) => ({ id: op.id, description: op.description || "", effect: op.effect, strategies: op.run.map((strategy) => strategy.using) })), json: parsed.options.json === true };
+  }
+  if (command === "record") {
+    const action = parsed.positional[0];
+    const tool = `playbook.record.${action}`;
+    let args = {};
+    if (action === "start") args = { site: parsed.positional[1], op: parsed.options.op, watch: parsed.options.watch === true, network: parsed.options.network === true, includeInputValues: parsed.options["include-input-values"] === true };
+    else if (action === "mark") args = { label: parsed.positional.slice(1).join(" ") };
+    else if (action === "stop") args = { draft: parsed.options.draft === true };
+    else if (!["status", "pause", "resume", "discard"].includes(action)) throw new Error("Unknown record command");
+    const value = await requestHost(endpoint, tool, args, { tabId: parsed.options["tab-id"] });
+    return { handled: true, value, json: parsed.options.json === true };
+  }
+  if (command === "suggest") return { handled: true, value: suggestions({ since: parsed.options.since || "1h" }), json: parsed.options.json === true };
+  if (command === "save") {
+    let value;
+    if (parsed.options["from-record"]) value = saveFromRecord({ recordId: parsed.options["from-record"], scope: parsed.options.project ? "project" : "user", cwd });
+    else if (parsed.options["from-recent"] || parsed.positional[0]) value = saveFromRecent({ site: parsed.positional[0], op: parsed.options.op, since: parsed.options["from-recent"] === true ? "1h" : parsed.options["from-recent"] || "1h", scope: parsed.options.project ? "project" : "user", cwd });
+    else throw new Error("save requires --from-record <id> or <site> --op <name> --from-recent");
+    return { handled: true, value, json: parsed.options.json === true };
+  }
+  if (command === "client") {
+    const action = parsed.positional[0];
+    if (action === "derive") return { handled: true, value: deriveClient(parsed.positional[1], parsed.options.op, parsed.options.out, { recordId: parsed.options["from-record"], requestId: parsed.options["request-id"] }), json: parsed.options.json === true };
+    if (action === "export") {
+      const playbook = parsed.positional[1];
+      const resolved = resolvePlaybook(playbook, { cwd });
+      const op = parsed.options.op || [...resolved.ops.keys()][0];
+      return { handled: true, value: exportClient(playbook, op, parsed.options.out, { cwd }), json: parsed.options.json === true };
+    }
+    if (action === "verify") return { handled: true, value: await verifyClient(parsed.positional[1], { live: parsed.options.live === true ? true : undefined }), json: parsed.options.json === true };
+    throw new Error("Unknown client command");
+  }
+  if (command === "trace" && parsed.positional[0] === "export") {
+    if (!parsed.options["from-record"] || !parsed.options.har) throw new Error("trace export requires --from-record <id> --har <path>");
+    return { handled: true, value: exportRecordHar(parsed.options["from-record"], path.resolve(parsed.options.har)), json: parsed.options.json === true };
+  }
+  if (command === "export") return { handled: true, value: exportPlaybookDirectory(parsed.positional[0], { out: parsed.options.out, cwd }), json: parsed.options.json === true };
+  if (command === "import") return { handled: true, value: importPlaybookDirectory(parsed.positional[0], { scope: parsed.options.project ? "project" : "user", cwd }), json: parsed.options.json === true };
+  throw new Error(`Unknown playbook command: ${command}`);
+}
+
+function formatPlaybookOutput(value, json = false) {
+  if (json || typeof value !== "string") return JSON.stringify(value, null, 2);
+  return value;
+}
+
+module.exports = { formatPlaybookOutput, handlePlaybookCli, playbookCommandNeedsBrowser };