surf-cli 2.8.0 → 2.10.0

package/native/playbook-receipts.cjs

@@ -0,0 +1,109 @@
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+const {
+  atomicWriteJson,
+  ensurePrivateDir,
+  getPrivateStateRoot,
+  readPrivateFile,
+  readPrivateJson,
+  writePrivateFileExclusive,
+} = require("./private-state.cjs");
+
+function receiptRoot(root = getPrivateStateRoot()) {
+  return path.join(root, "playbook-receipts");
+}
+
+function loadSalt(root) {
+  const directory = receiptRoot(root);
+  ensurePrivateDir(directory, root);
+  const saltPath = path.join(directory, ".salt");
+  const existing = readPrivateFile(saltPath, { root, allowMissing: true, fallback: null, encoding: "utf8" });
+  if (existing) return Buffer.from(existing.trim(), "hex");
+  const salt = crypto.randomBytes(32);
+  try {
+    writePrivateFileExclusive(saltPath, `${salt.toString("hex")}\n`, { root, encoding: "utf8" });
+  } catch (error) {
+    if (error?.code !== "EEXIST") throw error;
+  }
+  return Buffer.from(readPrivateFile(saltPath, { root, encoding: "utf8" }).trim(), "hex");
+}
+
+function semanticClaimKey({ playbookId, op, args, root = getPrivateStateRoot() }) {
+  const semantic = {};
+  for (const name of op.safety.key) {
+    if (args[name] === undefined) throw new Error(`semantic safety argument is missing: ${name}`);
+    semantic[name] = args[name];
+  }
+  const message = JSON.stringify({ playbook: playbookId, op: op.id, semantic });
+  return crypto.createHmac("sha256", loadSalt(root)).update(message).digest("hex");
+}
+
+function attemptFiles(claimDir) {
+  if (!fs.existsSync(claimDir)) return [];
+  return fs.readdirSync(claimDir)
+    .filter((name) => name.endsWith(".json"))
+    .map((name) => readPrivateJson(path.join(claimDir, name), null, { root: path.dirname(claimDir) }))
+    .filter(Boolean)
+    .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)));
+}
+
+function reserveReceipt({ playbookId, op, args, repeat = false, retryAttempt, overrideInDoubt = false, attemptId = crypto.randomUUID(), now = Date.now(), root = getPrivateStateRoot() }) {
+  if (op.effect !== "write") return null;
+  const base = receiptRoot(root);
+  ensurePrivateDir(base, root);
+  const claimKey = semanticClaimKey({ playbookId, op, args, root });
+  const claimDir = path.join(base, claimKey);
+  let created = false;
+  try {
+    fs.mkdirSync(claimDir, { mode: 0o700 });
+    created = true;
+  } catch (error) {
+    if (error?.code !== "EEXIST") throw error;
+  }
+  if (!created) {
+    const existing = attemptFiles(claimDir);
+    const latest = existing.at(-1);
+    if (!latest) {
+      throw new Error("write blocked by an unresolved semantic claim created before an attempt receipt was durable");
+    }
+    if (retryAttempt) {
+      const retry = existing.find((attempt) => attempt.attemptId === retryAttempt);
+      if (!retry) throw new Error(`receipt attempt not found: ${retryAttempt}`);
+      if (retry.status === "verified") throw new Error(`receipt attempt is already verified: ${retryAttempt}`);
+      if (retry.status !== "reserved" && retry.status !== "not_dispatched" && !op.safety.serverIdempotency) {
+        throw new Error("--retry-attempt after dispatch requires declared server idempotency");
+      }
+      return { claimKey, claimDir, attemptId: retryAttempt, path: path.join(claimDir, `${retryAttempt}.json`), receipt: retry };
+    }
+    if (latest) {
+      const ageMs = Math.max(0, now - Date.parse(latest.updatedAt || latest.createdAt));
+      const windowMs = op.safety.windowMs ?? 30000;
+      const windowElapsed = op.safety.duplicate === "repeatable-window" && latest.status === "verified" && ageMs >= windowMs;
+      const deliberateOverride = overrideInDoubt && latest.status === "indeterminate";
+      const deliberateRepeat = repeat && latest.status === "verified";
+      if (!windowElapsed && !deliberateOverride && !deliberateRepeat) {
+        throw new Error(`write blocked by ${latest.status} receipt ${latest.attemptId}; use the policy-specific retry or override flag`);
+      }
+    }
+  }
+  const createdAt = new Date(now).toISOString();
+  const receipt = { version: 1, playbook: playbookId, op: op.id, claimKey, attemptId, status: "reserved", duplicate: op.safety.duplicate, createdAt, updatedAt: createdAt };
+  const receiptPath = path.join(claimDir, `${attemptId}.json`);
+  atomicWriteJson(path.join(claimDir, "claim"), { version: 1, playbook: playbookId, op: op.id, claimKey }, { root });
+  atomicWriteJson(receiptPath, receipt, { root });
+  return { claimKey, claimDir, attemptId, path: receiptPath, receipt };
+}
+
+function updateReceipt(handle, status, details = {}, root = getPrivateStateRoot()) {
+  if (!handle) return null;
+  if (!["reserved", "not_dispatched", "dispatched", "indeterminate", "verified"].includes(status)) throw new Error(`invalid receipt status: ${status}`);
+  const current = readPrivateJson(handle.path, null, { root });
+  if (!current) throw new Error(`receipt is missing: ${handle.attemptId}`);
+  const receipt = { ...current, status, updatedAt: new Date().toISOString(), ...(details.error ? { errorCode: crypto.createHash("sha256").update(String(details.error)).digest("hex").slice(0, 16) } : {}) };
+  atomicWriteJson(handle.path, receipt, { root });
+  handle.receipt = receipt;
+  return receipt;
+}
+
+module.exports = { receiptRoot, reserveReceipt, semanticClaimKey, updateReceipt };

package/native/playbook-records.cjs

@@ -0,0 +1,208 @@
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+const {
+  appendPrivateJsonLine,
+  assertWithin,
+  atomicWriteJson,
+  ensurePrivateDir,
+  getPrivateStateRoot,
+  readPrivateFile,
+  readPrivateJson,
+} = require("./private-state.cjs");
+const { redactSensitiveFields, redactUrlSecrets, safeHeaders } = require("./redaction.cjs");
+const { commandMetadata, promoteRedactedStepArgs } = require("./workflow-definition.cjs");
+const { version: PACKAGE_VERSION } = require("../package.json");
+
+function recordsRoot(root = getPrivateStateRoot()) {
+  return path.join(root, "records");
+}
+
+function activePath(root = getPrivateStateRoot()) {
+  return path.join(recordsRoot(root), "active.json");
+}
+
+function recordDirectory(recordId, root = getPrivateStateRoot()) {
+  if (typeof recordId !== "string" || !/^rec-[a-zA-Z0-9-]+$/.test(recordId)) throw new Error("record ID is invalid");
+  return path.join(recordsRoot(root), recordId);
+}
+
+function readRecord(recordId, root = getPrivateStateRoot()) {
+  return readPrivateJson(path.join(recordDirectory(recordId, root), "record.json"), null, { root });
+}
+
+function activeRecord(root = getPrivateStateRoot()) {
+  const active = readPrivateJson(activePath(root), null, { root });
+  return active ? readRecord(active.recordId, root) : null;
+}
+
+function writeRecord(record, root) {
+  atomicWriteJson(path.join(recordDirectory(record.id, root), "record.json"), record, { root });
+}
+
+function startRecord({ site, op, watch = false, network = false, includeInputValues = false, tabId, origin, root = getPrivateStateRoot() }) {
+  if (!site || !op) throw new Error("record start requires site and op");
+  if (activeRecord(root)) throw new Error("a playbook record is already active");
+  const id = `rec-${Date.now()}-${crypto.randomBytes(4).toString("hex")}`;
+  const directory = recordDirectory(id, root);
+  ensurePrivateDir(directory, root);
+  const record = {
+    version: 1,
+    id,
+    site,
+    op,
+    status: "recording",
+    capture: { watch, network },
+    redaction: { includeInputValues },
+    provenance: { generator: "surf-cli", version: PACKAGE_VERSION },
+    ...(tabId ? { tabId } : {}),
+    ...(origin ? { origin } : {}),
+    startedAt: new Date().toISOString(),
+    eventCount: 0,
+  };
+  writeRecord(record, root);
+  atomicWriteJson(activePath(root), { recordId: id }, { root });
+  return record;
+}
+
+function appendRecordEvent(event, { root = getPrivateStateRoot(), allowPaused = false } = {}) {
+  const record = activeRecord(root);
+  if (!record || (record.status !== "recording" && !allowPaused)) return false;
+  appendPrivateJsonLine(path.join(recordDirectory(record.id, root), "events.jsonl"), { version: 1, recordId: record.id, ...event }, { root });
+  record.eventCount++;
+  writeRecord(record, root);
+  return true;
+}
+
+function updateActiveStatus(status, root = getPrivateStateRoot()) {
+  const record = activeRecord(root);
+  if (!record) throw new Error("no active playbook record");
+  const allowed = { recording: ["paused", "stopping"], paused: ["recording", "stopping"] };
+  if (!allowed[record.status]?.includes(status)) throw new Error(`cannot change record from ${record.status} to ${status}`);
+  record.status = status;
+  writeRecord(record, root);
+  return record;
+}
+
+function updateRecordContext({ tabId, origin }, root = getPrivateStateRoot()) {
+  const record = activeRecord(root);
+  if (!record) throw new Error("no active playbook record");
+  if (tabId) record.tabId = tabId;
+  if (origin) record.origin = origin;
+  writeRecord(record, root);
+  return record;
+}
+
+function markRecord(label, root = getPrivateStateRoot()) {
+  if (typeof label !== "string" || !label.trim()) throw new Error("record mark requires text");
+  const record = activeRecord(root);
+  if (!record) throw new Error("no active playbook record");
+  appendRecordEvent({ type: "mark", label: label.trim(), timestamp: new Date().toISOString() }, { root, allowPaused: true });
+  return activeRecord(root);
+}
+
+function readEvents(recordId, root = getPrivateStateRoot()) {
+  const filePath = path.join(recordDirectory(recordId, root), "events.jsonl");
+  const content = readPrivateFile(filePath, { root, allowMissing: true, fallback: "", encoding: "utf8" });
+  return content.split("\n").filter(Boolean).map((line) => JSON.parse(line));
+}
+
+function sanitizeTraceEntry(entry, includeInputValues = false) {
+  return {
+    ...redactSensitiveFields(entry),
+    url: redactUrlSecrets(entry.url),
+    requestHeaders: safeHeaders(entry.requestHeaders),
+    responseHeaders: safeHeaders(entry.responseHeaders),
+    responseBody: entry.responseBody === undefined ? undefined : "<response-body>",
+    ...(includeInputValues ? {} : { requestBody: entry.requestBody === undefined ? undefined : "<request-body>" }),
+  };
+}
+
+function attachNetworkTrace(recordId, entries, root = getPrivateStateRoot()) {
+  const record = readRecord(recordId, root);
+  if (!record) throw new Error(`record not found: ${recordId}`);
+  const sanitized = entries.map((entry) => sanitizeTraceEntry(entry, record.redaction.includeInputValues));
+  const directory = path.join(recordDirectory(recordId, root), "network");
+  ensurePrivateDir(directory, root);
+  const tracePath = path.join(directory, "trace.json");
+  atomicWriteJson(tracePath, { version: 1, recordId, capturedAt: new Date().toISOString(), entries: sanitized }, { root });
+  record.trace = { path: "network/trace.json", count: sanitized.length };
+  writeRecord(record, root);
+  return { path: tracePath, count: sanitized.length };
+}
+
+function draftFromRecord(recordId, root = getPrivateStateRoot()) {
+  const record = readRecord(recordId, root);
+  if (!record) throw new Error(`record not found: ${recordId}`);
+  const events = readEvents(recordId, root).filter((event) => event.type === "tool.completed");
+  let effect = "read";
+  const steps = [];
+  for (const event of events) {
+    const metadata = commandMetadata(event.command);
+    if (!metadata.recordable) continue;
+    if (["page-write", "unknown"].includes(metadata.effect)) effect = "write";
+    steps.push({ tool: event.command, args: event.argsRedacted || {} });
+  }
+  const tracePath = path.join(recordDirectory(recordId, root), "network", "trace.json");
+  const trace = readPrivateJson(tracePath, null, { root });
+  const strategies = [];
+  const observed = trace?.entries?.findLast((entry) => ["GET", "HEAD", "OPTIONS", "POST"].includes(entry.method) && entry.status >= 200 && entry.status < 400);
+  if (observed) {
+    const url = new URL(observed.url);
+    const query = Object.fromEntries(url.searchParams.entries());
+    url.search = "";
+    strategies.push({ using: "network", request: { method: observed.method, url: url.toString(), query, headers: safeHeaders(observed.requestHeaders), ...(observed.requestBody !== undefined ? { body: observed.requestBody } : {}) } });
+  }
+  let args = {};
+  if (steps.length > 0) {
+    const promoted = promoteRedactedStepArgs(steps);
+    args = promoted.args;
+    strategies.push({ using: "workflow", steps: promoted.steps });
+  }
+  if (strategies.length === 0) throw new Error("record has no executable evidence");
+  const safety = effect === "write" ? { authorization: "explicit", duplicate: "transactional", key: Object.keys(args).length ? Object.keys(args) : ["review_key"] } : undefined;
+  if (effect === "write" && !args.review_key && safety.key.includes("review_key")) args.review_key = { required: true, desc: "Semantic key for reviewed write replay" };
+  const op = { id: record.op, description: `Drafted from ${record.id}`, effect, args, ...(safety ? { safety } : {}), run: strategies, provenance: { recordId: record.id } };
+  const draftDir = path.join(recordDirectory(recordId, root), "draft");
+  ensurePrivateDir(draftDir, root);
+  atomicWriteJson(path.join(draftDir, "op.json"), op, { root });
+  return op;
+}
+
+function stopRecord({ draft = false, root = getPrivateStateRoot() } = {}) {
+  let record = updateActiveStatus("stopping", root);
+  let op;
+  if (draft) op = draftFromRecord(record.id, root);
+  record = readRecord(record.id, root);
+  record.status = draft ? "draft_created" : "stopped";
+  record.stoppedAt = new Date().toISOString();
+  writeRecord(record, root);
+  fs.unlinkSync(activePath(root));
+  return { record, ...(op ? { draft: op } : {}) };
+}
+
+function discardRecord(root = getPrivateStateRoot()) {
+  const record = activeRecord(root);
+  if (!record) throw new Error("no active playbook record");
+  const directory = assertWithin(recordsRoot(root), recordDirectory(record.id, root));
+  fs.rmSync(directory, { recursive: true, force: true });
+  try { fs.unlinkSync(activePath(root)); } catch {}
+  return { discarded: record.id };
+}
+
+module.exports = {
+  activeRecord,
+  appendRecordEvent,
+  attachNetworkTrace,
+  discardRecord,
+  draftFromRecord,
+  markRecord,
+  pauseRecord: (root) => updateActiveStatus("paused", root),
+  readEvents,
+  readRecord,
+  recordsRoot,
+  resumeRecord: (root) => updateActiveStatus("recording", root),
+  startRecord,
+  stopRecord,
+  updateRecordContext,
+};

package/native/playbook-runtime.cjs

@@ -0,0 +1,177 @@
+const { executeWorkflow } = require("./workflow-runtime.cjs");
+
+function applyTemplate(value, args) {
+  if (typeof value === "string") {
+    const exact = value.match(/^\{\{([a-zA-Z0-9._-]+)\}\}$/);
+    if (exact) return args[exact[1]];
+    return value.replace(/\{\{([a-zA-Z0-9._-]+)\}\}/g, (_, name) => {
+      if (args[name] === undefined) throw new Error(`missing template argument: ${name}`);
+      return String(args[name]);
+    });
+  }
+  if (Array.isArray(value)) return value.map((item) => applyTemplate(item, args));
+  if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, applyTemplate(item, args)]));
+  return value;
+}
+
+function resolveArgs(op, provided) {
+  const args = { ...provided };
+  for (const [name, spec] of Object.entries(op.args || {})) {
+    if (args[name] === undefined && spec.default !== undefined) args[name] = spec.default;
+    if (spec.required && args[name] === undefined) throw new Error(`Missing required argument: --${name}`);
+  }
+  return args;
+}
+
+function getJsonPath(value, jsonPath) {
+  if (!jsonPath || jsonPath === "$") return value;
+  if (typeof jsonPath !== "string" || !jsonPath.startsWith("$.")) throw new Error(`unsupported jsonPath: ${jsonPath}`);
+  let current = value;
+  for (const part of jsonPath.slice(2).split(".")) {
+    if (current === null || current === undefined || !Object.hasOwn(Object(current), part)) throw new Error(`extract path not found: ${jsonPath}`);
+    current = current[part];
+  }
+  return current;
+}
+
+function extractResult(result, extract) {
+  if (!extract) return result;
+  if (extract.jsonPath) {
+    const source = result.bodyJson !== undefined ? result.bodyJson : result;
+    return getJsonPath(source, extract.jsonPath);
+  }
+  if (extract.field) return result[extract.field];
+  return result;
+}
+
+function verifyResult(value, expect = {}, raw = value) {
+  if (expect.status !== undefined && raw?.status !== expect.status) throw new Error(`expected status ${expect.status}`);
+  if (expect.minItems !== undefined && (!Array.isArray(value) || value.length < expect.minItems)) throw new Error(`expected at least ${expect.minItems} items`);
+  if (expect.truthy && !value) throw new Error("expected a truthy result");
+  return true;
+}
+
+function withServerIdempotency(request, idempotency, attemptId) {
+  if (!idempotency || !attemptId) return request;
+  const next = { ...request };
+  if (idempotency.header) next.headers = { ...(request.headers || {}), [idempotency.header]: attemptId };
+  if (idempotency.query) next.query = { ...(request.query || {}), [idempotency.query]: attemptId };
+  if (idempotency.body) {
+    if (next.body === undefined) next.body = {};
+    if (!next.body || typeof next.body !== "object" || Array.isArray(next.body)) {
+      throw new Error("server idempotency body field requires an object request body");
+    }
+    next.body = { ...next.body, [idempotency.body]: attemptId };
+  }
+  return next;
+}
+
+function networkScript(request, allowedOrigins = []) {
+  const headers = request.headers || {};
+  const query = request.query || {};
+  const body = request.body;
+  return `(async () => {
+const url = new URL(${JSON.stringify(request.url)}, location.href);
+const allowedOrigins = ${JSON.stringify(allowedOrigins)};
+if (allowedOrigins.length > 0 && !allowedOrigins.includes(url.origin)) {
+  throw new Error(\`playbook network origin is not allowed: \${url.origin}\`);
+}
+for (const [key, value] of Object.entries(${JSON.stringify(query)})) {
+  if (value !== undefined && value !== null) url.searchParams.set(key, String(value));
+}
+const response = await fetch(url.toString(), {
+  method: ${JSON.stringify(request.method || "GET")},
+  credentials: "include",
+  headers: ${JSON.stringify(headers)},
+  ${body === undefined ? "" : `body: ${JSON.stringify(typeof body === "string" ? body : JSON.stringify(body))},`}
+});
+const text = await response.text();
+let json;
+try { json = JSON.parse(text); } catch {}
+return { status: response.status, ok: response.ok, url: response.url, headers: Object.fromEntries(response.headers.entries()), body: text, bodyJson: json };
+})()`;
+}
+
+async function runStrategy(strategy, context) {
+  if (strategy.using === "workflow") {
+    const result = await executeWorkflow(applyTemplate(strategy.steps, context.args), {
+      autoWait: strategy.autoWait !== false,
+      executeTool: async (tool, args, options) => {
+        await context.markDispatched?.();
+        return context.executeTool(tool, args, options);
+      },
+      onError: strategy.onError || "stop",
+      onEvent: context.onEvent,
+      signal: context.signal,
+      sleep: context.sleep,
+      stepDelay: strategy.stepDelay ?? 100,
+      vars: context.args,
+    });
+    if (result.status !== "completed") throw new Error(result.error || "workflow strategy failed");
+    return result;
+  }
+  if (strategy.using === "network") {
+    const request = withServerIdempotency(
+      applyTemplate(strategy.request, context.args),
+      context.serverIdempotency,
+      context.attemptId,
+    );
+    await context.markDispatched?.();
+    const response = await context.executeTool("javascript_tool", { code: networkScript(request, context.allowedOrigins) }, { signal: context.signal });
+    if (response?.error) throw new Error(typeof response.error === "string" ? response.error : JSON.stringify(response.error));
+    const result = response?.output !== undefined ? JSON.parse(response.output) : response?.value ?? response;
+    if (!result?.ok && (strategy.acceptStatus || []).includes(result?.status) === false) throw new Error(`network strategy returned ${result?.status || "an unknown status"}`);
+    return result;
+  }
+  if (strategy.using === "native") {
+    if (typeof context.executeNative !== "function") throw new Error("native strategy is not available");
+    return context.executeNative(strategy.handler, context.args, { signal: context.signal, markDispatched: context.markDispatched });
+  }
+  throw new Error(`unsupported strategy: ${strategy.using}`);
+}
+
+async function runPlaybookOp({ playbook, op, args: providedArgs = {}, attemptId, executeTool, executeNative, signal, sleep, onEvent = () => {}, beforeDispatch = async () => {}, afterDispatch = async () => {} }) {
+  const args = resolveArgs(op, providedArgs);
+  const attempts = [];
+  for (let index = 0; index < op.run.length; index++) {
+    const strategy = op.run[index];
+    let dispatched = false;
+    const markDispatched = async () => {
+      if (dispatched) return;
+      dispatched = true;
+      await beforeDispatch({ strategy, index, args });
+    };
+    onEvent({ type: "strategy.started", playbook: playbook.id, op: op.id, using: strategy.using, index, startedAt: new Date().toISOString() });
+    try {
+      const raw = await runStrategy(strategy, {
+        args,
+        attemptId,
+        executeTool,
+        executeNative,
+        signal,
+        sleep,
+        onEvent,
+        allowedOrigins: op.origins || playbook.origins || [],
+        markDispatched: op.effect === "write" ? markDispatched : undefined,
+        serverIdempotency: op.effect === "write" ? op.safety?.serverIdempotency : undefined,
+      });
+      const value = extractResult(raw, strategy.extract);
+      verifyResult(value, strategy.verify || strategy.expect || op.on?.success?.expect, raw);
+      if (op.effect === "write") await afterDispatch({ status: "verified", strategy, index });
+      onEvent({ type: "strategy.completed", playbook: playbook.id, op: op.id, using: strategy.using, index, endedAt: new Date().toISOString() });
+      return { status: "completed", playbook: playbook.id, op: op.id, strategy: strategy.using, value, provenance: playbook.provenance, attempts };
+    } catch (error) {
+      const message = error?.message || String(error);
+      attempts.push({ using: strategy.using, error: message });
+      onEvent({ type: "strategy.failed", playbook: playbook.id, op: op.id, using: strategy.using, index, error: message, endedAt: new Date().toISOString() });
+      if (op.effect === "write") {
+        await afterDispatch({ status: dispatched ? "indeterminate" : "not_dispatched", strategy, index, error: message });
+        throw error;
+      }
+    }
+  }
+  const detail = attempts.map((attempt) => `${attempt.using}: ${attempt.error}`).join("; ");
+  throw new Error(`All strategies failed${detail ? ` (${detail})` : ""}`);
+}
+
+module.exports = { resolveArgs, runPlaybookOp };