surf-cli 2.9.0 → 2.10.0

package/native/network-store.cjs

@@ -11,11 +11,17 @@ const fs = require("fs");
 const path = require("path");
 const crypto = require("crypto");
 const readline = require("readline");
+const {
+  appendPrivateJsonLine,
+  assertNotSymlink,
+  atomicWriteFile,
+  atomicWriteJson,
+  ensurePrivateDir,
+  privateStatePath,
+} = require("./private-state.cjs");
 
 // Configuration
-const DEFAULT_BASE = process.platform === "win32"
-  ? require("path").join(require("os").tmpdir(), "surf")
-  : "/tmp/surf";
+const DEFAULT_BASE = privateStatePath("network");
 const DEFAULT_TTL = 24 * 60 * 60 * 1000;  // 24 hours
 const DEFAULT_MAX_SIZE = 200 * 1024 * 1024; // 200MB
 const AUTO_CLEANUP_INTERVAL = 60 * 60 * 1000; // 1 hour
@@ -23,36 +29,26 @@ const AUTO_CLEANUP_INTERVAL = 60 * 60 * 1000; // 1 hour
 // Lock file for concurrent access
 let writeLock = Promise.resolve();
 
-// Runtime override for base path (set via CLI --network-path)
-let runtimeBasePath = null;
-
-/**
- * Set base path at runtime (from CLI --network-path flag)
- */
-function setBasePath(newPath) {
-  runtimeBasePath = newPath;
-}
-
 /**
  * Get base path for network storage
- * Priority: runtime override > SURF_NETWORK_PATH env var > default
+ * Priority: SURF_NETWORK_PATH env var > default
  */
-function getBasePath() {
-  return runtimeBasePath || process.env.SURF_NETWORK_PATH || DEFAULT_BASE;
+function getBasePath(basePath) {
+  return basePath || process.env.SURF_NETWORK_PATH || DEFAULT_BASE;
 }
 
 /**
  * Get path to requests.jsonl
  */
-function getRequestsPath() {
-  return path.join(getBasePath(), "requests.jsonl");
+function getRequestsPath(basePath) {
+  return path.join(getBasePath(basePath), "requests.jsonl");
 }
 
 /**
  * Get path to bodies directory
  */
-function getBodiesPath() {
-  return path.join(getBasePath(), "bodies");
+function getBodiesPath(basePath) {
+  return path.join(getBasePath(basePath), "bodies");
 }
 
 /**
@@ -65,16 +61,11 @@ function getMetaPath() {
 /**
  * Ensure all required directories exist
  */
-function ensureDirectories() {
-  const base = getBasePath();
-  const bodies = getBodiesPath();
-  
-  if (!fs.existsSync(base)) {
-    fs.mkdirSync(base, { recursive: true });
-  }
-  if (!fs.existsSync(bodies)) {
-    fs.mkdirSync(bodies, { recursive: true });
-  }
+function ensureDirectories(basePath) {
+  const base = getBasePath(basePath);
+  const bodies = getBodiesPath(basePath);
+  ensurePrivateDir(base, base);
+  ensurePrivateDir(bodies, base);
 }
 
 /**
@@ -84,6 +75,7 @@ function readMeta() {
   const metaPath = getMetaPath();
   try {
     if (fs.existsSync(metaPath)) {
+      assertNotSymlink(metaPath, false);
       return JSON.parse(fs.readFileSync(metaPath, "utf-8"));
     }
   } catch (err) {
@@ -98,7 +90,7 @@ function readMeta() {
 function writeMeta(meta) {
   const metaPath = getMetaPath();
   ensureDirectories();
-  fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2));
+  atomicWriteJson(metaPath, meta, { root: getBasePath() });
 }
 
 /**
@@ -124,7 +116,7 @@ function storeBody(content, isRequest = false) {
   
   // Only write if doesn't exist (dedup)
   if (!fs.existsSync(bodyPath)) {
-    fs.writeFileSync(bodyPath, buffer);
+    atomicWriteFile(bodyPath, buffer, { root: getBasePath() });
   }
   
   return hash;
@@ -142,6 +134,7 @@ function readBody(hash, isRequest = false) {
   
   try {
     if (fs.existsSync(bodyPath)) {
+      assertNotSymlink(bodyPath, false);
       return fs.readFileSync(bodyPath);
     }
   } catch (err) {
@@ -186,10 +179,7 @@ async function appendEntry(entry) {
       ...entry
     };
     
-    const line = JSON.stringify(fullEntry) + "\n";
-    
-    // Atomic append using flag 'a'
-    fs.appendFileSync(getRequestsPath(), line, { flag: "a" });
+    appendPrivateJsonLine(getRequestsPath(), fullEntry, { root: getBasePath() });
     
     return fullEntry;
   } finally {
@@ -202,8 +192,8 @@ async function appendEntry(entry) {
  * @param {Object} entry - Network entry to append
  * @returns {Object} The entry with assigned ID
  */
-function appendEntrySync(entry) {
-  ensureDirectories();
+function appendEntrySync(entry, basePath) {
+  ensureDirectories(basePath);
   
   const id = entry.id || generateId();
   const timestamp = entry.timestamp || Date.now();
@@ -214,15 +204,14 @@ function appendEntrySync(entry) {
     ...entry
   };
   
-  const line = JSON.stringify(fullEntry) + "\n";
-  
   // Use a simple lock file for synchronous operations
-  const lockPath = path.join(getBasePath(), ".lock");
+  const lockPath = path.join(getBasePath(basePath), ".lock");
   let lockFd;
   
   try {
     // Try to acquire lock
-    lockFd = fs.openSync(lockPath, "wx");
+    assertNotSymlink(lockPath, true);
+    lockFd = fs.openSync(lockPath, "wx", 0o600);
   } catch (err) {
     // Lock exists - check if stale and remove, otherwise proceed without lock
     try {
@@ -230,7 +219,7 @@ function appendEntrySync(entry) {
       if (Date.now() - stat.mtimeMs > 5000) {
         fs.unlinkSync(lockPath);
         try {
-          lockFd = fs.openSync(lockPath, "wx");
+          lockFd = fs.openSync(lockPath, "wx", 0o600);
         } catch (e) {
           // Still can't get lock, proceed without it
         }
@@ -241,13 +230,13 @@ function appendEntrySync(entry) {
     
     if (lockFd === undefined) {
       // Proceed without lock as fallback
-      fs.appendFileSync(getRequestsPath(), line, { flag: "a" });
+      appendPrivateJsonLine(getRequestsPath(basePath), fullEntry, { root: getBasePath(basePath) });
       return fullEntry;
     }
   }
   
   try {
-    fs.appendFileSync(getRequestsPath(), line, { flag: "a" });
+    appendPrivateJsonLine(getRequestsPath(basePath), fullEntry, { root: getBasePath(basePath) });
   } finally {
     if (lockFd !== undefined) {
       fs.closeSync(lockFd);
@@ -385,6 +374,7 @@ async function readEntries(filters = {}) {
   if (!fs.existsSync(requestsPath)) {
     return [];
   }
+  assertNotSymlink(requestsPath, false);
   
   const { last } = filters;
   const entries = [];
@@ -433,6 +423,7 @@ function readEntriesSync(filters = {}) {
   if (!fs.existsSync(requestsPath)) {
     return [];
   }
+  assertNotSymlink(requestsPath, false);
   
   const { last } = filters;
   const entries = [];
@@ -682,10 +673,8 @@ async function cleanup(options = {}) {
   
   if (deletedEntries > 0 || entries.length === 0) {
     // Atomic write: write to temp then rename
-    const tempPath = requestsPath + ".tmp";
     const content = entries.map(e => JSON.stringify(e)).join("\n") + (entries.length > 0 ? "\n" : "");
-    fs.writeFileSync(tempPath, content);
-    fs.renameSync(tempPath, requestsPath);
+    atomicWriteFile(requestsPath, content, { root: getBasePath(), encoding: "utf8" });
   }
   
   // 6. Update meta
@@ -777,10 +766,8 @@ async function clear(options = {}) {
   
   // Rewrite entries file
   if (deletedEntries > 0) {
-    const tempPath = requestsPath + ".tmp";
     const content = remaining.map(e => JSON.stringify(e)).join("\n") + (remaining.length > 0 ? "\n" : "");
-    fs.writeFileSync(tempPath, content);
-    fs.renameSync(tempPath, requestsPath);
+    atomicWriteFile(requestsPath, content, { root: getBasePath(), encoding: "utf8" });
   }
   
   return { deletedEntries, deletedBodies };
@@ -807,9 +794,6 @@ function maybeAutoCleanup() {
   }
 }
 
-// Run auto-cleanup check on module load
-maybeAutoCleanup();
-
 module.exports = {
   // Configuration
   getBasePath,
@@ -841,10 +825,6 @@ module.exports = {
   clear,
   maybeAutoCleanup,
   
-  // Configuration
-  setBasePath,
-  getBasePath,
-  
   // Constants
   DEFAULT_BASE,
   DEFAULT_TTL,

package/native/playbook-authoring.cjs

@@ -0,0 +1,44 @@
+const fs = require("fs");
+const path = require("path");
+const { readRecent } = require("./activity-journal.cjs");
+const { writeNetworkExport } = require("./network-export.cjs");
+const { getPrivateStateRoot, readPrivateJson } = require("./private-state.cjs");
+const { draftFromRecord, readRecord, recordsRoot } = require("./playbook-records.cjs");
+const { savePlaybook, validateOp } = require("./playbooks.cjs");
+const { commandMetadata, promoteRedactedStepArgs } = require("./workflow-definition.cjs");
+
+function suggestions({ since = "1h", root } = {}) {
+  const events = readRecent({ since, root });
+  const counts = new Map();
+  for (const event of events.filter((entry) => entry.type === "tool.issued")) counts.set(event.command, (counts.get(event.command) || 0) + 1);
+  return [...counts.entries()].map(([command, count]) => ({ command, count })).sort((a, b) => b.count - a.count || a.command.localeCompare(b.command));
+}
+
+function saveFromRecent({ site, op: opId, since = "1h", scope = "user", cwd, home, root }) {
+  const events = readRecent({ since, root }).filter((event) => event.type === "tool.issued");
+  if (events.length === 0) throw new Error("no recent Surf activity to save");
+  if (events.some((event) => ["page-write", "unknown"].includes(commandMetadata(event.command).effect))) {
+    throw new Error("recent activity includes write-capable commands; use an explicit record and review its draft before saving");
+  }
+  const promoted = promoteRedactedStepArgs(events.map((event) => ({ tool: event.command, args: event.argsRedacted || {} })));
+  const op = { id: opId, description: "Drafted from recent Surf activity", effect: "read", args: promoted.args, run: [{ using: "workflow", steps: promoted.steps }], provenance: { recentSince: since } };
+  validateOp(op, { origins: [] });
+  return savePlaybook({ manifest: { id: site, name: site, version: "1.0.0", origins: [] }, op, scope, cwd, home });
+}
+
+function saveFromRecord({ recordId, scope = "user", cwd, home, root = getPrivateStateRoot() }) {
+  const record = readRecord(recordId, root);
+  if (!record) throw new Error(`record not found: ${recordId}`);
+  const draftPath = path.join(recordsRoot(root), recordId, "draft", "op.json");
+  const op = fs.existsSync(draftPath) ? readPrivateJson(draftPath, null, { root }) : draftFromRecord(recordId, root);
+  validateOp(op, { origins: record.origin ? [record.origin] : [] });
+  return savePlaybook({ manifest: { id: record.site, name: record.site, version: "1.0.0", origins: record.origin ? [record.origin] : [] }, op, scope, cwd, home });
+}
+
+function exportRecordHar(recordId, output, root = getPrivateStateRoot()) {
+  const trace = readPrivateJson(path.join(recordsRoot(root), recordId, "network", "trace.json"), null, { root });
+  if (!trace) throw new Error(`record ${recordId} has no network trace`);
+  return writeNetworkExport(path.resolve(output), trace.entries, "har");
+}
+
+module.exports = { exportRecordHar, saveFromRecent, saveFromRecord, suggestions };

package/native/playbook-cli.cjs

@@ -0,0 +1,157 @@
+const path = require("path");
+const { openClientTransport } = require("./client-transport.cjs");
+const { resolveRequestDeadlineMs } = require("./host-sessions.cjs");
+const { exportRecordHar, saveFromRecent, saveFromRecord, suggestions } = require("./playbook-authoring.cjs");
+const { deriveClient, exportClient, verifyClient } = require("./playbook-client.cjs");
+const { exportPlaybookDirectory, importPlaybookDirectory, listPlaybooks, resolvePlaybook } = require("./playbooks.cjs");
+
+function parseCommandArgs(argv) {
+  const positional = [];
+  const options = {};
+  for (let index = 0; index < argv.length; index++) {
+    const value = argv[index];
+    if (!value.startsWith("--")) {
+      positional.push(value);
+      continue;
+    }
+    const name = value.slice(2);
+    const next = argv[index + 1];
+    if (next !== undefined && !next.startsWith("--")) {
+      options[name] = /^-?\d+(?:\.\d+)?$/.test(next) ? Number(next) : next;
+      index++;
+    } else options[name] = true;
+  }
+  return { positional, options };
+}
+
+function unwrapResponse(response) {
+  if (response.error) throw new Error(response.error.content?.[0]?.text || JSON.stringify(response.error));
+  const text = response.result?.content?.[0]?.text;
+  if (text === undefined) return response.result;
+  try { return JSON.parse(text); } catch { return text; }
+}
+
+async function requestHost(endpoint, tool, args, options = {}) {
+  const transport = await openClientTransport(endpoint, { requestTimeoutMs: options.timeoutMs || 11 * 60 * 1000 });
+  try {
+    const request = { type: "tool_request", method: "execute_tool", params: { tool, args }, id: `playbook-${Date.now()}-${Math.random()}` };
+    if (options.tabId) request.tabId = options.tabId;
+    return unwrapResponse(await transport.request(request, options.timeoutMs || 11 * 60 * 1000));
+  } finally {
+    await transport.close();
+  }
+}
+
+function runSpec(argv) {
+  const direct = argv[0] === "use";
+  const offset = direct ? 1 : 2;
+  const parsed = parseCommandArgs(argv.slice(offset));
+  const [playbook, op] = parsed.positional;
+  if (!playbook || !op) throw new Error(direct ? "Usage: surf use <playbook> <op> [--arg value]" : "Usage: surf pb run <playbook> <op> [--arg value]");
+  const reserved = new Set(["json", "no-lock", "tab-id", "write", "repeat", "retry-attempt", "override-in-doubt", "pin-built-in"]);
+  const args = Object.fromEntries(Object.entries(parsed.options).filter(([name]) => !reserved.has(name)));
+  return { playbook, op, args, options: parsed.options };
+}
+
+function resolveRunTimeout(spec, cwd) {
+  const explicit = Number(spec.args.timeout);
+  if (Number.isFinite(explicit) && explicit > 0) return explicit;
+  try {
+    const playbook = resolvePlaybook(spec.playbook, { cwd, pinBuiltIn: spec.options["pin-built-in"] === true });
+    const op = playbook.ops.get(spec.op);
+    const value = Number(op?.args?.timeout?.default);
+    return Number.isFinite(value) && value > 0 ? value : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function playbookCommandNeedsBrowser(argv) {
+  if (argv[0] === "use") return true;
+  const subcommand = argv[1];
+  if (subcommand === "run") return true;
+  return subcommand === "record" && ["start", "stop", "discard"].includes(argv[2]);
+}
+
+async function handlePlaybookCli(argv, { endpoint, cwd = process.cwd() }) {
+  if (!["playbook", "pb", "use"].includes(argv[0])) return { handled: false };
+  if (argv[0] === "use" || argv[1] === "run") {
+    const spec = runSpec(argv);
+    const timeout = resolveRunTimeout(spec, cwd);
+    const args = {
+      playbook: spec.playbook,
+      op: spec.op,
+      args: spec.args,
+      projectDir: cwd,
+      ...(timeout ? { timeout } : {}),
+      write: spec.options.write === true,
+      repeat: spec.options.repeat === true,
+      retryAttempt: spec.options["retry-attempt"],
+      overrideInDoubt: spec.options["override-in-doubt"] === true,
+      pinBuiltIn: spec.options["pin-built-in"] === true,
+    };
+    const value = await requestHost(endpoint, "playbook.run", args, {
+      tabId: spec.options["tab-id"],
+      timeoutMs: resolveRequestDeadlineMs("playbook.run", args),
+    });
+    return { handled: true, value, json: spec.options.json === true };
+  }
+  const command = argv[1];
+  const parsed = parseCommandArgs(argv.slice(2));
+  if (!command || command === "help") return { handled: true, value: "Usage: surf playbook|pb <list|show|ops|run|record|suggest|save|client|trace|export|import>" };
+  if (endpoint?.kind === "remote" && ["list", "show", "ops"].includes(command)) throw new Error(`playbook ${command} is local-only with --remote because runs resolve on the browser host`);
+  if (command === "list") return { handled: true, value: listPlaybooks({ cwd }), json: parsed.options.json === true };
+  if (command === "show") {
+    const playbook = resolvePlaybook(parsed.positional[0], { cwd });
+    return { handled: true, value: { id: playbook.id, name: playbook.name, version: playbook.version, description: playbook.description, origins: playbook.origins, provenance: playbook.provenance, ops: [...playbook.ops.keys()] }, json: parsed.options.json === true };
+  }
+  if (command === "ops") {
+    const playbook = resolvePlaybook(parsed.positional[0], { cwd });
+    return { handled: true, value: [...playbook.ops.values()].map((op) => ({ id: op.id, description: op.description || "", effect: op.effect, strategies: op.run.map((strategy) => strategy.using) })), json: parsed.options.json === true };
+  }
+  if (command === "record") {
+    const action = parsed.positional[0];
+    const tool = `playbook.record.${action}`;
+    let args = {};
+    if (action === "start") args = { site: parsed.positional[1], op: parsed.options.op, watch: parsed.options.watch === true, network: parsed.options.network === true, includeInputValues: parsed.options["include-input-values"] === true };
+    else if (action === "mark") args = { label: parsed.positional.slice(1).join(" ") };
+    else if (action === "stop") args = { draft: parsed.options.draft === true };
+    else if (!["status", "pause", "resume", "discard"].includes(action)) throw new Error("Unknown record command");
+    const value = await requestHost(endpoint, tool, args, { tabId: parsed.options["tab-id"] });
+    return { handled: true, value, json: parsed.options.json === true };
+  }
+  if (command === "suggest") return { handled: true, value: suggestions({ since: parsed.options.since || "1h" }), json: parsed.options.json === true };
+  if (command === "save") {
+    let value;
+    if (parsed.options["from-record"]) value = saveFromRecord({ recordId: parsed.options["from-record"], scope: parsed.options.project ? "project" : "user", cwd });
+    else if (parsed.options["from-recent"] || parsed.positional[0]) value = saveFromRecent({ site: parsed.positional[0], op: parsed.options.op, since: parsed.options["from-recent"] === true ? "1h" : parsed.options["from-recent"] || "1h", scope: parsed.options.project ? "project" : "user", cwd });
+    else throw new Error("save requires --from-record <id> or <site> --op <name> --from-recent");
+    return { handled: true, value, json: parsed.options.json === true };
+  }
+  if (command === "client") {
+    const action = parsed.positional[0];
+    if (action === "derive") return { handled: true, value: deriveClient(parsed.positional[1], parsed.options.op, parsed.options.out, { recordId: parsed.options["from-record"], requestId: parsed.options["request-id"] }), json: parsed.options.json === true };
+    if (action === "export") {
+      const playbook = parsed.positional[1];
+      const resolved = resolvePlaybook(playbook, { cwd });
+      const op = parsed.options.op || [...resolved.ops.keys()][0];
+      return { handled: true, value: exportClient(playbook, op, parsed.options.out, { cwd }), json: parsed.options.json === true };
+    }
+    if (action === "verify") return { handled: true, value: await verifyClient(parsed.positional[1], { live: parsed.options.live === true ? true : undefined }), json: parsed.options.json === true };
+    throw new Error("Unknown client command");
+  }
+  if (command === "trace" && parsed.positional[0] === "export") {
+    if (!parsed.options["from-record"] || !parsed.options.har) throw new Error("trace export requires --from-record <id> --har <path>");
+    return { handled: true, value: exportRecordHar(parsed.options["from-record"], path.resolve(parsed.options.har)), json: parsed.options.json === true };
+  }
+  if (command === "export") return { handled: true, value: exportPlaybookDirectory(parsed.positional[0], { out: parsed.options.out, cwd }), json: parsed.options.json === true };
+  if (command === "import") return { handled: true, value: importPlaybookDirectory(parsed.positional[0], { scope: parsed.options.project ? "project" : "user", cwd }), json: parsed.options.json === true };
+  throw new Error(`Unknown playbook command: ${command}`);
+}
+
+function formatPlaybookOutput(value, json = false) {
+  if (json || typeof value !== "string") return JSON.stringify(value, null, 2);
+  return value;
+}
+
+module.exports = { formatPlaybookOutput, handlePlaybookCli, playbookCommandNeedsBrowser };