surf-cli 2.9.0 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +61 -4
  2. package/dist/content/accessibility-tree.js +11 -0
  3. package/dist/content/accessibility-tree.js.map +1 -0
  4. package/dist/content/visual-indicator.js +111 -0
  5. package/dist/content/visual-indicator.js.map +1 -0
  6. package/dist/manifest.json +11 -2
  7. package/dist/options/options.js +3 -3
  8. package/dist/options/options.js.map +1 -1
  9. package/dist/service-worker/index.js +61 -261
  10. package/dist/service-worker/index.js.map +1 -1
  11. package/native/activity-journal.cjs +55 -0
  12. package/native/chatgpt-client-response.cjs +336 -0
  13. package/native/chatgpt-client-selection.cjs +119 -0
  14. package/native/chatgpt-client-ui.cjs +481 -0
  15. package/native/chatgpt-client.cjs +254 -664
  16. package/native/cli.cjs +100 -273
  17. package/native/do-executor.cjs +52 -475
  18. package/native/do-parser.cjs +8 -249
  19. package/native/host-helpers.cjs +32 -15
  20. package/native/host-sessions.cjs +6 -1
  21. package/native/host.cjs +228 -6
  22. package/native/network-export.cjs +20 -17
  23. package/native/network-store.cjs +38 -58
  24. package/native/oracle-cli.cjs +434 -0
  25. package/native/oracle-context.cjs +311 -0
  26. package/native/oracle-host.cjs +301 -0
  27. package/native/oracle-jobs.cjs +253 -0
  28. package/native/playbook-authoring.cjs +44 -0
  29. package/native/playbook-cli.cjs +157 -0
  30. package/native/playbook-client.cjs +259 -0
  31. package/native/playbook-receipts.cjs +109 -0
  32. package/native/playbook-records.cjs +208 -0
  33. package/native/playbook-runtime.cjs +177 -0
  34. package/native/playbooks.cjs +235 -0
  35. package/native/private-state.cjs +156 -0
  36. package/native/redaction.cjs +104 -0
  37. package/native/workflow-definition.cjs +369 -0
  38. package/native/workflow-runtime.cjs +225 -0
  39. package/package.json +2 -1
  40. package/playbooks/page/ops/read.json +22 -0
  41. package/playbooks/page/playbook.json +7 -0
  42. package/skills/surf/SKILL.md +72 -1
  43. package/dist/content/index.js +0 -116
  44. package/dist/content/index.js.map +0 -1
package/native/host.cjs CHANGED
@@ -14,7 +14,8 @@ const perplexityClient = require("./perplexity-client.cjs");
14
14
  const grokClient = require("./grok-client.cjs");
15
15
  const aistudioClient = require("./aistudio-client.cjs");
16
16
  const aistudioBuild = require("./aistudio-build.cjs");
17
- const { mapToolToMessage, mapComputerAction, formatToolContent, buildProviderUploadMessage } = require("./host-helpers.cjs");
17
+ const { mapToolToMessage, mapComputerAction, formatToolContent, formatToolError, buildProviderUploadMessage } = require("./host-helpers.cjs");
18
+ const { createOracleHost } = require("./oracle-host.cjs");
18
19
 
19
20
  const IS_WIN = process.platform === "win32";
20
21
  const { SOCKET_PATH, SURF_TMP } = require("./socket-path.cjs");
@@ -22,11 +23,30 @@ const { parseListenEndpoint } = require("./listener.cjs");
22
23
  const { getStateDir } = require("./remote-auth.cjs");
23
24
  const { createFrameParser, createServerAuthSession, createSocketWriter, isClientAuthorized, writeFrame, MAX_FRAME_BYTES } = require("./remote-transport.cjs");
24
25
  const { HostSessionManager, resolveRequestDeadlineMs } = require("./host-sessions.cjs");
25
- const { abortError, throwIfAborted } = require("./abort.cjs");
26
+ const { abortError, abortableDelay, throwIfAborted } = require("./abort.cjs");
26
27
  const { BoundedAiQueue } = require("./ai-queue.cjs");
27
28
  const { RequestPendingMap } = require("./request-pending.cjs");
28
29
  const { cleanupFilePaths, createStagingDirectory, createTransferState, materializeRemoteTool, rewriteTransferPaths, streamFileDownload, transferError } = require("./file-transfer.cjs");
29
30
  const { writeNetworkExport } = require("./network-export.cjs");
31
+ const networkStore = require("./network-store.cjs");
32
+ const { redactUrlSecrets } = require("./redaction.cjs");
33
+ const { appendActivity, journalCommand } = require("./activity-journal.cjs");
34
+ const { reserveReceipt, updateReceipt } = require("./playbook-receipts.cjs");
35
+ const {
36
+ activeRecord,
37
+ appendRecordEvent,
38
+ attachNetworkTrace,
39
+ discardRecord,
40
+ markRecord,
41
+ pauseRecord,
42
+ resumeRecord,
43
+ startRecord,
44
+ stopRecord,
45
+ updateRecordContext,
46
+ } = require("./playbook-records.cjs");
47
+ const { resolveArgs, runPlaybookOp } = require("./playbook-runtime.cjs");
48
+ const { resolveOp } = require("./playbooks.cjs");
49
+ const { commandMetadata, redactCommandArgs } = require("./workflow-definition.cjs");
30
50
  const MAX_CLIENT_FRAME_BYTES = MAX_FRAME_BYTES;
31
51
  const TEST_REQUEST_DEADLINE_MS = process.env.SURF_TEST_MODE === "1" && Number.isFinite(Number(process.env.SURF_TEST_REQUEST_DEADLINE_MS))
32
52
  ? Number(process.env.SURF_TEST_REQUEST_DEADLINE_MS)
@@ -383,6 +403,15 @@ aiQueue = new BoundedAiQueue({
383
403
  : handler(),
384
404
  });
385
405
 
406
+ const oracleHost = createOracleHost({
407
+ queueAiRequest,
408
+ requestCallExtension,
409
+ buildProviderUploadMessage,
410
+ log,
411
+ });
412
+ const adoptedOracleJobs = oracleHost.adoptOrphans();
413
+ log(`Oracle adoption: ${adoptedOracleJobs.length} job(s); ids=${adoptedOracleJobs.map((job) => job.id).join(",") || "none"}`);
414
+
386
415
  function sendSocket(socket, value, options = {}) {
387
416
  const writer = socketWriters.get(socket);
388
417
  return writer ? writer.send(value, options) : writeFrame(socket, value);
@@ -428,6 +457,131 @@ function requestCallExtension(request, tool, message, timeoutMs = 30000, cleanup
428
457
  });
429
458
  }
430
459
 
460
+ async function executeMappedHostTool(request, tool, args, tabId) {
461
+ const extensionMsg = mapToolToMessage(tool, args, tabId);
462
+ if (!extensionMsg) throw new Error(`Unknown tool: ${tool}`);
463
+ if (extensionMsg.type === "UNSUPPORTED_ACTION") throw new Error(extensionMsg.message);
464
+ if (extensionMsg.type === "LOCAL_WAIT") {
465
+ await abortableDelay(extensionMsg.seconds * 1000, request.signal);
466
+ return { success: true };
467
+ }
468
+ if (extensionMsg.type === "BATCH_EXECUTE" || extensionMsg.type.endsWith("_QUERY")) {
469
+ throw new Error(`tool ${tool} is not available inside a host-owned workflow`);
470
+ }
471
+ return requestCallExtension(request, tool, extensionMsg, resolveRequestDeadlineMs(tool, args));
472
+ }
473
+
474
+ async function executeNativePlaybook(request, handler, args, options = {}) {
475
+ if (handler !== "chatgpt.ask") throw new Error(`unknown native playbook handler: ${handler}`);
476
+ const result = await chatgptClient.query({
477
+ prompt: args.prompt,
478
+ signal: request.signal,
479
+ model: args.model,
480
+ timeout: args.timeout ? Number(args.timeout) * 1000 : undefined,
481
+ getCookies: () => requestCallExtension(request, "get_cookies", { type: "GET_CHATGPT_COOKIES" }),
482
+ createTab: () => requestCallExtension(request, "create_tab", { type: "CHATGPT_NEW_TAB" }),
483
+ closeTab: (tabId) => requestCallExtension(request, "close_tab", { type: "CHATGPT_CLOSE_TAB", tabId }, 45000, true),
484
+ cdpEvaluate: (tabId, expression) => requestCallExtension(request, "cdp_evaluate", { type: "CHATGPT_EVALUATE", tabId, expression }),
485
+ cdpCommand: (tabId, method, params) => requestCallExtension(request, "cdp_command", { type: "CHATGPT_CDP_COMMAND", tabId, method, params }),
486
+ beforeSubmit: options.markDispatched,
487
+ log: (message) => log(`[playbook:chatgpt] ${message}`),
488
+ });
489
+ return { response: result.response, model: result.model, tookMs: result.tookMs };
490
+ }
491
+
492
+ async function runHostPlaybook(msg, request) {
493
+ const params = msg.params?.args || {};
494
+ const { playbook, op } = resolveOp(params.playbook, params.op, {
495
+ cwd: request.context?.isRemote ? process.cwd() : params.projectDir || process.cwd(),
496
+ pinBuiltIn: params.pinBuiltIn === true,
497
+ });
498
+ const runArgs = resolveArgs(op, params.args || {});
499
+ if (op.effect === "write" && op.safety.authorization === "explicit" && params.write !== true) {
500
+ throw new Error(`write op ${playbook.id} ${op.id} requires --write`);
501
+ }
502
+ const receipt = reserveReceipt({
503
+ playbookId: playbook.id,
504
+ op,
505
+ args: runArgs,
506
+ repeat: params.repeat === true,
507
+ retryAttempt: params.retryAttempt,
508
+ overrideInDoubt: params.overrideInDoubt === true,
509
+ });
510
+ const report = (event) => {
511
+ appendActivity(event);
512
+ appendRecordEvent(event);
513
+ };
514
+ return runPlaybookOp({
515
+ playbook,
516
+ op,
517
+ args: runArgs,
518
+ attemptId: receipt?.attemptId,
519
+ signal: request.signal,
520
+ executeTool: (tool, args) => executeMappedHostTool(request, tool, args, msg.tabId),
521
+ executeNative: (handler, args, options) => executeNativePlaybook(request, handler, args, options),
522
+ sleep: (ms) => abortableDelay(ms, request.signal),
523
+ onEvent: report,
524
+ beforeDispatch: async () => updateReceipt(receipt, "dispatched"),
525
+ afterDispatch: async ({ status, error }) => updateReceipt(receipt, status, { error }),
526
+ });
527
+ }
528
+
529
+ async function handleRecordRequest(tool, args, msg, request) {
530
+ if (tool === "playbook.record.start") {
531
+ let record = startRecord({ ...args, tabId: msg.tabId });
532
+ try {
533
+ const context = await requestCallExtension(request, tool, {
534
+ type: "GET_PLAYBOOK_RECORD_CONTEXT",
535
+ tabId: msg.tabId,
536
+ });
537
+ record = updateRecordContext({
538
+ tabId: context._resolvedTabId || msg.tabId,
539
+ origin: context.origin,
540
+ });
541
+ if (record.capture.network) {
542
+ const result = await requestCallExtension(request, tool, { type: "START_NETWORK_CAPTURE", tabId: record.tabId, bodyMode: "text" });
543
+ record = updateRecordContext({ tabId: result._resolvedTabId || record.tabId });
544
+ }
545
+ if (record.capture.watch) {
546
+ const result = await requestCallExtension(request, tool, { type: "START_PLAYBOOK_WATCH", tabId: record.tabId || msg.tabId, includeInputValues: record.redaction.includeInputValues });
547
+ record = updateRecordContext({ tabId: result._resolvedTabId || record.tabId || msg.tabId });
548
+ }
549
+ return record;
550
+ } catch (error) {
551
+ if (record?.capture.network) await requestCallExtension(request, tool, { type: "STOP_NETWORK_CAPTURE", tabId: record.tabId }, 30000, true).catch(() => {});
552
+ if (record?.capture.watch) await requestCallExtension(request, tool, { type: "STOP_PLAYBOOK_WATCH", tabId: record.tabId }, 30000, true).catch(() => {});
553
+ discardRecord();
554
+ throw error;
555
+ }
556
+ }
557
+ if (tool === "playbook.record.status") return activeRecord() || { status: "idle" };
558
+ if (tool === "playbook.record.mark") return markRecord(args.label);
559
+ if (tool === "playbook.record.pause") return pauseRecord();
560
+ if (tool === "playbook.record.resume") return resumeRecord();
561
+ if (tool === "playbook.record.discard") {
562
+ const record = activeRecord();
563
+ if (record?.capture.network) await requestCallExtension(request, tool, { type: "STOP_NETWORK_CAPTURE", tabId: record.tabId }, 30000, true).catch(() => {});
564
+ if (record?.capture.watch) await requestCallExtension(request, tool, { type: "STOP_PLAYBOOK_WATCH", tabId: record.tabId }, 30000, true).catch(() => {});
565
+ return discardRecord();
566
+ }
567
+ if (tool === "playbook.record.stop") {
568
+ const record = activeRecord();
569
+ if (!record) throw new Error("no active playbook record");
570
+ if (record.capture.network) {
571
+ try {
572
+ const result = await requestCallExtension(request, tool, { type: "READ_NETWORK_REQUESTS", tabId: record.tabId, full: true, limit: 500 });
573
+ const cutoff = Date.parse(record.startedAt);
574
+ attachNetworkTrace(record.id, (result.entries || []).filter((entry) => entry.ts >= cutoff));
575
+ } finally {
576
+ await requestCallExtension(request, tool, { type: "STOP_NETWORK_CAPTURE", tabId: record.tabId }, 30000, true).catch(() => {});
577
+ }
578
+ }
579
+ if (record.capture.watch) await requestCallExtension(request, tool, { type: "STOP_PLAYBOOK_WATCH", tabId: record.tabId }, 30000, true).catch(() => {});
580
+ return stopRecord({ draft: args.draft === true });
581
+ }
582
+ throw new Error(`Unknown record command: ${tool}`);
583
+ }
584
+
431
585
  const sessionManager = new HostSessionManager({
432
586
  audit: auditSession,
433
587
  onTimeout(context, request) {
@@ -528,8 +682,29 @@ function sendToolResponse(socket, id, result, error) {
528
682
  } catch (transferFailure) {
529
683
  finalError = transferFailure.message;
530
684
  }
531
- if (finalError && request) {
532
- finalError = rewriteTransferPaths(finalError, request.pathRewrites || []);
685
+ const formattedError = finalError ? formatToolError(finalError) : null;
686
+ if (formattedError && request) {
687
+ const rewrittenMessage = rewriteTransferPaths(
688
+ formattedError.content[0].text,
689
+ request.pathRewrites || [],
690
+ );
691
+ formattedError.content[0].text = rewrittenMessage;
692
+ if (formattedError.message) formattedError.message = rewrittenMessage;
693
+ }
694
+ if (request?.tool && !request.tool.startsWith("playbook.")) {
695
+ const metadata = commandMetadata(request.tool);
696
+ if (metadata.recordable) {
697
+ const event = {
698
+ type: finalError ? "tool.failed" : "tool.completed",
699
+ command: metadata.name,
700
+ argsRedacted: redactCommandArgs(request.tool, request.args || {}),
701
+ startedAt: request.activityStartedAt || new Date().toISOString(),
702
+ endedAt: new Date().toISOString(),
703
+ resultSummary: finalError ? "failed" : "success",
704
+ };
705
+ appendActivity(event);
706
+ appendRecordEvent(event);
707
+ }
533
708
  }
534
709
  await cleanupRequestTransfers(request);
535
710
  if (request?.settled) return;
@@ -538,7 +713,7 @@ function sendToolResponse(socket, id, result, error) {
538
713
  : finalError ? "error" : "completed";
539
714
  await completeOwnedRequest(context, id, outcome);
540
715
  const response = { type: "tool_response", id };
541
- if (finalError) response.error = { content: [{ type: "text", text: finalError }] };
716
+ if (formattedError) response.error = formattedError;
542
717
  else response.result = { content: formatToolContent(output, log, { suppressImages: Boolean(context?.isRemote) }) };
543
718
  if (!context?.closed) await sendSocket(socket, response);
544
719
  })().catch((sendError) => log(`Error sending tool_response: ${sendError.message}`));
@@ -607,6 +782,22 @@ function handleToolRequest(msg, socket, requestContext = requestStorage.getStore
607
782
  sendToolResponse(socket, originalId, null, "No tool specified");
608
783
  return;
609
784
  }
785
+
786
+ requestContext.args = args || {};
787
+ requestContext.activityStartedAt = new Date().toISOString();
788
+ if (!tool.startsWith("playbook.")) journalCommand(tool, args || {}, { tabId });
789
+ if (tool === "playbook.run") {
790
+ runHostPlaybook(msg, requestContext)
791
+ .then((result) => sendToolResponse(socket, originalId, { output: JSON.stringify(result) }, null))
792
+ .catch((error) => sendToolResponse(socket, originalId, null, error.message));
793
+ return;
794
+ }
795
+ if (tool.startsWith("playbook.record.")) {
796
+ handleRecordRequest(tool, args || {}, msg, requestContext)
797
+ .then((result) => sendToolResponse(socket, originalId, result, null))
798
+ .catch((error) => sendToolResponse(socket, originalId, null, error.message));
799
+ return;
800
+ }
610
801
 
611
802
  const extensionMsg = mapToolToMessage(tool, args, tabId);
612
803
  if (!extensionMsg) {
@@ -625,6 +816,13 @@ function handleToolRequest(msg, socket, requestContext = requestStorage.getStore
625
816
  .catch((error) => sendToolResponse(socket, originalId, null, error.message));
626
817
  return;
627
818
  }
819
+
820
+ if (extensionMsg.type.startsWith("ORACLE_")) {
821
+ Promise.resolve(oracleHost.handle(requestContext, extensionMsg))
822
+ .then((result) => sendToolResponse(socket, originalId, result, null))
823
+ .catch((error) => sendToolResponse(socket, originalId, null, error));
824
+ return;
825
+ }
628
826
 
629
827
  if (extensionMsg.type === "BATCH_EXECUTE") {
630
828
  executeBatch(extensionMsg.actions, extensionMsg.tabId, socket, originalId, requestContext);
@@ -1264,7 +1462,9 @@ function handleToolRequest(msg, socket, requestContext = requestStorage.getStore
1264
1462
  autoScreenshot: args?.autoScreenshot === true,
1265
1463
  autoScreenshotOutput: args?.autoScreenshotOutput,
1266
1464
  networkExport: extensionMsg.type === "EXPORT_NETWORK_REQUESTS",
1465
+ persistNetwork: extensionMsg.type === "READ_NETWORK_REQUESTS" && extensionMsg.full && args?.["no-save"] !== true,
1267
1466
  networkExportPath: args?.output,
1467
+ networkPath: args?.["network-path"],
1268
1468
  networkExportFormat: extensionMsg.har ? "har" : extensionMsg.jsonl ? "jsonl" : "json",
1269
1469
  fullRes: extensionMsg.fullRes || args?.fullRes,
1270
1470
  maxSize: extensionMsg.maxSize || args?.maxSize,
@@ -1431,6 +1631,19 @@ function processInput() {
1431
1631
  handleApiRequest(msg, writeMessage);
1432
1632
  return;
1433
1633
  }
1634
+
1635
+ if (msg.type === "PLAYBOOK_WATCH_EVENT") {
1636
+ appendRecordEvent({
1637
+ type: "browser.event",
1638
+ event: msg.event,
1639
+ selector: msg.selector,
1640
+ value: msg.value,
1641
+ url: redactUrlSecrets(msg.url),
1642
+ tabId: msg.tabId,
1643
+ timestamp: msg.timestamp || new Date().toISOString(),
1644
+ });
1645
+ return;
1646
+ }
1434
1647
 
1435
1648
  if (msg.type === "STREAM_EVENT") {
1436
1649
  const stream = activeStreams.get(msg.streamId);
@@ -1489,6 +1702,14 @@ function processInput() {
1489
1702
  } catch (error) {
1490
1703
  sendToolResponse(socket, originalId, null, `Failed to export network requests: ${error.message}`);
1491
1704
  }
1705
+ } else if (pending.persistNetwork && Array.isArray(msg.entries)) {
1706
+ try {
1707
+ for (const entry of msg.entries) networkStore.appendEntrySync(entry, pending.networkPath);
1708
+ networkStore.maybeAutoCleanup();
1709
+ sendToolResponse(socket, originalId, msg, null);
1710
+ } catch (error) {
1711
+ sendToolResponse(socket, originalId, null, `Failed to persist network requests: ${error.message}`);
1712
+ }
1492
1713
  } else if (savePath && msg.base64) {
1493
1714
  try {
1494
1715
  const dir = path.dirname(savePath);
@@ -1750,6 +1971,7 @@ const handleClient = (socket) => {
1750
1971
  }
1751
1972
  log(`Handling tool_request: ${msg.method} ${tool}${principal ? ` for ${principal.label}` : ""}`);
1752
1973
  try {
1974
+ if (tool.startsWith("oracle.")) oracleHost.assertLocal(request);
1753
1975
  if (isRemote) {
1754
1976
  await applyRequestTransfers(msg, request, transferState, ensureTransferState);
1755
1977
  }
@@ -1757,7 +1979,7 @@ const handleClient = (socket) => {
1757
1979
  requestStorage.run(request, () => handleToolRequest(msg, socket, request));
1758
1980
  } catch (e) {
1759
1981
  await discardRequestTransfers(msg, transferState);
1760
- sendToolResponse(socket, msg.id || null, null, e.message || "Request failed");
1982
+ sendToolResponse(socket, msg.id || null, null, e?.code ? e : e.message || "Request failed");
1761
1983
  }
1762
1984
  return;
1763
1985
  }
@@ -1,7 +1,7 @@
1
- const crypto = require("crypto");
2
1
  const fs = require("fs");
3
2
  const path = require("path");
4
3
  const { version: PACKAGE_VERSION } = require("../package.json");
4
+ const { atomicWriteFile } = require("./private-state.cjs");
5
5
 
6
6
  const MAX_NETWORK_EXPORT_FILE_BYTES = 256 * 1024 * 1024;
7
7
  const INTERNAL_FIELDS = new Set(["_requestId", "_responseReceived", "_loadingFinished"]);
@@ -22,6 +22,20 @@ function headerList(headers) {
22
22
  return Object.entries(headers).map(([name, value]) => ({ name, value: String(value) }));
23
23
  }
24
24
 
25
+ function headerValue(headers, name) {
26
+ if (!headers || typeof headers !== "object") return "";
27
+ const match = Object.entries(headers).find(([key]) => key.toLowerCase() === name);
28
+ return match ? String(match[1]) : "";
29
+ }
30
+
31
+ function queryList(url) {
32
+ try {
33
+ return [...new URL(url).searchParams.entries()].map(([name, value]) => ({ name, value }));
34
+ } catch {
35
+ return [];
36
+ }
37
+ }
38
+
25
39
  function harEntry(entry) {
26
40
  const requestBody = entry.requestBody;
27
41
  const responseBody = entry.responseBody;
@@ -37,11 +51,11 @@ function harEntry(entry) {
37
51
  url: entry.url || "",
38
52
  httpVersion: "HTTP/1.1",
39
53
  headers: headerList(requestHeaders),
40
- queryString: [],
54
+ queryString: queryList(entry.url || ""),
41
55
  cookies: [],
42
56
  headersSize: -1,
43
57
  bodySize: Number.isFinite(entry.requestBodySize) ? entry.requestBodySize : requestBody ? Buffer.byteLength(String(requestBody)) : -1,
44
- ...(requestBody !== undefined ? { postData: { mimeType: "application/octet-stream", text: String(requestBody) } } : {}),
58
+ ...(requestBody !== undefined ? { postData: { mimeType: headerValue(requestHeaders, "content-type") || "application/octet-stream", text: String(requestBody) } } : {}),
45
59
  },
46
60
  response: {
47
61
  status: Number.isFinite(entry.status) ? entry.status : 0,
@@ -53,6 +67,8 @@ function harEntry(entry) {
53
67
  size: Number.isFinite(entry.responseBodySize) ? entry.responseBodySize : responseBody ? Buffer.byteLength(String(responseBody)) : 0,
54
68
  mimeType: entry.mimeType || "",
55
69
  ...(responseBody !== undefined ? { text: String(responseBody) } : {}),
70
+ ...(entry.responseBodyEncoding === "base64" ? { encoding: "base64" } : {}),
71
+ _surfBodyCapture: entry.bodyCapture || { mode: "none", complete: responseBody === undefined ? false : true },
56
72
  },
57
73
  redirectURL: "",
58
74
  headersSize: -1,
@@ -87,20 +103,7 @@ function writeNetworkExport(outputPath, entries, format = "json") {
87
103
  const bytes = Buffer.byteLength(content);
88
104
  if (bytes > MAX_NETWORK_EXPORT_FILE_BYTES) throw new Error("network export exceeds the 256 MiB file limit");
89
105
  fs.mkdirSync(path.dirname(outputPath), { recursive: true });
90
- const temporaryPath = path.join(path.dirname(outputPath), `.${path.basename(outputPath)}.surf-${crypto.randomBytes(12).toString("hex")}.tmp`);
91
- try {
92
- const fd = fs.openSync(temporaryPath, "wx", 0o600);
93
- try {
94
- fs.writeFileSync(fd, content, "utf8");
95
- fs.fchmodSync(fd, 0o600);
96
- } finally {
97
- fs.closeSync(fd);
98
- }
99
- fs.renameSync(temporaryPath, outputPath);
100
- } catch (error) {
101
- try { fs.rmSync(temporaryPath, { force: true }); } catch {}
102
- throw error;
103
- }
106
+ atomicWriteFile(outputPath, content, { encoding: "utf8" });
104
107
  return { path: outputPath, format, count: entries.length, bytes };
105
108
  }
106
109
 
@@ -11,11 +11,17 @@ const fs = require("fs");
11
11
  const path = require("path");
12
12
  const crypto = require("crypto");
13
13
  const readline = require("readline");
14
+ const {
15
+ appendPrivateJsonLine,
16
+ assertNotSymlink,
17
+ atomicWriteFile,
18
+ atomicWriteJson,
19
+ ensurePrivateDir,
20
+ privateStatePath,
21
+ } = require("./private-state.cjs");
14
22
 
15
23
  // Configuration
16
- const DEFAULT_BASE = process.platform === "win32"
17
- ? require("path").join(require("os").tmpdir(), "surf")
18
- : "/tmp/surf";
24
+ const DEFAULT_BASE = privateStatePath("network");
19
25
  const DEFAULT_TTL = 24 * 60 * 60 * 1000; // 24 hours
20
26
  const DEFAULT_MAX_SIZE = 200 * 1024 * 1024; // 200MB
21
27
  const AUTO_CLEANUP_INTERVAL = 60 * 60 * 1000; // 1 hour
@@ -23,36 +29,26 @@ const AUTO_CLEANUP_INTERVAL = 60 * 60 * 1000; // 1 hour
23
29
  // Lock file for concurrent access
24
30
  let writeLock = Promise.resolve();
25
31
 
26
- // Runtime override for base path (set via CLI --network-path)
27
- let runtimeBasePath = null;
28
-
29
- /**
30
- * Set base path at runtime (from CLI --network-path flag)
31
- */
32
- function setBasePath(newPath) {
33
- runtimeBasePath = newPath;
34
- }
35
-
36
32
  /**
37
33
  * Get base path for network storage
38
- * Priority: runtime override > SURF_NETWORK_PATH env var > default
34
+ * Priority: SURF_NETWORK_PATH env var > default
39
35
  */
40
- function getBasePath() {
41
- return runtimeBasePath || process.env.SURF_NETWORK_PATH || DEFAULT_BASE;
36
+ function getBasePath(basePath) {
37
+ return basePath || process.env.SURF_NETWORK_PATH || DEFAULT_BASE;
42
38
  }
43
39
 
44
40
  /**
45
41
  * Get path to requests.jsonl
46
42
  */
47
- function getRequestsPath() {
48
- return path.join(getBasePath(), "requests.jsonl");
43
+ function getRequestsPath(basePath) {
44
+ return path.join(getBasePath(basePath), "requests.jsonl");
49
45
  }
50
46
 
51
47
  /**
52
48
  * Get path to bodies directory
53
49
  */
54
- function getBodiesPath() {
55
- return path.join(getBasePath(), "bodies");
50
+ function getBodiesPath(basePath) {
51
+ return path.join(getBasePath(basePath), "bodies");
56
52
  }
57
53
 
58
54
  /**
@@ -65,16 +61,11 @@ function getMetaPath() {
65
61
  /**
66
62
  * Ensure all required directories exist
67
63
  */
68
- function ensureDirectories() {
69
- const base = getBasePath();
70
- const bodies = getBodiesPath();
71
-
72
- if (!fs.existsSync(base)) {
73
- fs.mkdirSync(base, { recursive: true });
74
- }
75
- if (!fs.existsSync(bodies)) {
76
- fs.mkdirSync(bodies, { recursive: true });
77
- }
64
+ function ensureDirectories(basePath) {
65
+ const base = getBasePath(basePath);
66
+ const bodies = getBodiesPath(basePath);
67
+ ensurePrivateDir(base, base);
68
+ ensurePrivateDir(bodies, base);
78
69
  }
79
70
 
80
71
  /**
@@ -84,6 +75,7 @@ function readMeta() {
84
75
  const metaPath = getMetaPath();
85
76
  try {
86
77
  if (fs.existsSync(metaPath)) {
78
+ assertNotSymlink(metaPath, false);
87
79
  return JSON.parse(fs.readFileSync(metaPath, "utf-8"));
88
80
  }
89
81
  } catch (err) {
@@ -98,7 +90,7 @@ function readMeta() {
98
90
  function writeMeta(meta) {
99
91
  const metaPath = getMetaPath();
100
92
  ensureDirectories();
101
- fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2));
93
+ atomicWriteJson(metaPath, meta, { root: getBasePath() });
102
94
  }
103
95
 
104
96
  /**
@@ -124,7 +116,7 @@ function storeBody(content, isRequest = false) {
124
116
 
125
117
  // Only write if doesn't exist (dedup)
126
118
  if (!fs.existsSync(bodyPath)) {
127
- fs.writeFileSync(bodyPath, buffer);
119
+ atomicWriteFile(bodyPath, buffer, { root: getBasePath() });
128
120
  }
129
121
 
130
122
  return hash;
@@ -142,6 +134,7 @@ function readBody(hash, isRequest = false) {
142
134
 
143
135
  try {
144
136
  if (fs.existsSync(bodyPath)) {
137
+ assertNotSymlink(bodyPath, false);
145
138
  return fs.readFileSync(bodyPath);
146
139
  }
147
140
  } catch (err) {
@@ -186,10 +179,7 @@ async function appendEntry(entry) {
186
179
  ...entry
187
180
  };
188
181
 
189
- const line = JSON.stringify(fullEntry) + "\n";
190
-
191
- // Atomic append using flag 'a'
192
- fs.appendFileSync(getRequestsPath(), line, { flag: "a" });
182
+ appendPrivateJsonLine(getRequestsPath(), fullEntry, { root: getBasePath() });
193
183
 
194
184
  return fullEntry;
195
185
  } finally {
@@ -202,8 +192,8 @@ async function appendEntry(entry) {
202
192
  * @param {Object} entry - Network entry to append
203
193
  * @returns {Object} The entry with assigned ID
204
194
  */
205
- function appendEntrySync(entry) {
206
- ensureDirectories();
195
+ function appendEntrySync(entry, basePath) {
196
+ ensureDirectories(basePath);
207
197
 
208
198
  const id = entry.id || generateId();
209
199
  const timestamp = entry.timestamp || Date.now();
@@ -214,15 +204,14 @@ function appendEntrySync(entry) {
214
204
  ...entry
215
205
  };
216
206
 
217
- const line = JSON.stringify(fullEntry) + "\n";
218
-
219
207
  // Use a simple lock file for synchronous operations
220
- const lockPath = path.join(getBasePath(), ".lock");
208
+ const lockPath = path.join(getBasePath(basePath), ".lock");
221
209
  let lockFd;
222
210
 
223
211
  try {
224
212
  // Try to acquire lock
225
- lockFd = fs.openSync(lockPath, "wx");
213
+ assertNotSymlink(lockPath, true);
214
+ lockFd = fs.openSync(lockPath, "wx", 0o600);
226
215
  } catch (err) {
227
216
  // Lock exists - check if stale and remove, otherwise proceed without lock
228
217
  try {
@@ -230,7 +219,7 @@ function appendEntrySync(entry) {
230
219
  if (Date.now() - stat.mtimeMs > 5000) {
231
220
  fs.unlinkSync(lockPath);
232
221
  try {
233
- lockFd = fs.openSync(lockPath, "wx");
222
+ lockFd = fs.openSync(lockPath, "wx", 0o600);
234
223
  } catch (e) {
235
224
  // Still can't get lock, proceed without it
236
225
  }
@@ -241,13 +230,13 @@ function appendEntrySync(entry) {
241
230
 
242
231
  if (lockFd === undefined) {
243
232
  // Proceed without lock as fallback
244
- fs.appendFileSync(getRequestsPath(), line, { flag: "a" });
233
+ appendPrivateJsonLine(getRequestsPath(basePath), fullEntry, { root: getBasePath(basePath) });
245
234
  return fullEntry;
246
235
  }
247
236
  }
248
237
 
249
238
  try {
250
- fs.appendFileSync(getRequestsPath(), line, { flag: "a" });
239
+ appendPrivateJsonLine(getRequestsPath(basePath), fullEntry, { root: getBasePath(basePath) });
251
240
  } finally {
252
241
  if (lockFd !== undefined) {
253
242
  fs.closeSync(lockFd);
@@ -385,6 +374,7 @@ async function readEntries(filters = {}) {
385
374
  if (!fs.existsSync(requestsPath)) {
386
375
  return [];
387
376
  }
377
+ assertNotSymlink(requestsPath, false);
388
378
 
389
379
  const { last } = filters;
390
380
  const entries = [];
@@ -433,6 +423,7 @@ function readEntriesSync(filters = {}) {
433
423
  if (!fs.existsSync(requestsPath)) {
434
424
  return [];
435
425
  }
426
+ assertNotSymlink(requestsPath, false);
436
427
 
437
428
  const { last } = filters;
438
429
  const entries = [];
@@ -682,10 +673,8 @@ async function cleanup(options = {}) {
682
673
 
683
674
  if (deletedEntries > 0 || entries.length === 0) {
684
675
  // Atomic write: write to temp then rename
685
- const tempPath = requestsPath + ".tmp";
686
676
  const content = entries.map(e => JSON.stringify(e)).join("\n") + (entries.length > 0 ? "\n" : "");
687
- fs.writeFileSync(tempPath, content);
688
- fs.renameSync(tempPath, requestsPath);
677
+ atomicWriteFile(requestsPath, content, { root: getBasePath(), encoding: "utf8" });
689
678
  }
690
679
 
691
680
  // 6. Update meta
@@ -777,10 +766,8 @@ async function clear(options = {}) {
777
766
 
778
767
  // Rewrite entries file
779
768
  if (deletedEntries > 0) {
780
- const tempPath = requestsPath + ".tmp";
781
769
  const content = remaining.map(e => JSON.stringify(e)).join("\n") + (remaining.length > 0 ? "\n" : "");
782
- fs.writeFileSync(tempPath, content);
783
- fs.renameSync(tempPath, requestsPath);
770
+ atomicWriteFile(requestsPath, content, { root: getBasePath(), encoding: "utf8" });
784
771
  }
785
772
 
786
773
  return { deletedEntries, deletedBodies };
@@ -807,9 +794,6 @@ function maybeAutoCleanup() {
807
794
  }
808
795
  }
809
796
 
810
- // Run auto-cleanup check on module load
811
- maybeAutoCleanup();
812
-
813
797
  module.exports = {
814
798
  // Configuration
815
799
  getBasePath,
@@ -841,10 +825,6 @@ module.exports = {
841
825
  clear,
842
826
  maybeAutoCleanup,
843
827
 
844
- // Configuration
845
- setBasePath,
846
- getBasePath,
847
-
848
828
  // Constants
849
829
  DEFAULT_BASE,
850
830
  DEFAULT_TTL,