surf-cli 2.9.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -4
- package/dist/content/accessibility-tree.js +11 -0
- package/dist/content/accessibility-tree.js.map +1 -0
- package/dist/content/visual-indicator.js +111 -0
- package/dist/content/visual-indicator.js.map +1 -0
- package/dist/manifest.json +11 -2
- package/dist/options/options.js +3 -3
- package/dist/options/options.js.map +1 -1
- package/dist/service-worker/index.js +61 -261
- package/dist/service-worker/index.js.map +1 -1
- package/native/activity-journal.cjs +55 -0
- package/native/chatgpt-client-response.cjs +336 -0
- package/native/chatgpt-client-selection.cjs +119 -0
- package/native/chatgpt-client-ui.cjs +481 -0
- package/native/chatgpt-client.cjs +254 -664
- package/native/cli.cjs +100 -273
- package/native/do-executor.cjs +52 -475
- package/native/do-parser.cjs +8 -249
- package/native/host-helpers.cjs +32 -15
- package/native/host-sessions.cjs +6 -1
- package/native/host.cjs +228 -6
- package/native/network-export.cjs +20 -17
- package/native/network-store.cjs +38 -58
- package/native/oracle-cli.cjs +434 -0
- package/native/oracle-context.cjs +311 -0
- package/native/oracle-host.cjs +301 -0
- package/native/oracle-jobs.cjs +253 -0
- package/native/playbook-authoring.cjs +44 -0
- package/native/playbook-cli.cjs +157 -0
- package/native/playbook-client.cjs +259 -0
- package/native/playbook-receipts.cjs +109 -0
- package/native/playbook-records.cjs +208 -0
- package/native/playbook-runtime.cjs +177 -0
- package/native/playbooks.cjs +235 -0
- package/native/private-state.cjs +156 -0
- package/native/redaction.cjs +104 -0
- package/native/workflow-definition.cjs +369 -0
- package/native/workflow-runtime.cjs +225 -0
- package/package.json +2 -1
- package/playbooks/page/ops/read.json +22 -0
- package/playbooks/page/playbook.json +7 -0
- package/skills/surf/SKILL.md +72 -1
- package/dist/content/index.js +0 -116
- package/dist/content/index.js.map +0 -1
package/native/host.cjs
CHANGED
|
@@ -14,7 +14,8 @@ const perplexityClient = require("./perplexity-client.cjs");
|
|
|
14
14
|
const grokClient = require("./grok-client.cjs");
|
|
15
15
|
const aistudioClient = require("./aistudio-client.cjs");
|
|
16
16
|
const aistudioBuild = require("./aistudio-build.cjs");
|
|
17
|
-
const { mapToolToMessage, mapComputerAction, formatToolContent, buildProviderUploadMessage } = require("./host-helpers.cjs");
|
|
17
|
+
const { mapToolToMessage, mapComputerAction, formatToolContent, formatToolError, buildProviderUploadMessage } = require("./host-helpers.cjs");
|
|
18
|
+
const { createOracleHost } = require("./oracle-host.cjs");
|
|
18
19
|
|
|
19
20
|
const IS_WIN = process.platform === "win32";
|
|
20
21
|
const { SOCKET_PATH, SURF_TMP } = require("./socket-path.cjs");
|
|
@@ -22,11 +23,30 @@ const { parseListenEndpoint } = require("./listener.cjs");
|
|
|
22
23
|
const { getStateDir } = require("./remote-auth.cjs");
|
|
23
24
|
const { createFrameParser, createServerAuthSession, createSocketWriter, isClientAuthorized, writeFrame, MAX_FRAME_BYTES } = require("./remote-transport.cjs");
|
|
24
25
|
const { HostSessionManager, resolveRequestDeadlineMs } = require("./host-sessions.cjs");
|
|
25
|
-
const { abortError, throwIfAborted } = require("./abort.cjs");
|
|
26
|
+
const { abortError, abortableDelay, throwIfAborted } = require("./abort.cjs");
|
|
26
27
|
const { BoundedAiQueue } = require("./ai-queue.cjs");
|
|
27
28
|
const { RequestPendingMap } = require("./request-pending.cjs");
|
|
28
29
|
const { cleanupFilePaths, createStagingDirectory, createTransferState, materializeRemoteTool, rewriteTransferPaths, streamFileDownload, transferError } = require("./file-transfer.cjs");
|
|
29
30
|
const { writeNetworkExport } = require("./network-export.cjs");
|
|
31
|
+
const networkStore = require("./network-store.cjs");
|
|
32
|
+
const { redactUrlSecrets } = require("./redaction.cjs");
|
|
33
|
+
const { appendActivity, journalCommand } = require("./activity-journal.cjs");
|
|
34
|
+
const { reserveReceipt, updateReceipt } = require("./playbook-receipts.cjs");
|
|
35
|
+
const {
|
|
36
|
+
activeRecord,
|
|
37
|
+
appendRecordEvent,
|
|
38
|
+
attachNetworkTrace,
|
|
39
|
+
discardRecord,
|
|
40
|
+
markRecord,
|
|
41
|
+
pauseRecord,
|
|
42
|
+
resumeRecord,
|
|
43
|
+
startRecord,
|
|
44
|
+
stopRecord,
|
|
45
|
+
updateRecordContext,
|
|
46
|
+
} = require("./playbook-records.cjs");
|
|
47
|
+
const { resolveArgs, runPlaybookOp } = require("./playbook-runtime.cjs");
|
|
48
|
+
const { resolveOp } = require("./playbooks.cjs");
|
|
49
|
+
const { commandMetadata, redactCommandArgs } = require("./workflow-definition.cjs");
|
|
30
50
|
const MAX_CLIENT_FRAME_BYTES = MAX_FRAME_BYTES;
|
|
31
51
|
const TEST_REQUEST_DEADLINE_MS = process.env.SURF_TEST_MODE === "1" && Number.isFinite(Number(process.env.SURF_TEST_REQUEST_DEADLINE_MS))
|
|
32
52
|
? Number(process.env.SURF_TEST_REQUEST_DEADLINE_MS)
|
|
@@ -383,6 +403,15 @@ aiQueue = new BoundedAiQueue({
|
|
|
383
403
|
: handler(),
|
|
384
404
|
});
|
|
385
405
|
|
|
406
|
+
const oracleHost = createOracleHost({
|
|
407
|
+
queueAiRequest,
|
|
408
|
+
requestCallExtension,
|
|
409
|
+
buildProviderUploadMessage,
|
|
410
|
+
log,
|
|
411
|
+
});
|
|
412
|
+
const adoptedOracleJobs = oracleHost.adoptOrphans();
|
|
413
|
+
log(`Oracle adoption: ${adoptedOracleJobs.length} job(s); ids=${adoptedOracleJobs.map((job) => job.id).join(",") || "none"}`);
|
|
414
|
+
|
|
386
415
|
function sendSocket(socket, value, options = {}) {
|
|
387
416
|
const writer = socketWriters.get(socket);
|
|
388
417
|
return writer ? writer.send(value, options) : writeFrame(socket, value);
|
|
@@ -428,6 +457,131 @@ function requestCallExtension(request, tool, message, timeoutMs = 30000, cleanup
|
|
|
428
457
|
});
|
|
429
458
|
}
|
|
430
459
|
|
|
460
|
+
async function executeMappedHostTool(request, tool, args, tabId) {
|
|
461
|
+
const extensionMsg = mapToolToMessage(tool, args, tabId);
|
|
462
|
+
if (!extensionMsg) throw new Error(`Unknown tool: ${tool}`);
|
|
463
|
+
if (extensionMsg.type === "UNSUPPORTED_ACTION") throw new Error(extensionMsg.message);
|
|
464
|
+
if (extensionMsg.type === "LOCAL_WAIT") {
|
|
465
|
+
await abortableDelay(extensionMsg.seconds * 1000, request.signal);
|
|
466
|
+
return { success: true };
|
|
467
|
+
}
|
|
468
|
+
if (extensionMsg.type === "BATCH_EXECUTE" || extensionMsg.type.endsWith("_QUERY")) {
|
|
469
|
+
throw new Error(`tool ${tool} is not available inside a host-owned workflow`);
|
|
470
|
+
}
|
|
471
|
+
return requestCallExtension(request, tool, extensionMsg, resolveRequestDeadlineMs(tool, args));
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
async function executeNativePlaybook(request, handler, args, options = {}) {
|
|
475
|
+
if (handler !== "chatgpt.ask") throw new Error(`unknown native playbook handler: ${handler}`);
|
|
476
|
+
const result = await chatgptClient.query({
|
|
477
|
+
prompt: args.prompt,
|
|
478
|
+
signal: request.signal,
|
|
479
|
+
model: args.model,
|
|
480
|
+
timeout: args.timeout ? Number(args.timeout) * 1000 : undefined,
|
|
481
|
+
getCookies: () => requestCallExtension(request, "get_cookies", { type: "GET_CHATGPT_COOKIES" }),
|
|
482
|
+
createTab: () => requestCallExtension(request, "create_tab", { type: "CHATGPT_NEW_TAB" }),
|
|
483
|
+
closeTab: (tabId) => requestCallExtension(request, "close_tab", { type: "CHATGPT_CLOSE_TAB", tabId }, 45000, true),
|
|
484
|
+
cdpEvaluate: (tabId, expression) => requestCallExtension(request, "cdp_evaluate", { type: "CHATGPT_EVALUATE", tabId, expression }),
|
|
485
|
+
cdpCommand: (tabId, method, params) => requestCallExtension(request, "cdp_command", { type: "CHATGPT_CDP_COMMAND", tabId, method, params }),
|
|
486
|
+
beforeSubmit: options.markDispatched,
|
|
487
|
+
log: (message) => log(`[playbook:chatgpt] ${message}`),
|
|
488
|
+
});
|
|
489
|
+
return { response: result.response, model: result.model, tookMs: result.tookMs };
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
async function runHostPlaybook(msg, request) {
|
|
493
|
+
const params = msg.params?.args || {};
|
|
494
|
+
const { playbook, op } = resolveOp(params.playbook, params.op, {
|
|
495
|
+
cwd: request.context?.isRemote ? process.cwd() : params.projectDir || process.cwd(),
|
|
496
|
+
pinBuiltIn: params.pinBuiltIn === true,
|
|
497
|
+
});
|
|
498
|
+
const runArgs = resolveArgs(op, params.args || {});
|
|
499
|
+
if (op.effect === "write" && op.safety.authorization === "explicit" && params.write !== true) {
|
|
500
|
+
throw new Error(`write op ${playbook.id} ${op.id} requires --write`);
|
|
501
|
+
}
|
|
502
|
+
const receipt = reserveReceipt({
|
|
503
|
+
playbookId: playbook.id,
|
|
504
|
+
op,
|
|
505
|
+
args: runArgs,
|
|
506
|
+
repeat: params.repeat === true,
|
|
507
|
+
retryAttempt: params.retryAttempt,
|
|
508
|
+
overrideInDoubt: params.overrideInDoubt === true,
|
|
509
|
+
});
|
|
510
|
+
const report = (event) => {
|
|
511
|
+
appendActivity(event);
|
|
512
|
+
appendRecordEvent(event);
|
|
513
|
+
};
|
|
514
|
+
return runPlaybookOp({
|
|
515
|
+
playbook,
|
|
516
|
+
op,
|
|
517
|
+
args: runArgs,
|
|
518
|
+
attemptId: receipt?.attemptId,
|
|
519
|
+
signal: request.signal,
|
|
520
|
+
executeTool: (tool, args) => executeMappedHostTool(request, tool, args, msg.tabId),
|
|
521
|
+
executeNative: (handler, args, options) => executeNativePlaybook(request, handler, args, options),
|
|
522
|
+
sleep: (ms) => abortableDelay(ms, request.signal),
|
|
523
|
+
onEvent: report,
|
|
524
|
+
beforeDispatch: async () => updateReceipt(receipt, "dispatched"),
|
|
525
|
+
afterDispatch: async ({ status, error }) => updateReceipt(receipt, status, { error }),
|
|
526
|
+
});
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
async function handleRecordRequest(tool, args, msg, request) {
|
|
530
|
+
if (tool === "playbook.record.start") {
|
|
531
|
+
let record = startRecord({ ...args, tabId: msg.tabId });
|
|
532
|
+
try {
|
|
533
|
+
const context = await requestCallExtension(request, tool, {
|
|
534
|
+
type: "GET_PLAYBOOK_RECORD_CONTEXT",
|
|
535
|
+
tabId: msg.tabId,
|
|
536
|
+
});
|
|
537
|
+
record = updateRecordContext({
|
|
538
|
+
tabId: context._resolvedTabId || msg.tabId,
|
|
539
|
+
origin: context.origin,
|
|
540
|
+
});
|
|
541
|
+
if (record.capture.network) {
|
|
542
|
+
const result = await requestCallExtension(request, tool, { type: "START_NETWORK_CAPTURE", tabId: record.tabId, bodyMode: "text" });
|
|
543
|
+
record = updateRecordContext({ tabId: result._resolvedTabId || record.tabId });
|
|
544
|
+
}
|
|
545
|
+
if (record.capture.watch) {
|
|
546
|
+
const result = await requestCallExtension(request, tool, { type: "START_PLAYBOOK_WATCH", tabId: record.tabId || msg.tabId, includeInputValues: record.redaction.includeInputValues });
|
|
547
|
+
record = updateRecordContext({ tabId: result._resolvedTabId || record.tabId || msg.tabId });
|
|
548
|
+
}
|
|
549
|
+
return record;
|
|
550
|
+
} catch (error) {
|
|
551
|
+
if (record?.capture.network) await requestCallExtension(request, tool, { type: "STOP_NETWORK_CAPTURE", tabId: record.tabId }, 30000, true).catch(() => {});
|
|
552
|
+
if (record?.capture.watch) await requestCallExtension(request, tool, { type: "STOP_PLAYBOOK_WATCH", tabId: record.tabId }, 30000, true).catch(() => {});
|
|
553
|
+
discardRecord();
|
|
554
|
+
throw error;
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
if (tool === "playbook.record.status") return activeRecord() || { status: "idle" };
|
|
558
|
+
if (tool === "playbook.record.mark") return markRecord(args.label);
|
|
559
|
+
if (tool === "playbook.record.pause") return pauseRecord();
|
|
560
|
+
if (tool === "playbook.record.resume") return resumeRecord();
|
|
561
|
+
if (tool === "playbook.record.discard") {
|
|
562
|
+
const record = activeRecord();
|
|
563
|
+
if (record?.capture.network) await requestCallExtension(request, tool, { type: "STOP_NETWORK_CAPTURE", tabId: record.tabId }, 30000, true).catch(() => {});
|
|
564
|
+
if (record?.capture.watch) await requestCallExtension(request, tool, { type: "STOP_PLAYBOOK_WATCH", tabId: record.tabId }, 30000, true).catch(() => {});
|
|
565
|
+
return discardRecord();
|
|
566
|
+
}
|
|
567
|
+
if (tool === "playbook.record.stop") {
|
|
568
|
+
const record = activeRecord();
|
|
569
|
+
if (!record) throw new Error("no active playbook record");
|
|
570
|
+
if (record.capture.network) {
|
|
571
|
+
try {
|
|
572
|
+
const result = await requestCallExtension(request, tool, { type: "READ_NETWORK_REQUESTS", tabId: record.tabId, full: true, limit: 500 });
|
|
573
|
+
const cutoff = Date.parse(record.startedAt);
|
|
574
|
+
attachNetworkTrace(record.id, (result.entries || []).filter((entry) => entry.ts >= cutoff));
|
|
575
|
+
} finally {
|
|
576
|
+
await requestCallExtension(request, tool, { type: "STOP_NETWORK_CAPTURE", tabId: record.tabId }, 30000, true).catch(() => {});
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
if (record.capture.watch) await requestCallExtension(request, tool, { type: "STOP_PLAYBOOK_WATCH", tabId: record.tabId }, 30000, true).catch(() => {});
|
|
580
|
+
return stopRecord({ draft: args.draft === true });
|
|
581
|
+
}
|
|
582
|
+
throw new Error(`Unknown record command: ${tool}`);
|
|
583
|
+
}
|
|
584
|
+
|
|
431
585
|
const sessionManager = new HostSessionManager({
|
|
432
586
|
audit: auditSession,
|
|
433
587
|
onTimeout(context, request) {
|
|
@@ -528,8 +682,29 @@ function sendToolResponse(socket, id, result, error) {
|
|
|
528
682
|
} catch (transferFailure) {
|
|
529
683
|
finalError = transferFailure.message;
|
|
530
684
|
}
|
|
531
|
-
|
|
532
|
-
|
|
685
|
+
const formattedError = finalError ? formatToolError(finalError) : null;
|
|
686
|
+
if (formattedError && request) {
|
|
687
|
+
const rewrittenMessage = rewriteTransferPaths(
|
|
688
|
+
formattedError.content[0].text,
|
|
689
|
+
request.pathRewrites || [],
|
|
690
|
+
);
|
|
691
|
+
formattedError.content[0].text = rewrittenMessage;
|
|
692
|
+
if (formattedError.message) formattedError.message = rewrittenMessage;
|
|
693
|
+
}
|
|
694
|
+
if (request?.tool && !request.tool.startsWith("playbook.")) {
|
|
695
|
+
const metadata = commandMetadata(request.tool);
|
|
696
|
+
if (metadata.recordable) {
|
|
697
|
+
const event = {
|
|
698
|
+
type: finalError ? "tool.failed" : "tool.completed",
|
|
699
|
+
command: metadata.name,
|
|
700
|
+
argsRedacted: redactCommandArgs(request.tool, request.args || {}),
|
|
701
|
+
startedAt: request.activityStartedAt || new Date().toISOString(),
|
|
702
|
+
endedAt: new Date().toISOString(),
|
|
703
|
+
resultSummary: finalError ? "failed" : "success",
|
|
704
|
+
};
|
|
705
|
+
appendActivity(event);
|
|
706
|
+
appendRecordEvent(event);
|
|
707
|
+
}
|
|
533
708
|
}
|
|
534
709
|
await cleanupRequestTransfers(request);
|
|
535
710
|
if (request?.settled) return;
|
|
@@ -538,7 +713,7 @@ function sendToolResponse(socket, id, result, error) {
|
|
|
538
713
|
: finalError ? "error" : "completed";
|
|
539
714
|
await completeOwnedRequest(context, id, outcome);
|
|
540
715
|
const response = { type: "tool_response", id };
|
|
541
|
-
if (
|
|
716
|
+
if (formattedError) response.error = formattedError;
|
|
542
717
|
else response.result = { content: formatToolContent(output, log, { suppressImages: Boolean(context?.isRemote) }) };
|
|
543
718
|
if (!context?.closed) await sendSocket(socket, response);
|
|
544
719
|
})().catch((sendError) => log(`Error sending tool_response: ${sendError.message}`));
|
|
@@ -607,6 +782,22 @@ function handleToolRequest(msg, socket, requestContext = requestStorage.getStore
|
|
|
607
782
|
sendToolResponse(socket, originalId, null, "No tool specified");
|
|
608
783
|
return;
|
|
609
784
|
}
|
|
785
|
+
|
|
786
|
+
requestContext.args = args || {};
|
|
787
|
+
requestContext.activityStartedAt = new Date().toISOString();
|
|
788
|
+
if (!tool.startsWith("playbook.")) journalCommand(tool, args || {}, { tabId });
|
|
789
|
+
if (tool === "playbook.run") {
|
|
790
|
+
runHostPlaybook(msg, requestContext)
|
|
791
|
+
.then((result) => sendToolResponse(socket, originalId, { output: JSON.stringify(result) }, null))
|
|
792
|
+
.catch((error) => sendToolResponse(socket, originalId, null, error.message));
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
if (tool.startsWith("playbook.record.")) {
|
|
796
|
+
handleRecordRequest(tool, args || {}, msg, requestContext)
|
|
797
|
+
.then((result) => sendToolResponse(socket, originalId, result, null))
|
|
798
|
+
.catch((error) => sendToolResponse(socket, originalId, null, error.message));
|
|
799
|
+
return;
|
|
800
|
+
}
|
|
610
801
|
|
|
611
802
|
const extensionMsg = mapToolToMessage(tool, args, tabId);
|
|
612
803
|
if (!extensionMsg) {
|
|
@@ -625,6 +816,13 @@ function handleToolRequest(msg, socket, requestContext = requestStorage.getStore
|
|
|
625
816
|
.catch((error) => sendToolResponse(socket, originalId, null, error.message));
|
|
626
817
|
return;
|
|
627
818
|
}
|
|
819
|
+
|
|
820
|
+
if (extensionMsg.type.startsWith("ORACLE_")) {
|
|
821
|
+
Promise.resolve(oracleHost.handle(requestContext, extensionMsg))
|
|
822
|
+
.then((result) => sendToolResponse(socket, originalId, result, null))
|
|
823
|
+
.catch((error) => sendToolResponse(socket, originalId, null, error));
|
|
824
|
+
return;
|
|
825
|
+
}
|
|
628
826
|
|
|
629
827
|
if (extensionMsg.type === "BATCH_EXECUTE") {
|
|
630
828
|
executeBatch(extensionMsg.actions, extensionMsg.tabId, socket, originalId, requestContext);
|
|
@@ -1264,7 +1462,9 @@ function handleToolRequest(msg, socket, requestContext = requestStorage.getStore
|
|
|
1264
1462
|
autoScreenshot: args?.autoScreenshot === true,
|
|
1265
1463
|
autoScreenshotOutput: args?.autoScreenshotOutput,
|
|
1266
1464
|
networkExport: extensionMsg.type === "EXPORT_NETWORK_REQUESTS",
|
|
1465
|
+
persistNetwork: extensionMsg.type === "READ_NETWORK_REQUESTS" && extensionMsg.full && args?.["no-save"] !== true,
|
|
1267
1466
|
networkExportPath: args?.output,
|
|
1467
|
+
networkPath: args?.["network-path"],
|
|
1268
1468
|
networkExportFormat: extensionMsg.har ? "har" : extensionMsg.jsonl ? "jsonl" : "json",
|
|
1269
1469
|
fullRes: extensionMsg.fullRes || args?.fullRes,
|
|
1270
1470
|
maxSize: extensionMsg.maxSize || args?.maxSize,
|
|
@@ -1431,6 +1631,19 @@ function processInput() {
|
|
|
1431
1631
|
handleApiRequest(msg, writeMessage);
|
|
1432
1632
|
return;
|
|
1433
1633
|
}
|
|
1634
|
+
|
|
1635
|
+
if (msg.type === "PLAYBOOK_WATCH_EVENT") {
|
|
1636
|
+
appendRecordEvent({
|
|
1637
|
+
type: "browser.event",
|
|
1638
|
+
event: msg.event,
|
|
1639
|
+
selector: msg.selector,
|
|
1640
|
+
value: msg.value,
|
|
1641
|
+
url: redactUrlSecrets(msg.url),
|
|
1642
|
+
tabId: msg.tabId,
|
|
1643
|
+
timestamp: msg.timestamp || new Date().toISOString(),
|
|
1644
|
+
});
|
|
1645
|
+
return;
|
|
1646
|
+
}
|
|
1434
1647
|
|
|
1435
1648
|
if (msg.type === "STREAM_EVENT") {
|
|
1436
1649
|
const stream = activeStreams.get(msg.streamId);
|
|
@@ -1489,6 +1702,14 @@ function processInput() {
|
|
|
1489
1702
|
} catch (error) {
|
|
1490
1703
|
sendToolResponse(socket, originalId, null, `Failed to export network requests: ${error.message}`);
|
|
1491
1704
|
}
|
|
1705
|
+
} else if (pending.persistNetwork && Array.isArray(msg.entries)) {
|
|
1706
|
+
try {
|
|
1707
|
+
for (const entry of msg.entries) networkStore.appendEntrySync(entry, pending.networkPath);
|
|
1708
|
+
networkStore.maybeAutoCleanup();
|
|
1709
|
+
sendToolResponse(socket, originalId, msg, null);
|
|
1710
|
+
} catch (error) {
|
|
1711
|
+
sendToolResponse(socket, originalId, null, `Failed to persist network requests: ${error.message}`);
|
|
1712
|
+
}
|
|
1492
1713
|
} else if (savePath && msg.base64) {
|
|
1493
1714
|
try {
|
|
1494
1715
|
const dir = path.dirname(savePath);
|
|
@@ -1750,6 +1971,7 @@ const handleClient = (socket) => {
|
|
|
1750
1971
|
}
|
|
1751
1972
|
log(`Handling tool_request: ${msg.method} ${tool}${principal ? ` for ${principal.label}` : ""}`);
|
|
1752
1973
|
try {
|
|
1974
|
+
if (tool.startsWith("oracle.")) oracleHost.assertLocal(request);
|
|
1753
1975
|
if (isRemote) {
|
|
1754
1976
|
await applyRequestTransfers(msg, request, transferState, ensureTransferState);
|
|
1755
1977
|
}
|
|
@@ -1757,7 +1979,7 @@ const handleClient = (socket) => {
|
|
|
1757
1979
|
requestStorage.run(request, () => handleToolRequest(msg, socket, request));
|
|
1758
1980
|
} catch (e) {
|
|
1759
1981
|
await discardRequestTransfers(msg, transferState);
|
|
1760
|
-
sendToolResponse(socket, msg.id || null, null, e.message || "Request failed");
|
|
1982
|
+
sendToolResponse(socket, msg.id || null, null, e?.code ? e : e.message || "Request failed");
|
|
1761
1983
|
}
|
|
1762
1984
|
return;
|
|
1763
1985
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
const crypto = require("crypto");
|
|
2
1
|
const fs = require("fs");
|
|
3
2
|
const path = require("path");
|
|
4
3
|
const { version: PACKAGE_VERSION } = require("../package.json");
|
|
4
|
+
const { atomicWriteFile } = require("./private-state.cjs");
|
|
5
5
|
|
|
6
6
|
const MAX_NETWORK_EXPORT_FILE_BYTES = 256 * 1024 * 1024;
|
|
7
7
|
const INTERNAL_FIELDS = new Set(["_requestId", "_responseReceived", "_loadingFinished"]);
|
|
@@ -22,6 +22,20 @@ function headerList(headers) {
|
|
|
22
22
|
return Object.entries(headers).map(([name, value]) => ({ name, value: String(value) }));
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
function headerValue(headers, name) {
|
|
26
|
+
if (!headers || typeof headers !== "object") return "";
|
|
27
|
+
const match = Object.entries(headers).find(([key]) => key.toLowerCase() === name);
|
|
28
|
+
return match ? String(match[1]) : "";
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function queryList(url) {
|
|
32
|
+
try {
|
|
33
|
+
return [...new URL(url).searchParams.entries()].map(([name, value]) => ({ name, value }));
|
|
34
|
+
} catch {
|
|
35
|
+
return [];
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
25
39
|
function harEntry(entry) {
|
|
26
40
|
const requestBody = entry.requestBody;
|
|
27
41
|
const responseBody = entry.responseBody;
|
|
@@ -37,11 +51,11 @@ function harEntry(entry) {
|
|
|
37
51
|
url: entry.url || "",
|
|
38
52
|
httpVersion: "HTTP/1.1",
|
|
39
53
|
headers: headerList(requestHeaders),
|
|
40
|
-
queryString:
|
|
54
|
+
queryString: queryList(entry.url || ""),
|
|
41
55
|
cookies: [],
|
|
42
56
|
headersSize: -1,
|
|
43
57
|
bodySize: Number.isFinite(entry.requestBodySize) ? entry.requestBodySize : requestBody ? Buffer.byteLength(String(requestBody)) : -1,
|
|
44
|
-
...(requestBody !== undefined ? { postData: { mimeType: "application/octet-stream", text: String(requestBody) } } : {}),
|
|
58
|
+
...(requestBody !== undefined ? { postData: { mimeType: headerValue(requestHeaders, "content-type") || "application/octet-stream", text: String(requestBody) } } : {}),
|
|
45
59
|
},
|
|
46
60
|
response: {
|
|
47
61
|
status: Number.isFinite(entry.status) ? entry.status : 0,
|
|
@@ -53,6 +67,8 @@ function harEntry(entry) {
|
|
|
53
67
|
size: Number.isFinite(entry.responseBodySize) ? entry.responseBodySize : responseBody ? Buffer.byteLength(String(responseBody)) : 0,
|
|
54
68
|
mimeType: entry.mimeType || "",
|
|
55
69
|
...(responseBody !== undefined ? { text: String(responseBody) } : {}),
|
|
70
|
+
...(entry.responseBodyEncoding === "base64" ? { encoding: "base64" } : {}),
|
|
71
|
+
_surfBodyCapture: entry.bodyCapture || { mode: "none", complete: responseBody === undefined ? false : true },
|
|
56
72
|
},
|
|
57
73
|
redirectURL: "",
|
|
58
74
|
headersSize: -1,
|
|
@@ -87,20 +103,7 @@ function writeNetworkExport(outputPath, entries, format = "json") {
|
|
|
87
103
|
const bytes = Buffer.byteLength(content);
|
|
88
104
|
if (bytes > MAX_NETWORK_EXPORT_FILE_BYTES) throw new Error("network export exceeds the 256 MiB file limit");
|
|
89
105
|
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
|
90
|
-
|
|
91
|
-
try {
|
|
92
|
-
const fd = fs.openSync(temporaryPath, "wx", 0o600);
|
|
93
|
-
try {
|
|
94
|
-
fs.writeFileSync(fd, content, "utf8");
|
|
95
|
-
fs.fchmodSync(fd, 0o600);
|
|
96
|
-
} finally {
|
|
97
|
-
fs.closeSync(fd);
|
|
98
|
-
}
|
|
99
|
-
fs.renameSync(temporaryPath, outputPath);
|
|
100
|
-
} catch (error) {
|
|
101
|
-
try { fs.rmSync(temporaryPath, { force: true }); } catch {}
|
|
102
|
-
throw error;
|
|
103
|
-
}
|
|
106
|
+
atomicWriteFile(outputPath, content, { encoding: "utf8" });
|
|
104
107
|
return { path: outputPath, format, count: entries.length, bytes };
|
|
105
108
|
}
|
|
106
109
|
|
package/native/network-store.cjs
CHANGED
|
@@ -11,11 +11,17 @@ const fs = require("fs");
|
|
|
11
11
|
const path = require("path");
|
|
12
12
|
const crypto = require("crypto");
|
|
13
13
|
const readline = require("readline");
|
|
14
|
+
const {
|
|
15
|
+
appendPrivateJsonLine,
|
|
16
|
+
assertNotSymlink,
|
|
17
|
+
atomicWriteFile,
|
|
18
|
+
atomicWriteJson,
|
|
19
|
+
ensurePrivateDir,
|
|
20
|
+
privateStatePath,
|
|
21
|
+
} = require("./private-state.cjs");
|
|
14
22
|
|
|
15
23
|
// Configuration
|
|
16
|
-
const DEFAULT_BASE =
|
|
17
|
-
? require("path").join(require("os").tmpdir(), "surf")
|
|
18
|
-
: "/tmp/surf";
|
|
24
|
+
const DEFAULT_BASE = privateStatePath("network");
|
|
19
25
|
const DEFAULT_TTL = 24 * 60 * 60 * 1000; // 24 hours
|
|
20
26
|
const DEFAULT_MAX_SIZE = 200 * 1024 * 1024; // 200MB
|
|
21
27
|
const AUTO_CLEANUP_INTERVAL = 60 * 60 * 1000; // 1 hour
|
|
@@ -23,36 +29,26 @@ const AUTO_CLEANUP_INTERVAL = 60 * 60 * 1000; // 1 hour
|
|
|
23
29
|
// Lock file for concurrent access
|
|
24
30
|
let writeLock = Promise.resolve();
|
|
25
31
|
|
|
26
|
-
// Runtime override for base path (set via CLI --network-path)
|
|
27
|
-
let runtimeBasePath = null;
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Set base path at runtime (from CLI --network-path flag)
|
|
31
|
-
*/
|
|
32
|
-
function setBasePath(newPath) {
|
|
33
|
-
runtimeBasePath = newPath;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
32
|
/**
|
|
37
33
|
* Get base path for network storage
|
|
38
|
-
* Priority:
|
|
34
|
+
* Priority: SURF_NETWORK_PATH env var > default
|
|
39
35
|
*/
|
|
40
|
-
function getBasePath() {
|
|
41
|
-
return
|
|
36
|
+
function getBasePath(basePath) {
|
|
37
|
+
return basePath || process.env.SURF_NETWORK_PATH || DEFAULT_BASE;
|
|
42
38
|
}
|
|
43
39
|
|
|
44
40
|
/**
|
|
45
41
|
* Get path to requests.jsonl
|
|
46
42
|
*/
|
|
47
|
-
function getRequestsPath() {
|
|
48
|
-
return path.join(getBasePath(), "requests.jsonl");
|
|
43
|
+
function getRequestsPath(basePath) {
|
|
44
|
+
return path.join(getBasePath(basePath), "requests.jsonl");
|
|
49
45
|
}
|
|
50
46
|
|
|
51
47
|
/**
|
|
52
48
|
* Get path to bodies directory
|
|
53
49
|
*/
|
|
54
|
-
function getBodiesPath() {
|
|
55
|
-
return path.join(getBasePath(), "bodies");
|
|
50
|
+
function getBodiesPath(basePath) {
|
|
51
|
+
return path.join(getBasePath(basePath), "bodies");
|
|
56
52
|
}
|
|
57
53
|
|
|
58
54
|
/**
|
|
@@ -65,16 +61,11 @@ function getMetaPath() {
|
|
|
65
61
|
/**
|
|
66
62
|
* Ensure all required directories exist
|
|
67
63
|
*/
|
|
68
|
-
function ensureDirectories() {
|
|
69
|
-
const base = getBasePath();
|
|
70
|
-
const bodies = getBodiesPath();
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
fs.mkdirSync(base, { recursive: true });
|
|
74
|
-
}
|
|
75
|
-
if (!fs.existsSync(bodies)) {
|
|
76
|
-
fs.mkdirSync(bodies, { recursive: true });
|
|
77
|
-
}
|
|
64
|
+
function ensureDirectories(basePath) {
|
|
65
|
+
const base = getBasePath(basePath);
|
|
66
|
+
const bodies = getBodiesPath(basePath);
|
|
67
|
+
ensurePrivateDir(base, base);
|
|
68
|
+
ensurePrivateDir(bodies, base);
|
|
78
69
|
}
|
|
79
70
|
|
|
80
71
|
/**
|
|
@@ -84,6 +75,7 @@ function readMeta() {
|
|
|
84
75
|
const metaPath = getMetaPath();
|
|
85
76
|
try {
|
|
86
77
|
if (fs.existsSync(metaPath)) {
|
|
78
|
+
assertNotSymlink(metaPath, false);
|
|
87
79
|
return JSON.parse(fs.readFileSync(metaPath, "utf-8"));
|
|
88
80
|
}
|
|
89
81
|
} catch (err) {
|
|
@@ -98,7 +90,7 @@ function readMeta() {
|
|
|
98
90
|
function writeMeta(meta) {
|
|
99
91
|
const metaPath = getMetaPath();
|
|
100
92
|
ensureDirectories();
|
|
101
|
-
|
|
93
|
+
atomicWriteJson(metaPath, meta, { root: getBasePath() });
|
|
102
94
|
}
|
|
103
95
|
|
|
104
96
|
/**
|
|
@@ -124,7 +116,7 @@ function storeBody(content, isRequest = false) {
|
|
|
124
116
|
|
|
125
117
|
// Only write if doesn't exist (dedup)
|
|
126
118
|
if (!fs.existsSync(bodyPath)) {
|
|
127
|
-
|
|
119
|
+
atomicWriteFile(bodyPath, buffer, { root: getBasePath() });
|
|
128
120
|
}
|
|
129
121
|
|
|
130
122
|
return hash;
|
|
@@ -142,6 +134,7 @@ function readBody(hash, isRequest = false) {
|
|
|
142
134
|
|
|
143
135
|
try {
|
|
144
136
|
if (fs.existsSync(bodyPath)) {
|
|
137
|
+
assertNotSymlink(bodyPath, false);
|
|
145
138
|
return fs.readFileSync(bodyPath);
|
|
146
139
|
}
|
|
147
140
|
} catch (err) {
|
|
@@ -186,10 +179,7 @@ async function appendEntry(entry) {
|
|
|
186
179
|
...entry
|
|
187
180
|
};
|
|
188
181
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
// Atomic append using flag 'a'
|
|
192
|
-
fs.appendFileSync(getRequestsPath(), line, { flag: "a" });
|
|
182
|
+
appendPrivateJsonLine(getRequestsPath(), fullEntry, { root: getBasePath() });
|
|
193
183
|
|
|
194
184
|
return fullEntry;
|
|
195
185
|
} finally {
|
|
@@ -202,8 +192,8 @@ async function appendEntry(entry) {
|
|
|
202
192
|
* @param {Object} entry - Network entry to append
|
|
203
193
|
* @returns {Object} The entry with assigned ID
|
|
204
194
|
*/
|
|
205
|
-
function appendEntrySync(entry) {
|
|
206
|
-
ensureDirectories();
|
|
195
|
+
function appendEntrySync(entry, basePath) {
|
|
196
|
+
ensureDirectories(basePath);
|
|
207
197
|
|
|
208
198
|
const id = entry.id || generateId();
|
|
209
199
|
const timestamp = entry.timestamp || Date.now();
|
|
@@ -214,15 +204,14 @@ function appendEntrySync(entry) {
|
|
|
214
204
|
...entry
|
|
215
205
|
};
|
|
216
206
|
|
|
217
|
-
const line = JSON.stringify(fullEntry) + "\n";
|
|
218
|
-
|
|
219
207
|
// Use a simple lock file for synchronous operations
|
|
220
|
-
const lockPath = path.join(getBasePath(), ".lock");
|
|
208
|
+
const lockPath = path.join(getBasePath(basePath), ".lock");
|
|
221
209
|
let lockFd;
|
|
222
210
|
|
|
223
211
|
try {
|
|
224
212
|
// Try to acquire lock
|
|
225
|
-
|
|
213
|
+
assertNotSymlink(lockPath, true);
|
|
214
|
+
lockFd = fs.openSync(lockPath, "wx", 0o600);
|
|
226
215
|
} catch (err) {
|
|
227
216
|
// Lock exists - check if stale and remove, otherwise proceed without lock
|
|
228
217
|
try {
|
|
@@ -230,7 +219,7 @@ function appendEntrySync(entry) {
|
|
|
230
219
|
if (Date.now() - stat.mtimeMs > 5000) {
|
|
231
220
|
fs.unlinkSync(lockPath);
|
|
232
221
|
try {
|
|
233
|
-
lockFd = fs.openSync(lockPath, "wx");
|
|
222
|
+
lockFd = fs.openSync(lockPath, "wx", 0o600);
|
|
234
223
|
} catch (e) {
|
|
235
224
|
// Still can't get lock, proceed without it
|
|
236
225
|
}
|
|
@@ -241,13 +230,13 @@ function appendEntrySync(entry) {
|
|
|
241
230
|
|
|
242
231
|
if (lockFd === undefined) {
|
|
243
232
|
// Proceed without lock as fallback
|
|
244
|
-
|
|
233
|
+
appendPrivateJsonLine(getRequestsPath(basePath), fullEntry, { root: getBasePath(basePath) });
|
|
245
234
|
return fullEntry;
|
|
246
235
|
}
|
|
247
236
|
}
|
|
248
237
|
|
|
249
238
|
try {
|
|
250
|
-
|
|
239
|
+
appendPrivateJsonLine(getRequestsPath(basePath), fullEntry, { root: getBasePath(basePath) });
|
|
251
240
|
} finally {
|
|
252
241
|
if (lockFd !== undefined) {
|
|
253
242
|
fs.closeSync(lockFd);
|
|
@@ -385,6 +374,7 @@ async function readEntries(filters = {}) {
|
|
|
385
374
|
if (!fs.existsSync(requestsPath)) {
|
|
386
375
|
return [];
|
|
387
376
|
}
|
|
377
|
+
assertNotSymlink(requestsPath, false);
|
|
388
378
|
|
|
389
379
|
const { last } = filters;
|
|
390
380
|
const entries = [];
|
|
@@ -433,6 +423,7 @@ function readEntriesSync(filters = {}) {
|
|
|
433
423
|
if (!fs.existsSync(requestsPath)) {
|
|
434
424
|
return [];
|
|
435
425
|
}
|
|
426
|
+
assertNotSymlink(requestsPath, false);
|
|
436
427
|
|
|
437
428
|
const { last } = filters;
|
|
438
429
|
const entries = [];
|
|
@@ -682,10 +673,8 @@ async function cleanup(options = {}) {
|
|
|
682
673
|
|
|
683
674
|
if (deletedEntries > 0 || entries.length === 0) {
|
|
684
675
|
// Atomic write: write to temp then rename
|
|
685
|
-
const tempPath = requestsPath + ".tmp";
|
|
686
676
|
const content = entries.map(e => JSON.stringify(e)).join("\n") + (entries.length > 0 ? "\n" : "");
|
|
687
|
-
|
|
688
|
-
fs.renameSync(tempPath, requestsPath);
|
|
677
|
+
atomicWriteFile(requestsPath, content, { root: getBasePath(), encoding: "utf8" });
|
|
689
678
|
}
|
|
690
679
|
|
|
691
680
|
// 6. Update meta
|
|
@@ -777,10 +766,8 @@ async function clear(options = {}) {
|
|
|
777
766
|
|
|
778
767
|
// Rewrite entries file
|
|
779
768
|
if (deletedEntries > 0) {
|
|
780
|
-
const tempPath = requestsPath + ".tmp";
|
|
781
769
|
const content = remaining.map(e => JSON.stringify(e)).join("\n") + (remaining.length > 0 ? "\n" : "");
|
|
782
|
-
|
|
783
|
-
fs.renameSync(tempPath, requestsPath);
|
|
770
|
+
atomicWriteFile(requestsPath, content, { root: getBasePath(), encoding: "utf8" });
|
|
784
771
|
}
|
|
785
772
|
|
|
786
773
|
return { deletedEntries, deletedBodies };
|
|
@@ -807,9 +794,6 @@ function maybeAutoCleanup() {
|
|
|
807
794
|
}
|
|
808
795
|
}
|
|
809
796
|
|
|
810
|
-
// Run auto-cleanup check on module load
|
|
811
|
-
maybeAutoCleanup();
|
|
812
|
-
|
|
813
797
|
module.exports = {
|
|
814
798
|
// Configuration
|
|
815
799
|
getBasePath,
|
|
@@ -841,10 +825,6 @@ module.exports = {
|
|
|
841
825
|
clear,
|
|
842
826
|
maybeAutoCleanup,
|
|
843
827
|
|
|
844
|
-
// Configuration
|
|
845
|
-
setBasePath,
|
|
846
|
-
getBasePath,
|
|
847
|
-
|
|
848
828
|
// Constants
|
|
849
829
|
DEFAULT_BASE,
|
|
850
830
|
DEFAULT_TTL,
|