runline 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/engine.d.ts +22 -2
- package/dist/core/engine.js +217 -163
- package/dist/plugins/steel/src/browser.js +175 -0
- package/dist/plugins/steel/src/captchas.js +19 -0
- package/dist/plugins/steel/src/credentials.js +38 -0
- package/dist/plugins/steel/src/extensions.js +46 -0
- package/dist/plugins/steel/src/files.js +96 -0
- package/dist/plugins/steel/src/index.js +21 -374
- package/dist/plugins/steel/src/profiles.js +55 -0
- package/dist/plugins/steel/src/sessions.js +119 -0
- package/dist/plugins/steel/src/shared.js +72 -0
- package/dist/tests/helpers/engine-harness.d.ts +1 -0
- package/dist/tests/helpers/engine-harness.js +236 -0
- package/package.json +9 -2
package/dist/core/engine.d.ts
CHANGED
|
@@ -9,6 +9,28 @@ export interface EngineOptions {
|
|
|
9
9
|
timeoutMs?: number;
|
|
10
10
|
memoryLimitBytes?: number;
|
|
11
11
|
}
|
|
12
|
+
/**
|
|
13
|
+
* Whether to arm the host-side RSS watchdog for a worker run.
|
|
14
|
+
*
|
|
15
|
+
* node enforces resourceLimits.maxOldGenerationSizeMb natively, and the
|
|
16
|
+
* watchdog measures *whole-process* RSS — arming it there would risk false
|
|
17
|
+
* kills from unrelated host allocations (e.g. a concurrent execute). bun
|
|
18
|
+
* ignores resourceLimits, so the watchdog is the only memory backstop.
|
|
19
|
+
*/
|
|
20
|
+
export declare function shouldArmRssWatchdog(versions?: Partial<Record<string, string>>): boolean;
|
|
21
|
+
/**
|
|
22
|
+
* Executes agent code in a node:worker_threads worker.
|
|
23
|
+
*
|
|
24
|
+
* The worker is an ergonomic coding surface, not a security sandbox: agent
|
|
25
|
+
* code gets the full host JS runtime (Buffer, crypto, etc.) plus injected
|
|
26
|
+
* action proxies. Isolation properties we do enforce, fail-soft:
|
|
27
|
+
*
|
|
28
|
+
* - timeout: worker.terminate() — interrupts even `while(true){}`
|
|
29
|
+
* - memory: resourceLimits.maxOldGenerationSizeMb (node) + an RSS-delta
|
|
30
|
+
* watchdog fallback; both surface as a clean "Memory limit exceeded" error
|
|
31
|
+
* - crash containment: a dead worker never takes the host process down, and
|
|
32
|
+
* each execute() gets a fresh worker, so the engine stays usable
|
|
33
|
+
*/
|
|
12
34
|
export declare class ExecutionEngine {
|
|
13
35
|
private registry;
|
|
14
36
|
private config;
|
|
@@ -16,6 +38,4 @@ export declare class ExecutionEngine {
|
|
|
16
38
|
execute(code: string, options?: EngineOptions): Promise<ExecuteResult>;
|
|
17
39
|
private invokeAction;
|
|
18
40
|
private resolveConnection;
|
|
19
|
-
private drainAsync;
|
|
20
|
-
private drainJobs;
|
|
21
41
|
}
|
package/dist/core/engine.js
CHANGED
|
@@ -1,7 +1,34 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
|
-
import {
|
|
2
|
+
import { Worker } from "node:worker_threads";
|
|
3
3
|
import { applyEnvOverrides, updateConnectionConfig } from "../config/loader.js";
|
|
4
4
|
import { formatValidationError, helpInputs, isTypedInputSchema, validateTypedInput, } from "../plugin/schema.js";
|
|
5
|
+
/**
|
|
6
|
+
* Whether to arm the host-side RSS watchdog for a worker run.
|
|
7
|
+
*
|
|
8
|
+
* node enforces resourceLimits.maxOldGenerationSizeMb natively, and the
|
|
9
|
+
* watchdog measures *whole-process* RSS — arming it there would risk false
|
|
10
|
+
* kills from unrelated host allocations (e.g. a concurrent execute). bun
|
|
11
|
+
* ignores resourceLimits, so the watchdog is the only memory backstop.
|
|
12
|
+
*/
|
|
13
|
+
export function shouldArmRssWatchdog(versions = process.versions) {
|
|
14
|
+
return Boolean(versions.bun);
|
|
15
|
+
}
|
|
16
|
+
// Extra slack on top of the configured memory limit for the RSS watchdog,
|
|
17
|
+
// since whole-process RSS includes the host's own working set.
|
|
18
|
+
const RSS_WATCHDOG_SLACK_BYTES = 128 * 1024 * 1024;
|
|
19
|
+
/**
|
|
20
|
+
* Executes agent code in a node:worker_threads worker.
|
|
21
|
+
*
|
|
22
|
+
* The worker is an ergonomic coding surface, not a security sandbox: agent
|
|
23
|
+
* code gets the full host JS runtime (Buffer, crypto, etc.) plus injected
|
|
24
|
+
* action proxies. Isolation properties we do enforce, fail-soft:
|
|
25
|
+
*
|
|
26
|
+
* - timeout: worker.terminate() — interrupts even `while(true){}`
|
|
27
|
+
* - memory: resourceLimits.maxOldGenerationSizeMb (node) + an RSS-delta
|
|
28
|
+
* watchdog fallback; both surface as a clean "Memory limit exceeded" error
|
|
29
|
+
* - crash containment: a dead worker never takes the host process down, and
|
|
30
|
+
* each execute() gets a fresh worker, so the engine stays usable
|
|
31
|
+
*/
|
|
5
32
|
export class ExecutionEngine {
|
|
6
33
|
registry;
|
|
7
34
|
config;
|
|
@@ -12,119 +39,121 @@ export class ExecutionEngine {
|
|
|
12
39
|
async execute(code, options) {
|
|
13
40
|
const timeoutMs = options?.timeoutMs ?? this.config.timeoutMs;
|
|
14
41
|
const memoryLimitBytes = options?.memoryLimitBytes ?? this.config.memoryLimitBytes;
|
|
15
|
-
const deadlineMs = Date.now() + timeoutMs;
|
|
16
42
|
const logs = [];
|
|
17
|
-
const
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
runtime.setInterruptHandler(shouldInterruptAfterDeadline(deadlineMs));
|
|
23
|
-
const context = runtime.newContext();
|
|
43
|
+
const plugins = this.registry.listPlugins();
|
|
44
|
+
const source = buildWorkerSource(code, plugins.map((p) => p.name), buildHelpData(plugins));
|
|
45
|
+
return new Promise((resolve) => {
|
|
46
|
+
const memoryLimitMb = Math.max(8, Math.floor(memoryLimitBytes / (1024 * 1024)));
|
|
47
|
+
let worker;
|
|
24
48
|
try {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
const line = context.getString(lineHandle);
|
|
29
|
-
logs.push(`[${level}] ${line}`);
|
|
30
|
-
return context.undefined;
|
|
49
|
+
worker = new Worker(source, {
|
|
50
|
+
eval: true,
|
|
51
|
+
resourceLimits: { maxOldGenerationSizeMb: memoryLimitMb },
|
|
31
52
|
});
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
const handle = context.newString(serialized);
|
|
53
|
-
deferred.resolve(handle);
|
|
54
|
-
handle.dispose();
|
|
55
|
-
}, (err) => {
|
|
56
|
-
if (!deferred.alive)
|
|
57
|
-
return;
|
|
58
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
59
|
-
const handle = context.newError(msg);
|
|
60
|
-
deferred.reject(handle);
|
|
61
|
-
handle.dispose();
|
|
62
|
-
});
|
|
63
|
-
return deferred.handle;
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
resolve({ result: null, error: formatError(err), logs });
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
let settled = false;
|
|
59
|
+
const finish = (r) => {
|
|
60
|
+
if (settled)
|
|
61
|
+
return;
|
|
62
|
+
settled = true;
|
|
63
|
+
clearTimeout(timeoutTimer);
|
|
64
|
+
clearInterval(rssTimer);
|
|
65
|
+
void worker.terminate();
|
|
66
|
+
resolve(r);
|
|
67
|
+
};
|
|
68
|
+
const timeoutTimer = setTimeout(() => {
|
|
69
|
+
finish({
|
|
70
|
+
result: null,
|
|
71
|
+
error: `Execution timed out after ${timeoutMs}ms`,
|
|
72
|
+
logs,
|
|
64
73
|
});
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
const
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
const error = context.dump(evaluated.error);
|
|
74
|
-
evaluated.error.dispose();
|
|
75
|
-
return { result: null, error: formatError(error), logs };
|
|
76
|
-
}
|
|
77
|
-
// Set up promise tracking
|
|
78
|
-
context.setProp(context.global, "__runline_result", evaluated.value);
|
|
79
|
-
evaluated.value.dispose();
|
|
80
|
-
const stateResult = context.evalCode(`(function(p){ var s = { v: void 0, e: void 0, settled: false };
|
|
81
|
-
var fmtErr = function(e){ if (e && typeof e === 'object') { var m = typeof e.message === 'string' ? e.message : ''; var st = typeof e.stack === 'string' ? e.stack : ''; if (m && st) return st.indexOf(m) === -1 ? m + '\\n' + st : st; if (m) return m; if (st) return st; } return String(e); };
|
|
82
|
-
p.then(function(v){ s.v = v; s.settled = true; }, function(e){ s.e = fmtErr(e); s.settled = true; }); return s; })(__runline_result)`);
|
|
83
|
-
if (stateResult.error) {
|
|
84
|
-
const error = context.dump(stateResult.error);
|
|
85
|
-
stateResult.error.dispose();
|
|
86
|
-
return { result: null, error: formatError(error), logs };
|
|
87
|
-
}
|
|
88
|
-
const stateHandle = stateResult.value;
|
|
89
|
-
try {
|
|
90
|
-
await this.drainAsync(context, runtime, pendingDeferreds, deadlineMs, timeoutMs);
|
|
91
|
-
const settled = readProp(context, stateHandle, "settled") === true;
|
|
92
|
-
if (!settled) {
|
|
93
|
-
return {
|
|
74
|
+
}, timeoutMs);
|
|
75
|
+
let rssTimer;
|
|
76
|
+
if (shouldArmRssWatchdog()) {
|
|
77
|
+
const baselineRss = process.memoryUsage().rss;
|
|
78
|
+
rssTimer = setInterval(() => {
|
|
79
|
+
const delta = process.memoryUsage().rss - baselineRss;
|
|
80
|
+
if (delta > memoryLimitBytes + RSS_WATCHDOG_SLACK_BYTES) {
|
|
81
|
+
finish({
|
|
94
82
|
result: null,
|
|
95
|
-
error: `
|
|
83
|
+
error: `Memory limit exceeded (${memoryLimitMb}MB)`,
|
|
96
84
|
logs,
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
|
-
const error = readProp(context, stateHandle, "e");
|
|
100
|
-
if (error !== undefined) {
|
|
101
|
-
return { result: null, error: formatError(error), logs };
|
|
85
|
+
});
|
|
102
86
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
finally {
|
|
106
|
-
stateHandle.dispose();
|
|
107
|
-
}
|
|
87
|
+
}, 100);
|
|
88
|
+
rssTimer.unref?.();
|
|
108
89
|
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
90
|
+
// A reply can race the worker's death; losing it is fine — the run is
|
|
91
|
+
// over either way — but it must never surface as an unhandled
|
|
92
|
+
// rejection in the host.
|
|
93
|
+
const reply = (message) => {
|
|
94
|
+
if (settled)
|
|
95
|
+
return;
|
|
96
|
+
try {
|
|
97
|
+
worker.postMessage(message);
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
// worker already gone
|
|
113
101
|
}
|
|
114
|
-
pendingDeferreds.clear();
|
|
115
|
-
context.dispose();
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
catch (err) {
|
|
119
|
-
return {
|
|
120
|
-
result: null,
|
|
121
|
-
error: formatError(err),
|
|
122
|
-
logs,
|
|
123
102
|
};
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
103
|
+
worker.on("message", (msg) => {
|
|
104
|
+
if (settled)
|
|
105
|
+
return;
|
|
106
|
+
if (msg.t === "log") {
|
|
107
|
+
logs.push(`[${msg.level}] ${msg.line}`);
|
|
108
|
+
}
|
|
109
|
+
else if (msg.t === "invoke") {
|
|
110
|
+
this.invokeAction(msg.path, msg.args).then((value) => {
|
|
111
|
+
let serialized;
|
|
112
|
+
try {
|
|
113
|
+
serialized = toPlainJson(value);
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
reply({
|
|
117
|
+
t: "result",
|
|
118
|
+
id: msg.id,
|
|
119
|
+
ok: false,
|
|
120
|
+
error: `Action result not JSON-serializable: ${err instanceof Error ? err.message : String(err)}`,
|
|
121
|
+
});
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
reply({ t: "result", id: msg.id, ok: true, value: serialized });
|
|
125
|
+
}, (err) => {
|
|
126
|
+
reply({
|
|
127
|
+
t: "result",
|
|
128
|
+
id: msg.id,
|
|
129
|
+
ok: false,
|
|
130
|
+
error: err instanceof Error ? err.message : String(err),
|
|
131
|
+
});
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
else if (msg.t === "done") {
|
|
135
|
+
finish(msg.ok
|
|
136
|
+
? { result: msg.result, logs }
|
|
137
|
+
: { result: null, error: msg.error ?? "Unknown error", logs });
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
worker.on("error", (err) => {
|
|
141
|
+
finish({
|
|
142
|
+
result: null,
|
|
143
|
+
error: err.code === "ERR_WORKER_OUT_OF_MEMORY"
|
|
144
|
+
? `Memory limit exceeded (${memoryLimitMb}MB)`
|
|
145
|
+
: formatError(err),
|
|
146
|
+
logs,
|
|
147
|
+
});
|
|
148
|
+
});
|
|
149
|
+
worker.on("exit", (exitCode) => {
|
|
150
|
+
finish({
|
|
151
|
+
result: null,
|
|
152
|
+
error: `Worker exited unexpectedly (code ${exitCode})`,
|
|
153
|
+
logs,
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
});
|
|
128
157
|
}
|
|
129
158
|
async invokeAction(path, args) {
|
|
130
159
|
const resolved = this.registry.resolveAction(path);
|
|
@@ -164,53 +193,17 @@ export class ExecutionEngine {
|
|
|
164
193
|
};
|
|
165
194
|
return applyEnvOverrides(base, plugin.connectionConfigSchema);
|
|
166
195
|
}
|
|
167
|
-
async drainAsync(context, runtime, pendingDeferreds, deadlineMs, timeoutMs) {
|
|
168
|
-
this.drainJobs(context, runtime, deadlineMs, timeoutMs);
|
|
169
|
-
while (pendingDeferreds.size > 0) {
|
|
170
|
-
const remainingMs = deadlineMs - Date.now();
|
|
171
|
-
if (remainingMs <= 0) {
|
|
172
|
-
throw new Error(`Execution timed out after ${timeoutMs}ms`);
|
|
173
|
-
}
|
|
174
|
-
let timer;
|
|
175
|
-
try {
|
|
176
|
-
await Promise.race([
|
|
177
|
-
Promise.race([...pendingDeferreds].map((d) => d.settled)),
|
|
178
|
-
new Promise((_, reject) => {
|
|
179
|
-
timer = setTimeout(() => reject(new Error(`Execution timed out after ${timeoutMs}ms`)), remainingMs);
|
|
180
|
-
}),
|
|
181
|
-
]);
|
|
182
|
-
}
|
|
183
|
-
finally {
|
|
184
|
-
if (timer)
|
|
185
|
-
clearTimeout(timer);
|
|
186
|
-
}
|
|
187
|
-
this.drainJobs(context, runtime, deadlineMs, timeoutMs);
|
|
188
|
-
}
|
|
189
|
-
this.drainJobs(context, runtime, deadlineMs, timeoutMs);
|
|
190
|
-
}
|
|
191
|
-
drainJobs(context, runtime, deadlineMs, timeoutMs) {
|
|
192
|
-
while (runtime.hasPendingJob()) {
|
|
193
|
-
if (Date.now() >= deadlineMs) {
|
|
194
|
-
throw new Error(`Execution timed out after ${timeoutMs}ms`);
|
|
195
|
-
}
|
|
196
|
-
const pending = runtime.executePendingJobs();
|
|
197
|
-
if (pending.error) {
|
|
198
|
-
const error = context.dump(pending.error);
|
|
199
|
-
pending.error.dispose();
|
|
200
|
-
throw error instanceof Error ? error : new Error(String(error));
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
196
|
}
|
|
205
197
|
// ── Helpers ──────────────────────────────────────────────
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
198
|
+
/**
|
|
199
|
+
* JSON round-trip to (a) guarantee structured-clone compatibility and
|
|
200
|
+
* (b) preserve the previous engine's value semantics, where every action
|
|
201
|
+
* result crossed a JSON boundary (Dates → ISO strings, no Maps, etc.).
|
|
202
|
+
*/
|
|
203
|
+
function toPlainJson(value) {
|
|
204
|
+
if (value === undefined)
|
|
205
|
+
return undefined;
|
|
206
|
+
return JSON.parse(JSON.stringify(value));
|
|
214
207
|
}
|
|
215
208
|
function formatError(cause) {
|
|
216
209
|
if (cause instanceof Error)
|
|
@@ -224,13 +217,13 @@ function formatError(cause) {
|
|
|
224
217
|
return String(cause);
|
|
225
218
|
}
|
|
226
219
|
// MiniSearch UMD bundle, vendored at the package root and inlined into the
|
|
227
|
-
//
|
|
228
|
-
//
|
|
220
|
+
// worker source. Inside the worker it is evaluated in a local scope with a
|
|
221
|
+
// fresh `module`/`exports` pair so the UMD takes its CommonJS branch.
|
|
229
222
|
//
|
|
230
223
|
// `../../vendor/...` resolves identically from src/core/engine.ts (dev) and
|
|
231
224
|
// dist/core/engine.js (published) because tsc preserves the `core/` subdir.
|
|
232
225
|
// See vendor/README.md for the upgrade procedure.
|
|
233
|
-
const
|
|
226
|
+
const minisearchSource = readFileSync(new URL("../../vendor/minisearch.umd.js", import.meta.url), "utf8");
|
|
234
227
|
function buildHelpData(plugins) {
|
|
235
228
|
const data = {};
|
|
236
229
|
for (const p of plugins) {
|
|
@@ -242,7 +235,7 @@ function buildHelpData(plugins) {
|
|
|
242
235
|
}
|
|
243
236
|
return data;
|
|
244
237
|
}
|
|
245
|
-
function
|
|
238
|
+
function buildWorkerSource(code, pluginNames = [], helpData = {}) {
|
|
246
239
|
const trimmed = code.trim();
|
|
247
240
|
const looksLikeArrow = (trimmed.startsWith("async") || trimmed.startsWith("(")) &&
|
|
248
241
|
trimmed.includes("=>");
|
|
@@ -250,18 +243,68 @@ function buildExecutionSource(code, pluginNames = [], helpData = {}) {
|
|
|
250
243
|
? `const __fn = (${trimmed});\nif (typeof __fn !== 'function') throw new Error('Code must evaluate to a function');\nreturn await __fn();`
|
|
251
244
|
: code;
|
|
252
245
|
const wrapped = `"use strict";
|
|
253
|
-
const
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
246
|
+
const { parentPort: __port } = require("node:worker_threads");
|
|
247
|
+
|
|
248
|
+
// process.exit would be runtime-divergent here: node kills the worker
|
|
249
|
+
// synchronously, bun lets the completion message race the exit and can
|
|
250
|
+
// report a silent undefined success. Make it a regular, catchable error.
|
|
251
|
+
process.exit = (code) => {
|
|
252
|
+
throw new Error('process.exit(' + (code ?? 0) + ') is not available in the runline sandbox; return a value instead');
|
|
253
|
+
};
|
|
254
|
+
|
|
255
|
+
// ── host bridge ──
|
|
256
|
+
let __seq = 0;
|
|
257
|
+
const __pending = new Map();
|
|
258
|
+
__port.on("message", (m) => {
|
|
259
|
+
if (!m || m.t !== "result") return;
|
|
260
|
+
const p = __pending.get(m.id);
|
|
261
|
+
if (!p) return;
|
|
262
|
+
__pending.delete(m.id);
|
|
263
|
+
if (m.ok) p.resolve(m.value);
|
|
264
|
+
else p.reject(new Error(m.error));
|
|
265
|
+
});
|
|
266
|
+
const __invoke = (path, args) => new Promise((resolve, reject) => {
|
|
267
|
+
const id = ++__seq;
|
|
268
|
+
__pending.set(id, { resolve, reject });
|
|
269
|
+
try {
|
|
270
|
+
__port.postMessage({ t: "invoke", id, path, args });
|
|
271
|
+
} catch (e) {
|
|
272
|
+
__pending.delete(id);
|
|
273
|
+
reject(e);
|
|
274
|
+
}
|
|
275
|
+
});
|
|
276
|
+
const __log = (level, line) => {
|
|
277
|
+
try { __port.postMessage({ t: "log", level, line }); } catch {}
|
|
278
|
+
};
|
|
257
279
|
|
|
258
280
|
const __fmt = (v) => {
|
|
259
281
|
if (typeof v === 'string') return v;
|
|
260
282
|
try { return JSON.stringify(v); } catch { return String(v); }
|
|
261
283
|
};
|
|
262
284
|
|
|
263
|
-
|
|
264
|
-
|
|
285
|
+
const __fmtErr = (e) => {
|
|
286
|
+
if (e && typeof e === 'object') {
|
|
287
|
+
const m = typeof e.message === 'string' ? e.message : '';
|
|
288
|
+
const st = typeof e.stack === 'string' ? e.stack : '';
|
|
289
|
+
if (m && st) return st.indexOf(m) === -1 ? m + '\\n' + st : st;
|
|
290
|
+
if (m) return m;
|
|
291
|
+
if (st) return st;
|
|
292
|
+
}
|
|
293
|
+
return String(e);
|
|
294
|
+
};
|
|
295
|
+
|
|
296
|
+
// JSON round-trip mirrors the host's toPlainJson: keeps results
|
|
297
|
+
// structured-clone-safe and preserves JSON value semantics.
|
|
298
|
+
const __toJson = (v) => v === undefined ? undefined : JSON.parse(JSON.stringify(v));
|
|
299
|
+
|
|
300
|
+
// Inlined MiniSearch UMD, evaluated with a local module/exports so the UMD
|
|
301
|
+
// takes its CommonJS branch regardless of the worker's module scope.
|
|
302
|
+
const MiniSearch = (function () {
|
|
303
|
+
const module = { exports: {} };
|
|
304
|
+
const exports = module.exports;
|
|
305
|
+
${minisearchSource}
|
|
306
|
+
return module.exports;
|
|
307
|
+
})();
|
|
265
308
|
|
|
266
309
|
const __help = ${JSON.stringify(helpData)};
|
|
267
310
|
|
|
@@ -273,8 +316,7 @@ const __makeProxy = (path = []) => new Proxy(() => undefined, {
|
|
|
273
316
|
apply(_t, _this, args) {
|
|
274
317
|
const p = path.join('.');
|
|
275
318
|
if (!p) throw new Error('Action path missing');
|
|
276
|
-
return
|
|
277
|
-
.then((raw) => raw === undefined ? undefined : JSON.parse(raw));
|
|
319
|
+
return __invoke(p, args[0]);
|
|
278
320
|
},
|
|
279
321
|
});
|
|
280
322
|
|
|
@@ -296,7 +338,7 @@ const __formatSignature = (plugin, entry) => {
|
|
|
296
338
|
return plugin + '.' + entry.action + (fields ? '({ ' + fields + ' })' : '()');
|
|
297
339
|
};
|
|
298
340
|
|
|
299
|
-
// Build a MiniSearch index over every action path. Indexed at
|
|
341
|
+
// Build a MiniSearch index over every action path. Indexed at worker
|
|
300
342
|
// startup, queried by actions.find().
|
|
301
343
|
const __search = (() => {
|
|
302
344
|
const docs = [];
|
|
@@ -416,6 +458,18 @@ const fetch = () => { throw new Error('fetch is disabled in runline sandbox'); }
|
|
|
416
458
|
|
|
417
459
|
(async () => {
|
|
418
460
|
${body}
|
|
419
|
-
})()
|
|
461
|
+
})().then(
|
|
462
|
+
(v) => {
|
|
463
|
+
try {
|
|
464
|
+
__port.postMessage({ t: "done", ok: true, result: __toJson(v) });
|
|
465
|
+
} catch (e) {
|
|
466
|
+
__port.postMessage({ t: "done", ok: false, error: __fmtErr(e) });
|
|
467
|
+
}
|
|
468
|
+
},
|
|
469
|
+
(e) => {
|
|
470
|
+
__port.postMessage({ t: "done", ok: false, error: __fmtErr(e) });
|
|
471
|
+
},
|
|
472
|
+
);
|
|
473
|
+
`;
|
|
420
474
|
return wrapped;
|
|
421
475
|
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import * as t from "typebox";
|
|
2
|
+
import { SESSION_OPTIONS_SCHEMA, api, apiKey, compactRecord } from "./shared.js";
|
|
3
|
+
const SCRAPE_SCHEMA = {
|
|
4
|
+
url: t.String({ description: "URL to scrape" }),
|
|
5
|
+
format: t.Optional(t.Array(t.String(), { description: "Formats: html, cleaned_html, markdown, readability" })),
|
|
6
|
+
delay: t.Optional(t.Number({ description: "Milliseconds to wait after navigation" })),
|
|
7
|
+
useProxy: t.Optional(t.Any({ description: "true or proxy config" })),
|
|
8
|
+
screenshot: t.Optional(t.Boolean({ description: "Also capture a screenshot URL" })),
|
|
9
|
+
pdf: t.Optional(t.Boolean({ description: "Also capture a PDF URL" })),
|
|
10
|
+
};
|
|
11
|
+
const SCREENSHOT_SCHEMA = {
|
|
12
|
+
url: t.String({ description: "URL to screenshot" }),
|
|
13
|
+
fullPage: t.Optional(t.Boolean({ description: "Capture full scrollable page" })),
|
|
14
|
+
delay: t.Optional(t.Number({ description: "Milliseconds to wait after navigation" })),
|
|
15
|
+
useProxy: t.Optional(t.Any({ description: "true or proxy config" })),
|
|
16
|
+
};
|
|
17
|
+
async function scrape(input, ctx) {
|
|
18
|
+
return api(ctx, "/v1/scrape", { method: "POST", body: compactRecord(input) });
|
|
19
|
+
}
|
|
20
|
+
async function screenshot(input, ctx) {
|
|
21
|
+
return api(ctx, "/v1/screenshot", { method: "POST", body: compactRecord(input) });
|
|
22
|
+
}
|
|
23
|
+
async function connectMiniCdp(cdpUrl) {
|
|
24
|
+
const ws = new WebSocket(cdpUrl);
|
|
25
|
+
let nextId = 0;
|
|
26
|
+
const pending = new Map();
|
|
27
|
+
await new Promise((resolve, reject) => {
|
|
28
|
+
const timer = setTimeout(() => reject(new Error("CDP websocket connection timed out")), 30000);
|
|
29
|
+
ws.addEventListener("open", () => { clearTimeout(timer); resolve(); }, { once: true });
|
|
30
|
+
ws.addEventListener("error", () => { clearTimeout(timer); reject(new Error("CDP websocket connection failed")); }, { once: true });
|
|
31
|
+
});
|
|
32
|
+
ws.addEventListener("message", (event) => {
|
|
33
|
+
let message;
|
|
34
|
+
try {
|
|
35
|
+
message = JSON.parse(String(event.data));
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
if (typeof message.id !== "number")
|
|
41
|
+
return;
|
|
42
|
+
const wait = pending.get(message.id);
|
|
43
|
+
if (!wait)
|
|
44
|
+
return;
|
|
45
|
+
pending.delete(message.id);
|
|
46
|
+
if (message.error)
|
|
47
|
+
wait.reject(new Error(JSON.stringify(message.error)));
|
|
48
|
+
else
|
|
49
|
+
wait.resolve(message.result);
|
|
50
|
+
});
|
|
51
|
+
const send = (method, params = {}, sessionId) => new Promise((resolve, reject) => {
|
|
52
|
+
const id = ++nextId;
|
|
53
|
+
pending.set(id, { resolve, reject });
|
|
54
|
+
ws.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }));
|
|
55
|
+
setTimeout(() => {
|
|
56
|
+
if (pending.delete(id))
|
|
57
|
+
reject(new Error(`CDP ${method} timed out`));
|
|
58
|
+
}, 30000);
|
|
59
|
+
});
|
|
60
|
+
const targets = await send("Target.getTargets");
|
|
61
|
+
const target = targets.targetInfos?.find((info) => info.type === "page") ?? targets.targetInfos?.[0];
|
|
62
|
+
if (!target)
|
|
63
|
+
throw new Error("Steel CDP session has no browser target");
|
|
64
|
+
const attached = await send("Target.attachToTarget", { targetId: target.targetId, flatten: true });
|
|
65
|
+
const sid = attached.sessionId;
|
|
66
|
+
const evaluate = async (expression) => {
|
|
67
|
+
const result = await send("Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true }, sid);
|
|
68
|
+
return result.result?.value;
|
|
69
|
+
};
|
|
70
|
+
const page = {
|
|
71
|
+
async goto(url, _options) {
|
|
72
|
+
await send("Page.navigate", { url }, sid);
|
|
73
|
+
await new Promise((resolve) => setTimeout(resolve, 1500));
|
|
74
|
+
return null;
|
|
75
|
+
},
|
|
76
|
+
title: () => evaluate("document.title"),
|
|
77
|
+
url: () => evaluate("location.href"),
|
|
78
|
+
text: () => evaluate("document.body?.innerText ?? ''"),
|
|
79
|
+
html: () => evaluate("document.documentElement?.outerHTML ?? ''"),
|
|
80
|
+
evaluate: (expression) => evaluate(`(${expression})()`),
|
|
81
|
+
};
|
|
82
|
+
return { page, browser: { close: () => ws.close() }, context: {}, close: () => ws.close() };
|
|
83
|
+
}
|
|
84
|
+
export function registerBrowserActions(rl) {
|
|
85
|
+
rl.registerAction("scrape", {
|
|
86
|
+
description: "One-shot Steel scrape. Loads a URL and returns requested formats such as markdown, html, cleaned_html, or readability.",
|
|
87
|
+
inputSchema: t.Object(SCRAPE_SCHEMA),
|
|
88
|
+
execute: scrape,
|
|
89
|
+
});
|
|
90
|
+
rl.registerAction("browser.scrape", {
|
|
91
|
+
description: "Backward-compatible alias for scrape.",
|
|
92
|
+
inputSchema: t.Object(SCRAPE_SCHEMA),
|
|
93
|
+
execute: scrape,
|
|
94
|
+
});
|
|
95
|
+
rl.registerAction("screenshot", {
|
|
96
|
+
description: "One-shot Steel screenshot. Returns a hosted PNG URL.",
|
|
97
|
+
inputSchema: t.Object(SCREENSHOT_SCHEMA),
|
|
98
|
+
execute: screenshot,
|
|
99
|
+
});
|
|
100
|
+
rl.registerAction("browser.screenshot", {
|
|
101
|
+
description: "Backward-compatible alias for screenshot.",
|
|
102
|
+
inputSchema: t.Object(SCREENSHOT_SCHEMA),
|
|
103
|
+
execute: screenshot,
|
|
104
|
+
});
|
|
105
|
+
rl.registerAction("browser.extract", {
|
|
106
|
+
description: "Fetch a page through Steel scrape and return selected content fields. Use selectors with browser.run for DOM-specific extraction.",
|
|
107
|
+
inputSchema: t.Object({
|
|
108
|
+
url: t.String({ description: "URL to scrape" }),
|
|
109
|
+
format: t.Optional(t.Array(t.String(), { description: "Formats to request; defaults to markdown and html" })),
|
|
110
|
+
delay: t.Optional(t.Number({ description: "Milliseconds to wait after navigation" })),
|
|
111
|
+
useProxy: t.Optional(t.Any({ description: "true or proxy config" })),
|
|
112
|
+
}),
|
|
113
|
+
async execute(input, ctx) {
|
|
114
|
+
const body = { format: ["markdown", "html"], ...input };
|
|
115
|
+
return api(ctx, "/v1/scrape", { method: "POST", body: compactRecord(body) });
|
|
116
|
+
},
|
|
117
|
+
});
|
|
118
|
+
rl.registerAction("pdf", {
|
|
119
|
+
description: "One-shot Steel PDF capture. Returns a hosted PDF URL.",
|
|
120
|
+
inputSchema: t.Object({
|
|
121
|
+
url: t.String({ description: "URL to render as PDF" }),
|
|
122
|
+
delay: t.Optional(t.Number({ description: "Milliseconds to wait after navigation" })),
|
|
123
|
+
useProxy: t.Optional(t.Any({ description: "true or proxy config" })),
|
|
124
|
+
}),
|
|
125
|
+
async execute(input, ctx) {
|
|
126
|
+
return api(ctx, "/v1/pdf", { method: "POST", body: compactRecord(input) });
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
rl.registerAction("browser.run", {
|
|
130
|
+
description: "Create a Steel session, connect with Playwright over CDP, run an async JavaScript script, then release by default. The script receives { page, browser, context, session }. Requires the host app to have playwright installed.",
|
|
131
|
+
inputSchema: t.Object({
|
|
132
|
+
script: t.String({ description: "Async JavaScript body. Example: await page.goto('https://example.com'); return { title: await page.title() };" }),
|
|
133
|
+
release: t.Optional(t.Boolean({ description: "Release the Steel session after the script finishes (default true)" })),
|
|
134
|
+
...SESSION_OPTIONS_SCHEMA,
|
|
135
|
+
}),
|
|
136
|
+
async execute(input, ctx) {
|
|
137
|
+
const { script, release, ...sessionOptions } = input;
|
|
138
|
+
let playwright;
|
|
139
|
+
try {
|
|
140
|
+
playwright = await import("playwright");
|
|
141
|
+
}
|
|
142
|
+
catch (_error) {
|
|
143
|
+
throw new Error("steel.browser.run requires the host project to install playwright. Install playwright or use session.create + session.cdpUrl instead.");
|
|
144
|
+
}
|
|
145
|
+
const session = await api(ctx, "/v1/sessions", { method: "POST", body: compactRecord(sessionOptions) });
|
|
146
|
+
const cdpUrl = `wss://connect.steel.dev?apiKey=${encodeURIComponent(apiKey(ctx))}&sessionId=${encodeURIComponent(String(session.id))}`;
|
|
147
|
+
let browser;
|
|
148
|
+
try {
|
|
149
|
+
let context;
|
|
150
|
+
let page;
|
|
151
|
+
try {
|
|
152
|
+
const playwrightBrowser = await playwright.chromium.connectOverCDP(cdpUrl, { timeout: 30000 });
|
|
153
|
+
browser = playwrightBrowser;
|
|
154
|
+
context = playwrightBrowser.contexts()[0] ?? await playwrightBrowser.newContext();
|
|
155
|
+
page = context.pages()[0] ?? await context.newPage();
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
const mini = await connectMiniCdp(cdpUrl);
|
|
159
|
+
browser = mini.browser;
|
|
160
|
+
context = mini.context;
|
|
161
|
+
page = mini.page;
|
|
162
|
+
}
|
|
163
|
+
const fn = new Function("page", "browser", "context", "session", `return (async () => {\n${script}\n})();`);
|
|
164
|
+
const result = await fn(page, browser, context, session);
|
|
165
|
+
return { session, result };
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
await browser?.close()?.catch?.(() => { });
|
|
169
|
+
if (release !== false && session.id) {
|
|
170
|
+
await api(ctx, `/v1/sessions/${encodeURIComponent(String(session.id))}/release`, { method: "POST" }).catch(() => { });
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
}
|