zenrows 2.0.2 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/assets/logo/dark.svg +3 -0
- package/.github/assets/logo/light.svg +3 -0
- package/README.md +123 -6
- package/dist/index.cjs +1378 -4
- package/dist/index.d.cts +771 -2
- package/dist/index.d.ts +771 -2
- package/dist/index.js +1343 -3
- package/package.json +8 -3
- package/vitest.config.ts +4 -0
package/dist/index.js
CHANGED
|
@@ -41,10 +41,11 @@ var package_default = {
|
|
|
41
41
|
"@biomejs/biome": "1.8.1",
|
|
42
42
|
"@types/async-retry": "^1.4.9",
|
|
43
43
|
"@types/node": "^20.19.39",
|
|
44
|
+
"@vitest/coverage-v8": "1.6.1",
|
|
44
45
|
msw: "^2.13.4",
|
|
45
46
|
tsup: "^8.5.1",
|
|
46
47
|
typescript: "^5.9.3",
|
|
47
|
-
vitest: "^
|
|
48
|
+
vitest: "^3.2.6"
|
|
48
49
|
},
|
|
49
50
|
dependencies: {
|
|
50
51
|
fastq: "^1.20.1",
|
|
@@ -52,16 +53,1283 @@ var package_default = {
|
|
|
52
53
|
}
|
|
53
54
|
};
|
|
54
55
|
|
|
56
|
+
// src/batch/download.ts
|
|
57
|
+
import { createWriteStream } from "fs";
|
|
58
|
+
import { mkdir } from "fs/promises";
|
|
59
|
+
import { dirname, join } from "path";
|
|
60
|
+
import { Readable } from "stream";
|
|
61
|
+
import { pipeline } from "stream/promises";
|
|
62
|
+
var DEFAULT_MAX_FILES = 1e5;
|
|
63
|
+
var DEFAULT_MAX_BYTES_PER_FILE = 50 * 1024 * 1024;
|
|
64
|
+
var DEFAULT_MAX_COUNT_IN_MEMORY = 1e4;
|
|
65
|
+
var DEFAULT_MAX_TOTAL_BYTES_IN_MEMORY = 500 * 1024 * 1024;
|
|
66
|
+
async function fetchResultBody(task, maxBytes) {
|
|
67
|
+
if (!task.result_url) {
|
|
68
|
+
throw new Error(`task ${task.task_id} has no result_url (not a successful task?)`);
|
|
69
|
+
}
|
|
70
|
+
const response = await fetch(task.result_url);
|
|
71
|
+
if (!response.ok) {
|
|
72
|
+
throw new Error(`downloading task ${task.task_id} failed with status ${response.status}`);
|
|
73
|
+
}
|
|
74
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
75
|
+
if (maxBytes !== void 0 && buffer.byteLength > maxBytes) {
|
|
76
|
+
throw new Error(
|
|
77
|
+
`task ${task.task_id}'s body (${buffer.byteLength} bytes) exceeds maxBytesPerFile (${maxBytes})`
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
return buffer;
|
|
81
|
+
}
|
|
82
|
+
async function runPool(items, concurrency, worker) {
|
|
83
|
+
let cursor = 0;
|
|
84
|
+
const errors = [];
|
|
85
|
+
async function next() {
|
|
86
|
+
while (cursor < items.length) {
|
|
87
|
+
const index = cursor++;
|
|
88
|
+
try {
|
|
89
|
+
await worker(items[index], index);
|
|
90
|
+
} catch (error) {
|
|
91
|
+
errors.push(error);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const workers = Array.from({ length: Math.max(1, concurrency) }, () => next());
|
|
96
|
+
await Promise.all(workers);
|
|
97
|
+
if (errors.length) {
|
|
98
|
+
throw errors[0];
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
function defaultFileName(task, useExternalId) {
|
|
102
|
+
const base = useExternalId && task.external_id ? task.external_id : task.task_id;
|
|
103
|
+
return base.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
104
|
+
}
|
|
105
|
+
async function downloadToDir(results, targetDir, {
|
|
106
|
+
nameFn,
|
|
107
|
+
useExternalId = false,
|
|
108
|
+
concurrency = 1,
|
|
109
|
+
maxFiles = DEFAULT_MAX_FILES,
|
|
110
|
+
maxBytesPerFile = DEFAULT_MAX_BYTES_PER_FILE
|
|
111
|
+
} = {}) {
|
|
112
|
+
await mkdir(targetDir, { recursive: true });
|
|
113
|
+
const tasks = [];
|
|
114
|
+
for await (const task of results) {
|
|
115
|
+
tasks.push(task);
|
|
116
|
+
if (tasks.length > maxFiles) {
|
|
117
|
+
throw new Error(`result count exceeds maxFiles (${maxFiles})`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
await runPool(tasks, concurrency, async (task) => {
|
|
121
|
+
const body = await fetchResultBody(task, maxBytesPerFile);
|
|
122
|
+
const fileName = nameFn ? nameFn(task) : defaultFileName(task, useExternalId);
|
|
123
|
+
const filePath = join(targetDir, fileName);
|
|
124
|
+
await mkdir(dirname(filePath), { recursive: true });
|
|
125
|
+
await pipeline(Readable.from(body), createWriteStream(filePath));
|
|
126
|
+
});
|
|
127
|
+
return tasks.length;
|
|
128
|
+
}
|
|
129
|
+
async function downloadToMemory(results, {
|
|
130
|
+
concurrency = 1,
|
|
131
|
+
maxCount = DEFAULT_MAX_COUNT_IN_MEMORY,
|
|
132
|
+
maxTotalBytes = DEFAULT_MAX_TOTAL_BYTES_IN_MEMORY,
|
|
133
|
+
maxBytesPerFile = DEFAULT_MAX_BYTES_PER_FILE
|
|
134
|
+
} = {}) {
|
|
135
|
+
const tasks = [];
|
|
136
|
+
for await (const task of results) {
|
|
137
|
+
tasks.push(task);
|
|
138
|
+
if (tasks.length > maxCount) {
|
|
139
|
+
throw new Error(`result count exceeds maxCount (${maxCount})`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
const downloaded = new Array(tasks.length);
|
|
143
|
+
let totalBytes = 0;
|
|
144
|
+
await runPool(tasks, concurrency, async (task, index) => {
|
|
145
|
+
const body = await fetchResultBody(task, maxBytesPerFile);
|
|
146
|
+
totalBytes += body.byteLength;
|
|
147
|
+
if (totalBytes > maxTotalBytes) {
|
|
148
|
+
throw new Error(`total downloaded bytes exceeds maxTotalBytes (${maxTotalBytes})`);
|
|
149
|
+
}
|
|
150
|
+
downloaded[index] = { taskId: task.task_id, externalId: task.external_id, url: task.url, body };
|
|
151
|
+
});
|
|
152
|
+
return downloaded;
|
|
153
|
+
}
|
|
154
|
+
async function downloadTaskToFile(task, target) {
|
|
155
|
+
const body = await fetchResultBody(task);
|
|
156
|
+
await mkdir(dirname(target), { recursive: true });
|
|
157
|
+
await pipeline(Readable.from(body), createWriteStream(target));
|
|
158
|
+
}
|
|
159
|
+
async function downloadTaskToMemory(task) {
|
|
160
|
+
return fetchResultBody(task);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// src/batch/errors.ts
|
|
164
|
+
var ZenRowsBatchError = class extends Error {
|
|
165
|
+
status;
|
|
166
|
+
problem;
|
|
167
|
+
code;
|
|
168
|
+
extras;
|
|
169
|
+
constructor(status, problem, extras) {
|
|
170
|
+
const message = problem ? `${status} ${problem.title ?? "Error"}: ${problem.detail ?? problem.code ?? "unknown"}` : `${status} (no problem body)`;
|
|
171
|
+
super(message);
|
|
172
|
+
this.name = "ZenRowsBatchError";
|
|
173
|
+
this.status = status;
|
|
174
|
+
this.problem = problem;
|
|
175
|
+
this.code = problem?.code ?? "internal";
|
|
176
|
+
this.extras = extras;
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
var PROBLEM_STANDARD_KEYS = /* @__PURE__ */ new Set(["type", "title", "status", "code", "detail", "instance"]);
|
|
180
|
+
async function parseProblem(response) {
|
|
181
|
+
let body;
|
|
182
|
+
try {
|
|
183
|
+
body = await response.json();
|
|
184
|
+
} catch {
|
|
185
|
+
return { problem: void 0, extras: void 0 };
|
|
186
|
+
}
|
|
187
|
+
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
188
|
+
return { problem: void 0, extras: void 0 };
|
|
189
|
+
}
|
|
190
|
+
const record = body;
|
|
191
|
+
const extras = {};
|
|
192
|
+
for (const [key, value] of Object.entries(record)) {
|
|
193
|
+
if (!PROBLEM_STANDARD_KEYS.has(key)) {
|
|
194
|
+
extras[key] = value;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
const problem = {
|
|
198
|
+
type: typeof record.type === "string" ? record.type : "about:blank",
|
|
199
|
+
title: typeof record.title === "string" ? record.title : "Error",
|
|
200
|
+
status: typeof record.status === "number" ? record.status : response.status,
|
|
201
|
+
code: typeof record.code === "string" ? record.code : "internal",
|
|
202
|
+
detail: typeof record.detail === "string" ? record.detail : void 0,
|
|
203
|
+
instance: typeof record.instance === "string" ? record.instance : void 0
|
|
204
|
+
};
|
|
205
|
+
return { problem, extras: Object.keys(extras).length ? extras : void 0 };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// src/batch/estimate.ts
|
|
209
|
+
var BASE_CREDITS = 1;
|
|
210
|
+
var JS_CREDITS = 5;
|
|
211
|
+
var PREMIUM_PROXY_CREDITS = 10;
|
|
212
|
+
var JS_AND_PROXY_CREDITS = 25;
|
|
213
|
+
var AUTO_MIN_CREDITS = 1;
|
|
214
|
+
var AUTO_MAX_CREDITS = 25;
|
|
215
|
+
var TIER_ORDER = [
|
|
216
|
+
"base",
|
|
217
|
+
"js_render",
|
|
218
|
+
"premium_proxy",
|
|
219
|
+
"js_render+premium_proxy",
|
|
220
|
+
"auto"
|
|
221
|
+
];
|
|
222
|
+
var TRUTHY_STRINGS = /* @__PURE__ */ new Set(["true", "1", "yes", "on"]);
|
|
223
|
+
function truthy(value) {
|
|
224
|
+
if (typeof value === "boolean") return value;
|
|
225
|
+
if (typeof value === "number") return value !== 0;
|
|
226
|
+
if (typeof value === "string") return TRUTHY_STRINGS.has(value.trim().toLowerCase());
|
|
227
|
+
return false;
|
|
228
|
+
}
|
|
229
|
+
function isAuto(params) {
|
|
230
|
+
return String(params.mode ?? "").trim().toLowerCase() === "auto";
|
|
231
|
+
}
|
|
232
|
+
function costForParams(params) {
|
|
233
|
+
if (isAuto(params)) {
|
|
234
|
+
return { tier: "auto", min: AUTO_MIN_CREDITS, max: AUTO_MAX_CREDITS };
|
|
235
|
+
}
|
|
236
|
+
const js = truthy(params.js_render);
|
|
237
|
+
const px = truthy(params.premium_proxy);
|
|
238
|
+
if (js && px) {
|
|
239
|
+
return {
|
|
240
|
+
tier: "js_render+premium_proxy",
|
|
241
|
+
min: JS_AND_PROXY_CREDITS,
|
|
242
|
+
max: JS_AND_PROXY_CREDITS
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
if (px) {
|
|
246
|
+
return { tier: "premium_proxy", min: PREMIUM_PROXY_CREDITS, max: PREMIUM_PROXY_CREDITS };
|
|
247
|
+
}
|
|
248
|
+
if (js) {
|
|
249
|
+
return { tier: "js_render", min: JS_CREDITS, max: JS_CREDITS };
|
|
250
|
+
}
|
|
251
|
+
return { tier: "base", min: BASE_CREDITS, max: BASE_CREDITS };
|
|
252
|
+
}
|
|
253
|
+
function taskParams(task) {
|
|
254
|
+
if (typeof task === "string") return {};
|
|
255
|
+
const record = task;
|
|
256
|
+
return record.zenrows_params ?? {};
|
|
257
|
+
}
|
|
258
|
+
function estimateCost(tasks, zenrowsParams) {
|
|
259
|
+
const jobParams = zenrowsParams ?? {};
|
|
260
|
+
const agg = /* @__PURE__ */ new Map();
|
|
261
|
+
let totalMin = 0;
|
|
262
|
+
let totalMax = 0;
|
|
263
|
+
let count = 0;
|
|
264
|
+
for (const task of tasks) {
|
|
265
|
+
count += 1;
|
|
266
|
+
const merged = { ...jobParams, ...taskParams(task) };
|
|
267
|
+
const cost = costForParams(merged);
|
|
268
|
+
totalMin += cost.min;
|
|
269
|
+
totalMax += cost.max;
|
|
270
|
+
const existing = agg.get(cost.tier);
|
|
271
|
+
if (existing) {
|
|
272
|
+
existing.count += 1;
|
|
273
|
+
} else {
|
|
274
|
+
agg.set(cost.tier, { count: 1, min: cost.min, max: cost.max });
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
const breakdown = TIER_ORDER.filter((tier) => agg.has(tier)).map((tier) => {
|
|
278
|
+
const entry = agg.get(tier);
|
|
279
|
+
if (!entry) throw new Error("unreachable");
|
|
280
|
+
return { tier, count: entry.count, unitMin: entry.min, unitMax: entry.max };
|
|
281
|
+
});
|
|
282
|
+
return {
|
|
283
|
+
taskCount: count,
|
|
284
|
+
min: totalMin,
|
|
285
|
+
max: totalMax,
|
|
286
|
+
breakdown,
|
|
287
|
+
exact: totalMin === totalMax
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// src/batch/resources.ts
|
|
292
|
+
var RunRef = class {
|
|
293
|
+
constructor(client, jobId, runId) {
|
|
294
|
+
this.jobId = jobId;
|
|
295
|
+
this.runId = runId;
|
|
296
|
+
this.#client = client;
|
|
297
|
+
}
|
|
298
|
+
jobId;
|
|
299
|
+
runId;
|
|
300
|
+
#client;
|
|
301
|
+
async load() {
|
|
302
|
+
return new RunHandle(
|
|
303
|
+
this.#client,
|
|
304
|
+
this.jobId,
|
|
305
|
+
this.runId,
|
|
306
|
+
await this.#client._getRunData(this.jobId, this.runId)
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
async delete() {
|
|
310
|
+
await this.#client._deleteRun(this.jobId, this.runId);
|
|
311
|
+
}
|
|
312
|
+
results(options = {}) {
|
|
313
|
+
return this.#client.iterResults(this.jobId, { runId: this.runId, status: options.status });
|
|
314
|
+
}
|
|
315
|
+
taskHistory(taskId) {
|
|
316
|
+
return this.#client.getTaskHistory(this.jobId, taskId, { runId: this.runId });
|
|
317
|
+
}
|
|
318
|
+
downloadToDir(targetDir, options = {}) {
|
|
319
|
+
return this.#client.downloadToDir(this.jobId, this.runId, targetDir, {
|
|
320
|
+
status: "successful",
|
|
321
|
+
...options
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
downloadToMemory(options = {}) {
|
|
325
|
+
return this.#client.downloadToMemory(this.jobId, this.runId, {
|
|
326
|
+
status: "successful",
|
|
327
|
+
...options
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
downloadTaskToFile(task, target) {
|
|
331
|
+
return this.#client.downloadTaskToFile(task, target);
|
|
332
|
+
}
|
|
333
|
+
downloadTaskToMemory(task) {
|
|
334
|
+
return this.#client.downloadTaskToMemory(task);
|
|
335
|
+
}
|
|
336
|
+
async wait(options = {}) {
|
|
337
|
+
const run = await this.#client.waitForRun(this.jobId, { ...options, runId: this.runId });
|
|
338
|
+
return new RunHandle(this.#client, this.jobId, this.runId, run);
|
|
339
|
+
}
|
|
340
|
+
startExport() {
|
|
341
|
+
return this.#client.startResultsExport(this.jobId, this.runId);
|
|
342
|
+
}
|
|
343
|
+
export(exportId) {
|
|
344
|
+
return new ExportRef(this.#client, this.jobId, this.runId, exportId);
|
|
345
|
+
}
|
|
346
|
+
downloadAllResults(targetPath, options = {}) {
|
|
347
|
+
return this.#client.downloadAllResults(this.jobId, this.runId, targetPath, options);
|
|
348
|
+
}
|
|
349
|
+
};
|
|
350
|
+
var RunHandle = class extends RunRef {
|
|
351
|
+
constructor(client, jobId, runId, data) {
|
|
352
|
+
super(client, jobId, runId);
|
|
353
|
+
this.data = data;
|
|
354
|
+
}
|
|
355
|
+
data;
|
|
356
|
+
get status() {
|
|
357
|
+
return this.data.status;
|
|
358
|
+
}
|
|
359
|
+
get stats() {
|
|
360
|
+
return this.data.stats;
|
|
361
|
+
}
|
|
362
|
+
};
|
|
363
|
+
var ExportRef = class {
|
|
364
|
+
constructor(client, jobId, runId, exportId, startResponse) {
|
|
365
|
+
this.jobId = jobId;
|
|
366
|
+
this.runId = runId;
|
|
367
|
+
this.exportId = exportId;
|
|
368
|
+
this.startResponse = startResponse;
|
|
369
|
+
this.#client = client;
|
|
370
|
+
}
|
|
371
|
+
jobId;
|
|
372
|
+
runId;
|
|
373
|
+
exportId;
|
|
374
|
+
startResponse;
|
|
375
|
+
#client;
|
|
376
|
+
async load() {
|
|
377
|
+
const data = await this.#client._getExport(this.jobId, this.runId, this.exportId);
|
|
378
|
+
return new ExportHandle(
|
|
379
|
+
this.#client,
|
|
380
|
+
this.jobId,
|
|
381
|
+
this.runId,
|
|
382
|
+
this.exportId,
|
|
383
|
+
data,
|
|
384
|
+
this.startResponse
|
|
385
|
+
);
|
|
386
|
+
}
|
|
387
|
+
async wait(options = {}) {
|
|
388
|
+
const data = await this.#client.waitForExport(this.jobId, this.runId, this.exportId, options);
|
|
389
|
+
return new ExportHandle(
|
|
390
|
+
this.#client,
|
|
391
|
+
this.jobId,
|
|
392
|
+
this.runId,
|
|
393
|
+
this.exportId,
|
|
394
|
+
data,
|
|
395
|
+
this.startResponse
|
|
396
|
+
);
|
|
397
|
+
}
|
|
398
|
+
/** Stream the export zip to `targetPath`. The export must already be `completed`. */
|
|
399
|
+
async downloadToPath(targetPath) {
|
|
400
|
+
const handle = await this.load();
|
|
401
|
+
if (handle.data.status !== "completed" || !handle.data.download_url) {
|
|
402
|
+
throw new Error(handle.data.error ?? "export not completed");
|
|
403
|
+
}
|
|
404
|
+
const { mkdir: mkdir2 } = await import("fs/promises");
|
|
405
|
+
const { dirname: dirname2 } = await import("path");
|
|
406
|
+
const { createWriteStream: createWriteStream2 } = await import("fs");
|
|
407
|
+
const { Readable: Readable2 } = await import("stream");
|
|
408
|
+
const { pipeline: pipeline2 } = await import("stream/promises");
|
|
409
|
+
await mkdir2(dirname2(targetPath), { recursive: true });
|
|
410
|
+
const response = await fetch(handle.data.download_url);
|
|
411
|
+
if (!response.ok) {
|
|
412
|
+
throw new Error(`export download failed with status ${response.status}`);
|
|
413
|
+
}
|
|
414
|
+
await pipeline2(Readable2.fromWeb(response.body), createWriteStream2(targetPath));
|
|
415
|
+
return targetPath;
|
|
416
|
+
}
|
|
417
|
+
};
|
|
418
|
+
var ExportHandle = class extends ExportRef {
|
|
419
|
+
constructor(client, jobId, runId, exportId, data, startResponse) {
|
|
420
|
+
super(client, jobId, runId, exportId, startResponse);
|
|
421
|
+
this.data = data;
|
|
422
|
+
}
|
|
423
|
+
data;
|
|
424
|
+
get status() {
|
|
425
|
+
return this.data.status;
|
|
426
|
+
}
|
|
427
|
+
};
|
|
428
|
+
var CurrentRun = class {
|
|
429
|
+
constructor(client, jobId) {
|
|
430
|
+
this.jobId = jobId;
|
|
431
|
+
this.#client = client;
|
|
432
|
+
}
|
|
433
|
+
jobId;
|
|
434
|
+
#client;
|
|
435
|
+
async currentRunId() {
|
|
436
|
+
const job = await this.#client._getJobData(this.jobId);
|
|
437
|
+
if (!job.latest_run) {
|
|
438
|
+
throw new Error(`job ${this.jobId} has no run yet`);
|
|
439
|
+
}
|
|
440
|
+
return job.latest_run.run_id;
|
|
441
|
+
}
|
|
442
|
+
async load() {
|
|
443
|
+
const job = await this.#client._getJobData(this.jobId);
|
|
444
|
+
if (!job.latest_run) {
|
|
445
|
+
throw new Error(`job ${this.jobId} has no run yet`);
|
|
446
|
+
}
|
|
447
|
+
return new RunHandle(this.#client, this.jobId, job.latest_run.run_id, job.latest_run);
|
|
448
|
+
}
|
|
449
|
+
async pause() {
|
|
450
|
+
const run = await this.#client._postPause(this.jobId);
|
|
451
|
+
return new RunHandle(this.#client, this.jobId, run.run_id, run);
|
|
452
|
+
}
|
|
453
|
+
async resume() {
|
|
454
|
+
const run = await this.#client._postResume(this.jobId);
|
|
455
|
+
return new RunHandle(this.#client, this.jobId, run.run_id, run);
|
|
456
|
+
}
|
|
457
|
+
async stop() {
|
|
458
|
+
const run = await this.#client._postStop(this.jobId);
|
|
459
|
+
return new RunHandle(this.#client, this.jobId, run.run_id, run);
|
|
460
|
+
}
|
|
461
|
+
cancel() {
|
|
462
|
+
return this.stop();
|
|
463
|
+
}
|
|
464
|
+
async wait(options = {}) {
|
|
465
|
+
const run = await this.#client.waitForRun(this.jobId, options);
|
|
466
|
+
return new RunHandle(this.#client, this.jobId, run.run_id, run);
|
|
467
|
+
}
|
|
468
|
+
results(options = {}) {
|
|
469
|
+
return this.#client.iterResults(this.jobId, { status: options.status });
|
|
470
|
+
}
|
|
471
|
+
taskHistory(taskId) {
|
|
472
|
+
return this.#client.getTaskHistory(this.jobId, taskId);
|
|
473
|
+
}
|
|
474
|
+
downloadToDir(targetDir, options = {}) {
|
|
475
|
+
return this.#client.downloadToDir(this.jobId, void 0, targetDir, {
|
|
476
|
+
status: "successful",
|
|
477
|
+
...options
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
downloadToMemory(options = {}) {
|
|
481
|
+
return this.#client.downloadToMemory(this.jobId, void 0, {
|
|
482
|
+
status: "successful",
|
|
483
|
+
...options
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
downloadTaskToFile(task, target) {
|
|
487
|
+
return this.#client.downloadTaskToFile(task, target);
|
|
488
|
+
}
|
|
489
|
+
downloadTaskToMemory(task) {
|
|
490
|
+
return this.#client.downloadTaskToMemory(task);
|
|
491
|
+
}
|
|
492
|
+
async startExport() {
|
|
493
|
+
return this.#client.startResultsExport(this.jobId, await this.currentRunId());
|
|
494
|
+
}
|
|
495
|
+
async downloadAllResults(targetPath, options = {}) {
|
|
496
|
+
return this.#client.downloadAllResults(
|
|
497
|
+
this.jobId,
|
|
498
|
+
await this.currentRunId(),
|
|
499
|
+
targetPath,
|
|
500
|
+
options
|
|
501
|
+
);
|
|
502
|
+
}
|
|
503
|
+
};
|
|
504
|
+
var ScheduleControls = class {
|
|
505
|
+
constructor(client, jobId) {
|
|
506
|
+
this.jobId = jobId;
|
|
507
|
+
this.#client = client;
|
|
508
|
+
}
|
|
509
|
+
jobId;
|
|
510
|
+
#client;
|
|
511
|
+
async pause() {
|
|
512
|
+
return new JobHandle(
|
|
513
|
+
this.#client,
|
|
514
|
+
this.jobId,
|
|
515
|
+
await this.#client._postScheduleState(this.jobId, "paused")
|
|
516
|
+
);
|
|
517
|
+
}
|
|
518
|
+
async resume() {
|
|
519
|
+
return new JobHandle(
|
|
520
|
+
this.#client,
|
|
521
|
+
this.jobId,
|
|
522
|
+
await this.#client._postScheduleState(this.jobId, "active")
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
async update(schedule) {
|
|
526
|
+
return new JobHandle(
|
|
527
|
+
this.#client,
|
|
528
|
+
this.jobId,
|
|
529
|
+
await this.#client._putSchedule(this.jobId, schedule.toRequestBody())
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
};
|
|
533
|
+
var JobRef = class {
|
|
534
|
+
constructor(client, jobId, submitResponse) {
|
|
535
|
+
this.jobId = jobId;
|
|
536
|
+
this.submitResponse = submitResponse;
|
|
537
|
+
this.#client = client;
|
|
538
|
+
}
|
|
539
|
+
jobId;
|
|
540
|
+
submitResponse;
|
|
541
|
+
#client;
|
|
542
|
+
runFacet;
|
|
543
|
+
scheduleFacet;
|
|
544
|
+
/** Operations on the CURRENT run — pause/resume/stop/wait/results/downloads/startExport. */
|
|
545
|
+
get run() {
|
|
546
|
+
if (!this.runFacet) this.runFacet = new CurrentRun(this.#client, this.jobId);
|
|
547
|
+
return this.runFacet;
|
|
548
|
+
}
|
|
549
|
+
/** Operations on the schedule — pause/resume/update (scheduled jobs only). */
|
|
550
|
+
get schedule() {
|
|
551
|
+
if (!this.scheduleFacet) this.scheduleFacet = new ScheduleControls(this.#client, this.jobId);
|
|
552
|
+
return this.scheduleFacet;
|
|
553
|
+
}
|
|
554
|
+
/** The job status from the submit response — only known on refs from a `submit*` call. */
|
|
555
|
+
get status() {
|
|
556
|
+
return this.submitResponse?.status;
|
|
557
|
+
}
|
|
558
|
+
get acceptedTasks() {
|
|
559
|
+
return this.submitResponse?.accepted_tasks;
|
|
560
|
+
}
|
|
561
|
+
async load() {
|
|
562
|
+
return new JobHandle(this.#client, this.jobId, await this.#client._getJobData(this.jobId));
|
|
563
|
+
}
|
|
564
|
+
async close() {
|
|
565
|
+
return new JobHandle(this.#client, this.jobId, await this.#client._postClose(this.jobId));
|
|
566
|
+
}
|
|
567
|
+
async delete() {
|
|
568
|
+
await this.#client._delete(this.jobId);
|
|
569
|
+
}
|
|
570
|
+
async rerun(options = {}) {
|
|
571
|
+
const resp = await this.#client._postRerun(this.jobId, options);
|
|
572
|
+
return new RunHandle(this.#client, this.jobId, resp.latest_run.run_id, resp.latest_run);
|
|
573
|
+
}
|
|
574
|
+
/** Shortcut for `rerun({ status: "failed" })` (or `"failed,pending"` with `includePending`). */
|
|
575
|
+
retryFailed(options = {}) {
|
|
576
|
+
const status = options.includePending ? ["failed", "pending"] : "failed";
|
|
577
|
+
return this.rerun({ status, idempotencyKey: options.idempotencyKey });
|
|
578
|
+
}
|
|
579
|
+
addTasks(tasks, options = {}) {
|
|
580
|
+
return this.#client._postTasks(this.jobId, tasks, options);
|
|
581
|
+
}
|
|
582
|
+
async *runs(pageSize) {
|
|
583
|
+
yield* this.#client.iterRuns(this.jobId, pageSize);
|
|
584
|
+
}
|
|
585
|
+
addFileInput(data, options) {
|
|
586
|
+
return this.#client.uploadCsv(data, options);
|
|
587
|
+
}
|
|
588
|
+
getWebhook() {
|
|
589
|
+
return this.#client.getJobWebhook(this.jobId);
|
|
590
|
+
}
|
|
591
|
+
setWebhook(url, signature) {
|
|
592
|
+
return this.#client.putJobWebhook(this.jobId, { url, signature });
|
|
593
|
+
}
|
|
594
|
+
async deleteWebhook() {
|
|
595
|
+
await this.#client.deleteJobWebhook(this.jobId);
|
|
596
|
+
}
|
|
597
|
+
/** Block until the current run's async-carrier ingestion has finished writing task rows. */
|
|
598
|
+
async waitForIngest(options = {}) {
|
|
599
|
+
return new JobHandle(
|
|
600
|
+
this.#client,
|
|
601
|
+
this.jobId,
|
|
602
|
+
await this.#client._waitForIngestRaw(this.jobId, options),
|
|
603
|
+
this.submitResponse
|
|
604
|
+
);
|
|
605
|
+
}
|
|
606
|
+
};
|
|
607
|
+
var JobHandle = class extends JobRef {
|
|
608
|
+
constructor(client, jobId, data, submitResponse) {
|
|
609
|
+
super(client, jobId, submitResponse);
|
|
610
|
+
this.data = data;
|
|
611
|
+
}
|
|
612
|
+
data;
|
|
613
|
+
get status() {
|
|
614
|
+
return this.data.status;
|
|
615
|
+
}
|
|
616
|
+
};
|
|
617
|
+
|
|
618
|
+
// src/batch/transport.ts
|
|
619
|
+
var RETRYABLE_STATUSES = /* @__PURE__ */ new Set([429, 502, 503, 504]);
|
|
620
|
+
var IDEMPOTENT_METHODS = /* @__PURE__ */ new Set(["GET", "PUT", "DELETE", "HEAD", "OPTIONS"]);
|
|
621
|
+
var BACKOFF_BASE_MS = 250;
|
|
622
|
+
var BACKOFF_CAP_MS = 1e4;
|
|
623
|
+
var DEFAULT_RETRIES = 3;
|
|
624
|
+
function isIdempotent(method, hasIdempotencyKey) {
|
|
625
|
+
return IDEMPOTENT_METHODS.has(method.toUpperCase()) || method.toUpperCase() === "POST" && hasIdempotencyKey;
|
|
626
|
+
}
|
|
627
|
+
function backoffMs(attempt) {
|
|
628
|
+
const base = Math.min(BACKOFF_BASE_MS * 2 ** attempt, BACKOFF_CAP_MS);
|
|
629
|
+
return base * (1 + (Math.random() * 2 - 1) * 0.2);
|
|
630
|
+
}
|
|
631
|
+
function retryAfterMs(response) {
|
|
632
|
+
const raw = response.headers.get("Retry-After");
|
|
633
|
+
if (!raw) return void 0;
|
|
634
|
+
const secs = Number(raw);
|
|
635
|
+
if (Number.isNaN(secs) || secs < 0) return void 0;
|
|
636
|
+
return secs * 1e3;
|
|
637
|
+
}
|
|
638
|
+
function sleep(ms) {
|
|
639
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
640
|
+
}
|
|
641
|
+
var BatchTransport = class {
|
|
642
|
+
constructor(baseURL, apiKey, retries = DEFAULT_RETRIES) {
|
|
643
|
+
this.baseURL = baseURL;
|
|
644
|
+
this.apiKey = apiKey;
|
|
645
|
+
this.retries = retries;
|
|
646
|
+
}
|
|
647
|
+
baseURL;
|
|
648
|
+
apiKey;
|
|
649
|
+
retries;
|
|
650
|
+
/** Send a request, parse the response, throw `ZenRowsBatchError` on non-2xx. */
|
|
651
|
+
async requestJson(method, path, options = {}) {
|
|
652
|
+
const response = await this.send(method, path, options);
|
|
653
|
+
if (!response.ok) {
|
|
654
|
+
const { problem, extras } = await parseProblem(response);
|
|
655
|
+
throw new ZenRowsBatchError(response.status, problem, extras);
|
|
656
|
+
}
|
|
657
|
+
const text = await response.text();
|
|
658
|
+
if (!text) {
|
|
659
|
+
return void 0;
|
|
660
|
+
}
|
|
661
|
+
return JSON.parse(text);
|
|
662
|
+
}
|
|
663
|
+
/** Send a request and return the raw `Response` — used for endpoints that don't return JSON. */
|
|
664
|
+
async requestRaw(method, path, options = {}) {
|
|
665
|
+
const response = await this.send(method, path, options);
|
|
666
|
+
if (!response.ok) {
|
|
667
|
+
const { problem, extras } = await parseProblem(response);
|
|
668
|
+
throw new ZenRowsBatchError(response.status, problem, extras);
|
|
669
|
+
}
|
|
670
|
+
return response;
|
|
671
|
+
}
|
|
672
|
+
buildUrl(path, query) {
|
|
673
|
+
const url = new URL(`${this.baseURL}${path}`);
|
|
674
|
+
if (query) {
|
|
675
|
+
for (const [key, value] of Object.entries(query)) {
|
|
676
|
+
if (value !== void 0) {
|
|
677
|
+
url.searchParams.append(key, String(value));
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
return url;
|
|
682
|
+
}
|
|
683
|
+
async send(method, path, options) {
|
|
684
|
+
const url = this.buildUrl(path, options.query);
|
|
685
|
+
const headers = {
|
|
686
|
+
"X-API-Key": this.apiKey,
|
|
687
|
+
"User-Agent": `zenrows/${package_default.version} node`,
|
|
688
|
+
...options.headers
|
|
689
|
+
};
|
|
690
|
+
if (options.body !== void 0) {
|
|
691
|
+
headers["Content-Type"] = "application/json";
|
|
692
|
+
}
|
|
693
|
+
if (options.idempotencyKey) {
|
|
694
|
+
headers["Idempotency-Key"] = options.idempotencyKey;
|
|
695
|
+
}
|
|
696
|
+
const idempotent = isIdempotent(method, Boolean(options.idempotencyKey));
|
|
697
|
+
let attempt = 0;
|
|
698
|
+
while (true) {
|
|
699
|
+
let response;
|
|
700
|
+
try {
|
|
701
|
+
response = await fetch(url.toString(), {
|
|
702
|
+
method,
|
|
703
|
+
headers,
|
|
704
|
+
body: options.body !== void 0 ? JSON.stringify(options.body) : void 0
|
|
705
|
+
});
|
|
706
|
+
} catch (error) {
|
|
707
|
+
if (idempotent && attempt < this.retries) {
|
|
708
|
+
await sleep(backoffMs(attempt));
|
|
709
|
+
attempt += 1;
|
|
710
|
+
continue;
|
|
711
|
+
}
|
|
712
|
+
throw error;
|
|
713
|
+
}
|
|
714
|
+
if (idempotent && attempt < this.retries && RETRYABLE_STATUSES.has(response.status)) {
|
|
715
|
+
const wait = retryAfterMs(response) ?? backoffMs(attempt);
|
|
716
|
+
await sleep(wait);
|
|
717
|
+
attempt += 1;
|
|
718
|
+
continue;
|
|
719
|
+
}
|
|
720
|
+
return response;
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
};
|
|
724
|
+
|
|
725
|
+
// src/batch/waiters.ts
|
|
726
|
+
var WaiterTimeoutError = class extends Error {
|
|
727
|
+
constructor(message) {
|
|
728
|
+
super(message);
|
|
729
|
+
this.name = "WaiterTimeoutError";
|
|
730
|
+
}
|
|
731
|
+
};
|
|
732
|
+
var WaiterFailureError = class extends Error {
|
|
733
|
+
constructor(message) {
|
|
734
|
+
super(message);
|
|
735
|
+
this.name = "WaiterFailureError";
|
|
736
|
+
}
|
|
737
|
+
};
|
|
738
|
+
async function pollUntil(fetch2, {
|
|
739
|
+
isDone,
|
|
740
|
+
isFailure,
|
|
741
|
+
timeout = 300,
|
|
742
|
+
initialInterval = 1,
|
|
743
|
+
maxInterval = 15,
|
|
744
|
+
backoff = 1.5,
|
|
745
|
+
jitter = 0.2
|
|
746
|
+
}) {
|
|
747
|
+
const deadline = Date.now() + timeout * 1e3;
|
|
748
|
+
let interval = initialInterval;
|
|
749
|
+
while (true) {
|
|
750
|
+
const value = await fetch2();
|
|
751
|
+
if (isDone(value)) {
|
|
752
|
+
return value;
|
|
753
|
+
}
|
|
754
|
+
if (isFailure?.(value)) {
|
|
755
|
+
throw new WaiterFailureError(
|
|
756
|
+
`waiter: resource entered failure state (${JSON.stringify(value)})`
|
|
757
|
+
);
|
|
758
|
+
}
|
|
759
|
+
const now = Date.now();
|
|
760
|
+
if (now >= deadline) {
|
|
761
|
+
throw new WaiterTimeoutError(`waiter: timed out after ${timeout}s waiting for target state`);
|
|
762
|
+
}
|
|
763
|
+
const remainingSeconds = (deadline - now) / 1e3;
|
|
764
|
+
const sleepSeconds = Math.min(interval, remainingSeconds) * (1 + jitter * (Math.random() * 2 - 1));
|
|
765
|
+
if (sleepSeconds > 0) {
|
|
766
|
+
await new Promise((resolve) => setTimeout(resolve, sleepSeconds * 1e3));
|
|
767
|
+
}
|
|
768
|
+
interval = Math.min(interval * backoff, maxInterval);
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
// src/batch/client.ts
|
|
773
|
+
var DEFAULT_BATCH_API_URL = "https://async.api.zenrows.com/v1";
|
|
774
|
+
var TERMINAL_RUN_STATUSES = /* @__PURE__ */ new Set(["completed", "stopped", "deleted"]);
|
|
775
|
+
var TERMINAL_EXPORT_STATUSES = /* @__PURE__ */ new Set(["completed", "failed"]);
|
|
776
|
+
function coerceUrl(item) {
|
|
777
|
+
return typeof item === "string" ? { url: item } : item;
|
|
778
|
+
}
|
|
779
|
+
var ZenRowsBatchClient = class {
|
|
780
|
+
apiKey;
|
|
781
|
+
transport;
|
|
782
|
+
constructor(apiKey, config = {}) {
|
|
783
|
+
this.apiKey = apiKey;
|
|
784
|
+
this.transport = new BatchTransport(
|
|
785
|
+
config.baseURL ?? DEFAULT_BATCH_API_URL,
|
|
786
|
+
apiKey,
|
|
787
|
+
config.retries
|
|
788
|
+
);
|
|
789
|
+
}
|
|
790
|
+
// ===== submit =====
|
|
791
|
+
async submitJob(options = {}) {
|
|
792
|
+
const { idempotencyKey, waitForIngest, ...body } = options;
|
|
793
|
+
const resp = await this.transport.requestJson("POST", "/jobs", { body, idempotencyKey });
|
|
794
|
+
const ref = new JobRef(this, resp.job_id, resp);
|
|
795
|
+
if (waitForIngest && resp.latest_run && resp.latest_run.ingest_status === "pending") {
|
|
796
|
+
return ref.waitForIngest();
|
|
797
|
+
}
|
|
798
|
+
return ref;
|
|
799
|
+
}
|
|
800
|
+
/** Submit a one-shot scraping job (closed, all tasks known upfront). */
|
|
801
|
+
async submitRegular(urls, fileInputId, opts = {}) {
|
|
802
|
+
if (urls !== void 0 && fileInputId !== void 0) {
|
|
803
|
+
throw new Error("submitRegular: pass urls OR fileInputId, not both.");
|
|
804
|
+
}
|
|
805
|
+
if (urls === void 0 && fileInputId === void 0) {
|
|
806
|
+
throw new Error(
|
|
807
|
+
"submitRegular: closed jobs require urls or fileInputId (use submitOpen for the open/extend pattern)."
|
|
808
|
+
);
|
|
809
|
+
}
|
|
810
|
+
return this.submitJob({
|
|
811
|
+
type: "regular",
|
|
812
|
+
status: "closed",
|
|
813
|
+
tasks: urls?.map(coerceUrl),
|
|
814
|
+
file_input_id: fileInputId,
|
|
815
|
+
zenrows_params: opts.zenrowsParams,
|
|
816
|
+
external_id: opts.externalId,
|
|
817
|
+
name: opts.name,
|
|
818
|
+
metadata: opts.metadata,
|
|
819
|
+
webhook: opts.webhook,
|
|
820
|
+
idempotencyKey: opts.idempotencyKey,
|
|
821
|
+
waitForIngest: opts.waitForIngest
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
/** Submit a streaming-style job that stays open for more tasks via `JobRef.addTasks`. */
|
|
825
|
+
submitOpen(urls, opts = {}) {
|
|
826
|
+
return this.submitJob({
|
|
827
|
+
type: "regular",
|
|
828
|
+
status: "open",
|
|
829
|
+
tasks: urls?.map(coerceUrl),
|
|
830
|
+
zenrows_params: opts.zenrowsParams,
|
|
831
|
+
external_id: opts.externalId,
|
|
832
|
+
name: opts.name,
|
|
833
|
+
metadata: opts.metadata,
|
|
834
|
+
webhook: opts.webhook,
|
|
835
|
+
idempotencyKey: opts.idempotencyKey,
|
|
836
|
+
waitForIngest: opts.waitForIngest
|
|
837
|
+
});
|
|
838
|
+
}
|
|
839
|
+
/** Submit a scheduled job. `schedule` is one of the `At`/`Rate`/`Calendar` builders. */
|
|
840
|
+
async submitScheduled(schedule, urls, fileInputId, opts = {}) {
|
|
841
|
+
if (urls !== void 0 && fileInputId !== void 0) {
|
|
842
|
+
throw new Error("submitScheduled: pass urls OR fileInputId, not both.");
|
|
843
|
+
}
|
|
844
|
+
return this.submitJob({
|
|
845
|
+
type: "scheduled",
|
|
846
|
+
status: "closed",
|
|
847
|
+
tasks: urls?.map(coerceUrl),
|
|
848
|
+
file_input_id: fileInputId,
|
|
849
|
+
zenrows_params: opts.zenrowsParams,
|
|
850
|
+
external_id: opts.externalId,
|
|
851
|
+
name: opts.name,
|
|
852
|
+
metadata: opts.metadata,
|
|
853
|
+
webhook: opts.webhook,
|
|
854
|
+
idempotencyKey: opts.idempotencyKey,
|
|
855
|
+
// biome-ignore lint/suspicious/noExplicitAny: schedule builder output matches JobSchedule at the wire boundary
|
|
856
|
+
...{ schedule: schedule.toRequestBody() }
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
// ===== cost estimation (local, no API call) =====
|
|
860
|
+
estimateCost(tasks, zenrowsParams) {
|
|
861
|
+
return estimateCost(tasks, zenrowsParams);
|
|
862
|
+
}
|
|
863
|
+
// ===== jobs =====
|
|
864
|
+
/** A `JobRef` for an existing job with no network call — prefer this to act on a known id. */
|
|
865
|
+
job(jobId) {
|
|
866
|
+
return new JobRef(this, jobId);
|
|
867
|
+
}
|
|
868
|
+
async getJob(jobId) {
|
|
869
|
+
return new JobHandle(this, jobId, await this._getJobData(jobId));
|
|
870
|
+
}
|
|
871
|
+
async listJobs(options = {}) {
|
|
872
|
+
return this.transport.requestJson("GET", "/jobs", { query: options });
|
|
873
|
+
}
|
|
874
|
+
async *iterJobs(options = {}) {
|
|
875
|
+
let cursor;
|
|
876
|
+
while (true) {
|
|
877
|
+
const page = await this.listJobs({ ...options, cursor });
|
|
878
|
+
for (const job of page.jobs) {
|
|
879
|
+
yield new JobHandle(this, job.job_id, job);
|
|
880
|
+
}
|
|
881
|
+
cursor = page.next_cursor;
|
|
882
|
+
if (!cursor) return;
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
// ===== runs =====
|
|
886
|
+
/** A `RunRef` for an existing run with no network call. */
|
|
887
|
+
run(jobId, runId) {
|
|
888
|
+
return new RunRef(this, jobId, runId);
|
|
889
|
+
}
|
|
890
|
+
async getRun(jobId, runId) {
|
|
891
|
+
return new RunHandle(this, jobId, runId, await this._getRunData(jobId, runId));
|
|
892
|
+
}
|
|
893
|
+
async listRuns(jobId, options = {}) {
|
|
894
|
+
return this.transport.requestJson("GET", `/jobs/${jobId}/runs`, { query: options });
|
|
895
|
+
}
|
|
896
|
+
async *iterRuns(jobId, pageSize) {
|
|
897
|
+
let cursor;
|
|
898
|
+
while (true) {
|
|
899
|
+
const page = await this.listRuns(jobId, { limit: pageSize, cursor });
|
|
900
|
+
for (const run of page.runs) {
|
|
901
|
+
yield new RunHandle(this, jobId, run.run_id, run);
|
|
902
|
+
}
|
|
903
|
+
cursor = page.next_cursor;
|
|
904
|
+
if (!cursor) return;
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
// ===== results / content =====
|
|
908
|
+
async getResults(jobId, options = {}) {
|
|
909
|
+
const { runId, ...query } = options;
|
|
910
|
+
const path = runId ? `/jobs/${jobId}/runs/${runId}/results` : `/jobs/${jobId}/results`;
|
|
911
|
+
return this.transport.requestJson("GET", path, { query });
|
|
912
|
+
}
|
|
913
|
+
async *iterResults(jobId, options = {}) {
|
|
914
|
+
let cursor;
|
|
915
|
+
while (true) {
|
|
916
|
+
const page = await this.getResults(jobId, { ...options, cursor });
|
|
917
|
+
yield* page.results;
|
|
918
|
+
cursor = page.next_cursor;
|
|
919
|
+
if (!cursor) return;
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
/** Returns the scraped page's raw content as-is (HTML/text, not JSON). */
|
|
923
|
+
async getTaskContent(jobId, taskId, options = {}) {
|
|
924
|
+
const path = options.runId ? `/jobs/${jobId}/runs/${options.runId}/tasks/${taskId}/content` : `/jobs/${jobId}/tasks/${taskId}/content`;
|
|
925
|
+
const response = await this.transport.requestRaw("GET", path);
|
|
926
|
+
return response.text();
|
|
927
|
+
}
|
|
928
|
+
getTaskHistory(jobId, taskId, options = {}) {
|
|
929
|
+
const path = options.runId ? `/jobs/${jobId}/runs/${options.runId}/tasks/${taskId}/history` : `/jobs/${jobId}/tasks/${taskId}/history`;
|
|
930
|
+
return this.transport.requestJson("GET", path);
|
|
931
|
+
}
|
|
932
|
+
// ===== downloads =====
|
|
933
|
+
downloadToDir(jobId, runId, targetDir, opts = {}) {
|
|
934
|
+
return downloadToDir(
|
|
935
|
+
this.iterResults(jobId, { runId, status: opts.status ?? "successful" }),
|
|
936
|
+
targetDir,
|
|
937
|
+
opts
|
|
938
|
+
);
|
|
939
|
+
}
|
|
940
|
+
downloadToMemory(jobId, runId, opts = {}) {
|
|
941
|
+
return downloadToMemory(
|
|
942
|
+
this.iterResults(jobId, { runId, status: opts.status ?? "successful" }),
|
|
943
|
+
opts
|
|
944
|
+
);
|
|
945
|
+
}
|
|
946
|
+
downloadTaskToFile(task, target) {
|
|
947
|
+
return downloadTaskToFile(task, target);
|
|
948
|
+
}
|
|
949
|
+
downloadTaskToMemory(task) {
|
|
950
|
+
return downloadTaskToMemory(task);
|
|
951
|
+
}
|
|
952
|
+
// ===== webhooks =====
|
|
953
|
+
getJobWebhook(jobId) {
|
|
954
|
+
return this.transport.requestJson("GET", `/jobs/${jobId}/webhook`);
|
|
955
|
+
}
|
|
956
|
+
putJobWebhook(jobId, config) {
|
|
957
|
+
return this.transport.requestJson("PUT", `/jobs/${jobId}/webhook`, {
|
|
958
|
+
body: { signature: false, ...config }
|
|
959
|
+
});
|
|
960
|
+
}
|
|
961
|
+
async deleteJobWebhook(jobId) {
|
|
962
|
+
await this.transport.requestJson("DELETE", `/jobs/${jobId}/webhook`);
|
|
963
|
+
}
|
|
964
|
+
testWebhook(config) {
|
|
965
|
+
return this.transport.requestJson("POST", "/webhook/test", {
|
|
966
|
+
body: { signature: false, ...config }
|
|
967
|
+
});
|
|
968
|
+
}
|
|
969
|
+
// ===== HMAC key lifecycle =====
|
|
970
|
+
listHmacKeys() {
|
|
971
|
+
return this.transport.requestJson("GET", "/hmac/keys");
|
|
972
|
+
}
|
|
973
|
+
/** Capture the returned `secret` HERE — it is not revealed again. */
|
|
974
|
+
rotateHmacKey() {
|
|
975
|
+
return this.transport.requestJson("POST", "/hmac/keys/rotate");
|
|
976
|
+
}
|
|
977
|
+
finalizeHmacKey() {
|
|
978
|
+
return this.transport.requestJson("POST", "/hmac/keys/rotate/finalize");
|
|
979
|
+
}
|
|
980
|
+
async cancelHmacRotation() {
|
|
981
|
+
await this.transport.requestJson("DELETE", "/hmac/keys/rotate");
|
|
982
|
+
}
|
|
983
|
+
// ===== file inputs (CSV uploads) =====
|
|
984
|
+
createJobInput(body) {
|
|
985
|
+
return this.transport.requestJson("POST", "/job_inputs", { body });
|
|
986
|
+
}
|
|
987
|
+
/** Allocate a CSV slot + PUT the body. Returns the `file_input_id` to pass to `submitJob`. */
|
|
988
|
+
async uploadCsv(data, options) {
|
|
989
|
+
const created = await this.createJobInput({
|
|
990
|
+
type: "csv",
|
|
991
|
+
csv: {
|
|
992
|
+
delimiter: options.delimiter ?? ",",
|
|
993
|
+
quote: options.quote ?? '"',
|
|
994
|
+
header: options.header ?? false,
|
|
995
|
+
fields: options.fields
|
|
996
|
+
}
|
|
997
|
+
});
|
|
998
|
+
const headers = { "Content-Type": "text/csv", ...created.upload.headers ?? {} };
|
|
999
|
+
const response = await fetch(created.upload.url, {
|
|
1000
|
+
method: created.upload.method,
|
|
1001
|
+
headers,
|
|
1002
|
+
body: data
|
|
1003
|
+
});
|
|
1004
|
+
if (!response.ok) {
|
|
1005
|
+
throw new ZenRowsBatchError(response.status, void 0);
|
|
1006
|
+
}
|
|
1007
|
+
return created.file_input_id;
|
|
1008
|
+
}
|
|
1009
|
+
// ===== results exports =====
|
|
1010
|
+
async startResultsExport(jobId, runId) {
|
|
1011
|
+
const resp = await this._postExportStart(jobId, runId);
|
|
1012
|
+
return new ExportRef(this, jobId, runId, resp.export_id, resp);
|
|
1013
|
+
}
|
|
1014
|
+
async getResultsExport(jobId, runId, exportId) {
|
|
1015
|
+
return new ExportHandle(
|
|
1016
|
+
this,
|
|
1017
|
+
jobId,
|
|
1018
|
+
runId,
|
|
1019
|
+
exportId,
|
|
1020
|
+
await this._getExport(jobId, runId, exportId)
|
|
1021
|
+
);
|
|
1022
|
+
}
|
|
1023
|
+
async waitForExport(jobId, runId, exportId, options = {}) {
|
|
1024
|
+
const target = options.targetStatuses ?? TERMINAL_EXPORT_STATUSES;
|
|
1025
|
+
return pollUntil(() => this._getExport(jobId, runId, exportId), {
|
|
1026
|
+
isDone: (e) => target.has(e.status),
|
|
1027
|
+
timeout: options.timeout ?? 600,
|
|
1028
|
+
initialInterval: options.pollInterval ?? 2,
|
|
1029
|
+
maxInterval: options.maxPollInterval ?? 15
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1032
|
+
/** Start an export, wait for it, and save the zip to `targetPath`. Capped at 1 GiB per run. */
|
|
1033
|
+
async downloadAllResults(jobId, runId, targetPath, options = {}) {
|
|
1034
|
+
const exportRef = await this.startResultsExport(jobId, runId);
|
|
1035
|
+
const final = await this.waitForExport(jobId, runId, exportRef.exportId, {
|
|
1036
|
+
timeout: options.waitTimeout ?? 600,
|
|
1037
|
+
pollInterval: options.pollInterval ?? 2
|
|
1038
|
+
});
|
|
1039
|
+
if (final.status !== "completed" || !final.download_url) {
|
|
1040
|
+
throw new Error(final.error ?? "export completed but server returned no download_url");
|
|
1041
|
+
}
|
|
1042
|
+
const { mkdir: mkdir2 } = await import("fs/promises");
|
|
1043
|
+
const { dirname: dirname2 } = await import("path");
|
|
1044
|
+
const { createWriteStream: createWriteStream2 } = await import("fs");
|
|
1045
|
+
const { Readable: Readable2 } = await import("stream");
|
|
1046
|
+
const { pipeline: pipeline2 } = await import("stream/promises");
|
|
1047
|
+
await mkdir2(dirname2(targetPath), { recursive: true });
|
|
1048
|
+
const response = await fetch(final.download_url);
|
|
1049
|
+
if (!response.ok) {
|
|
1050
|
+
throw new ZenRowsBatchError(response.status, void 0);
|
|
1051
|
+
}
|
|
1052
|
+
await pipeline2(Readable2.fromWeb(response.body), createWriteStream2(targetPath));
|
|
1053
|
+
return targetPath;
|
|
1054
|
+
}
|
|
1055
|
+
// ===== waiter =====
|
|
1056
|
+
async waitForRun(jobId, options = {}) {
|
|
1057
|
+
return this._waitForRunRaw(jobId, options);
|
|
1058
|
+
}
|
|
1059
|
+
// ==========================================================
|
|
1060
|
+
// "raw" methods — no resource wrapping; used by handles internally.
|
|
1061
|
+
// ==========================================================
|
|
1062
|
+
/** @internal */
|
|
1063
|
+
async _getJobData(jobId) {
|
|
1064
|
+
return this.transport.requestJson("GET", `/jobs/${jobId}`);
|
|
1065
|
+
}
|
|
1066
|
+
/** @internal */
|
|
1067
|
+
async _getRunData(jobId, runId) {
|
|
1068
|
+
return this.transport.requestJson("GET", `/jobs/${jobId}/runs/${runId}`);
|
|
1069
|
+
}
|
|
1070
|
+
/** @internal */
|
|
1071
|
+
async _postClose(jobId) {
|
|
1072
|
+
return this.transport.requestJson("POST", `/jobs/${jobId}/close`);
|
|
1073
|
+
}
|
|
1074
|
+
/** @internal */
|
|
1075
|
+
async _postStop(jobId) {
|
|
1076
|
+
return this.transport.requestJson("POST", `/jobs/${jobId}/stop`);
|
|
1077
|
+
}
|
|
1078
|
+
/** @internal */
|
|
1079
|
+
async _postPause(jobId) {
|
|
1080
|
+
return this.transport.requestJson("POST", `/jobs/${jobId}/pause`);
|
|
1081
|
+
}
|
|
1082
|
+
/** @internal */
|
|
1083
|
+
async _postResume(jobId) {
|
|
1084
|
+
return this.transport.requestJson("POST", `/jobs/${jobId}/resume`);
|
|
1085
|
+
}
|
|
1086
|
+
/** @internal */
|
|
1087
|
+
async _delete(jobId) {
|
|
1088
|
+
await this.transport.requestJson("DELETE", `/jobs/${jobId}`);
|
|
1089
|
+
}
|
|
1090
|
+
/** @internal */
|
|
1091
|
+
async _deleteRun(jobId, runId) {
|
|
1092
|
+
await this.transport.requestJson("DELETE", `/jobs/${jobId}/runs/${runId}`);
|
|
1093
|
+
}
|
|
1094
|
+
/** @internal */
|
|
1095
|
+
async _postRerun(jobId, options = {}) {
|
|
1096
|
+
const status = Array.isArray(options.status) ? options.status.join(",") : options.status;
|
|
1097
|
+
return this.transport.requestJson("POST", `/jobs/${jobId}/rerun`, {
|
|
1098
|
+
query: status ? { status } : void 0,
|
|
1099
|
+
idempotencyKey: options.idempotencyKey
|
|
1100
|
+
});
|
|
1101
|
+
}
|
|
1102
|
+
/** @internal */
|
|
1103
|
+
async _putSchedule(jobId, schedule) {
|
|
1104
|
+
return this.transport.requestJson("PUT", `/jobs/${jobId}/schedule`, { body: schedule });
|
|
1105
|
+
}
|
|
1106
|
+
/** @internal */
|
|
1107
|
+
async _postScheduleState(jobId, state) {
|
|
1108
|
+
return this.transport.requestJson("POST", `/jobs/${jobId}/schedule/state`, {
|
|
1109
|
+
body: { schedule_state: state }
|
|
1110
|
+
});
|
|
1111
|
+
}
|
|
1112
|
+
/** @internal */
|
|
1113
|
+
async _postTasks(jobId, tasks, options = {}) {
|
|
1114
|
+
return this.transport.requestJson("POST", `/jobs/${jobId}/tasks`, {
|
|
1115
|
+
body: { tasks, last_batch: options.lastBatch ?? false }
|
|
1116
|
+
});
|
|
1117
|
+
}
|
|
1118
|
+
/** @internal */
|
|
1119
|
+
async _postExportStart(jobId, runId) {
|
|
1120
|
+
return this.transport.requestJson("POST", `/jobs/${jobId}/runs/${runId}/exports`);
|
|
1121
|
+
}
|
|
1122
|
+
/** @internal */
|
|
1123
|
+
async _getExport(jobId, runId, exportId) {
|
|
1124
|
+
return this.transport.requestJson("GET", `/jobs/${jobId}/runs/${runId}/exports/${exportId}`);
|
|
1125
|
+
}
|
|
1126
|
+
/** @internal */
|
|
1127
|
+
async _waitForRunRaw(jobId, options) {
|
|
1128
|
+
const target = options.targetStatuses ?? TERMINAL_RUN_STATUSES;
|
|
1129
|
+
const fetchRun = async () => {
|
|
1130
|
+
if (!options.runId) {
|
|
1131
|
+
const job = await this._getJobData(jobId);
|
|
1132
|
+
return job.latest_run;
|
|
1133
|
+
}
|
|
1134
|
+
return this._getRunData(jobId, options.runId);
|
|
1135
|
+
};
|
|
1136
|
+
return pollUntil(fetchRun, {
|
|
1137
|
+
isDone: (run) => run !== void 0 && target.has(run.status),
|
|
1138
|
+
isFailure: (run) => run !== void 0 && Boolean(options.failureStatuses?.has(run.status)),
|
|
1139
|
+
timeout: options.timeout ?? 300,
|
|
1140
|
+
initialInterval: options.pollInterval ?? 2,
|
|
1141
|
+
maxInterval: options.maxPollInterval ?? 15
|
|
1142
|
+
// biome-ignore lint/suspicious/noExplicitAny: fetchRun can resolve undefined for a not-yet-fired scheduled job
|
|
1143
|
+
});
|
|
1144
|
+
}
|
|
1145
|
+
/** @internal */
|
|
1146
|
+
async _waitForIngestRaw(jobId, options = {}) {
|
|
1147
|
+
return pollUntil(() => this._getJobData(jobId), {
|
|
1148
|
+
isDone: (job) => {
|
|
1149
|
+
const run = job.latest_run;
|
|
1150
|
+
if (!run || TERMINAL_RUN_STATUSES.has(run.status)) return true;
|
|
1151
|
+
return run.ingest_status !== "pending";
|
|
1152
|
+
},
|
|
1153
|
+
timeout: options.timeout ?? 300,
|
|
1154
|
+
initialInterval: options.pollInterval ?? 2,
|
|
1155
|
+
maxInterval: options.maxPollInterval ?? 15
|
|
1156
|
+
});
|
|
1157
|
+
}
|
|
1158
|
+
};
|
|
1159
|
+
|
|
1160
|
+
// src/batch/schedule.ts
|
|
1161
|
+
var DAYS_OF_WEEK = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"];
|
|
1162
|
+
function validateTimezone(tz, field) {
|
|
1163
|
+
if (!tz) {
|
|
1164
|
+
throw new Error(`${field} is required (IANA name, e.g. "Europe/Berlin")`);
|
|
1165
|
+
}
|
|
1166
|
+
try {
|
|
1167
|
+
new Intl.DateTimeFormat(void 0, { timeZone: tz });
|
|
1168
|
+
} catch {
|
|
1169
|
+
throw new Error(`${field}: ${JSON.stringify(tz)} is not a valid IANA timezone`);
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
function validateFullHour(s) {
|
|
1173
|
+
if (s.length !== 5 || s[2] !== ":" || s.slice(3) !== "00") {
|
|
1174
|
+
throw new Error(
|
|
1175
|
+
`times_of_day entry ${JSON.stringify(s)} must be on the hour ("HH:00"); minute granularity is rejected.`
|
|
1176
|
+
);
|
|
1177
|
+
}
|
|
1178
|
+
const h = Number(s.slice(0, 2));
|
|
1179
|
+
if (!Number.isInteger(h) || h < 0 || h > 23) {
|
|
1180
|
+
throw new Error(`times_of_day entry ${JSON.stringify(s)} is not a valid hour (00..23)`);
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
function hasTzSuffix(raw) {
|
|
1184
|
+
const trimmed = raw.trim();
|
|
1185
|
+
for (const sep of ["T", " "]) {
|
|
1186
|
+
const i = trimmed.indexOf(sep);
|
|
1187
|
+
if (i >= 0) {
|
|
1188
|
+
const tail = trimmed.slice(i + 1);
|
|
1189
|
+
if (tail.endsWith("Z") || tail.endsWith("z")) return true;
|
|
1190
|
+
return tail.length >= 6 && (tail[tail.length - 6] === "+" || tail[tail.length - 6] === "-");
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
return false;
|
|
1194
|
+
}
|
|
1195
|
+
function formatLocalNaive(date) {
|
|
1196
|
+
const pad = (n, width = 2) => String(n).padStart(width, "0");
|
|
1197
|
+
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}T${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`;
|
|
1198
|
+
}
|
|
1199
|
+
var At = class {
|
|
1200
|
+
at;
|
|
1201
|
+
timezone;
|
|
1202
|
+
constructor(at, timezone) {
|
|
1203
|
+
validateTimezone(timezone, "At.timezone");
|
|
1204
|
+
if (at instanceof Date) {
|
|
1205
|
+
this.at = formatLocalNaive(at);
|
|
1206
|
+
} else {
|
|
1207
|
+
if (!at.trim()) {
|
|
1208
|
+
throw new Error("At.at must be a non-empty ISO timestamp string.");
|
|
1209
|
+
}
|
|
1210
|
+
if (hasTzSuffix(at)) {
|
|
1211
|
+
throw new Error(
|
|
1212
|
+
`At.at ${JSON.stringify(at)} must be tz-naive (no "Z", no offset). Supply timezone separately to keep DST transitions deterministic.`
|
|
1213
|
+
);
|
|
1214
|
+
}
|
|
1215
|
+
this.at = at;
|
|
1216
|
+
}
|
|
1217
|
+
this.timezone = timezone;
|
|
1218
|
+
}
|
|
1219
|
+
toRequestBody() {
|
|
1220
|
+
return { at: this.at, timezone: this.timezone };
|
|
1221
|
+
}
|
|
1222
|
+
};
|
|
1223
|
+
var Rate = class {
|
|
1224
|
+
every;
|
|
1225
|
+
unit;
|
|
1226
|
+
constructor(every, unit) {
|
|
1227
|
+
if (!Number.isInteger(every)) {
|
|
1228
|
+
throw new TypeError(`Rate.every must be an integer, got ${every}`);
|
|
1229
|
+
}
|
|
1230
|
+
if (every < 1) {
|
|
1231
|
+
throw new Error(`Rate.every must be >= 1, got ${every}`);
|
|
1232
|
+
}
|
|
1233
|
+
if (unit !== "minute" && unit !== "hour" && unit !== "day") {
|
|
1234
|
+
throw new Error(
|
|
1235
|
+
`Rate.unit must be one of "minute", "hour", "day"; got ${JSON.stringify(unit)}`
|
|
1236
|
+
);
|
|
1237
|
+
}
|
|
1238
|
+
this.every = every;
|
|
1239
|
+
this.unit = unit;
|
|
1240
|
+
}
|
|
1241
|
+
toRequestBody() {
|
|
1242
|
+
return { rate: { every: this.every, unit: this.unit } };
|
|
1243
|
+
}
|
|
1244
|
+
};
|
|
1245
|
+
var Daily = class {
|
|
1246
|
+
};
|
|
1247
|
+
var Weekly = class {
|
|
1248
|
+
days;
|
|
1249
|
+
constructor(days) {
|
|
1250
|
+
if (!days.length) {
|
|
1251
|
+
throw new Error("Weekly.days must be non-empty");
|
|
1252
|
+
}
|
|
1253
|
+
for (const d of days) {
|
|
1254
|
+
if (!DAYS_OF_WEEK.includes(d)) {
|
|
1255
|
+
throw new Error(
|
|
1256
|
+
`Weekly.days entry ${JSON.stringify(d)} is not a valid day (use one of ${DAYS_OF_WEEK.join(", ")})`
|
|
1257
|
+
);
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
this.days = days;
|
|
1261
|
+
}
|
|
1262
|
+
};
|
|
1263
|
+
var Monthly = class {
|
|
1264
|
+
days;
|
|
1265
|
+
constructor(days) {
|
|
1266
|
+
if (!days.length) {
|
|
1267
|
+
throw new Error("Monthly.days must be non-empty");
|
|
1268
|
+
}
|
|
1269
|
+
for (const d of days) {
|
|
1270
|
+
if (!Number.isInteger(d)) {
|
|
1271
|
+
throw new TypeError(`Monthly.days entries must be integers, got ${d}`);
|
|
1272
|
+
}
|
|
1273
|
+
if (d < 1 || d > 31) {
|
|
1274
|
+
throw new Error(`Monthly.days entry ${d} is out of range (1..31)`);
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
this.days = days;
|
|
1278
|
+
}
|
|
1279
|
+
};
|
|
1280
|
+
var Calendar = class {
|
|
1281
|
+
timesOfDay;
|
|
1282
|
+
cadence;
|
|
1283
|
+
timezone;
|
|
1284
|
+
constructor(timesOfDay, cadence, timezone) {
|
|
1285
|
+
validateTimezone(timezone, "Calendar.timezone");
|
|
1286
|
+
if (!timesOfDay.length) {
|
|
1287
|
+
throw new Error("Calendar.times_of_day must be non-empty");
|
|
1288
|
+
}
|
|
1289
|
+
for (const t of timesOfDay) {
|
|
1290
|
+
validateFullHour(t);
|
|
1291
|
+
}
|
|
1292
|
+
this.timesOfDay = timesOfDay;
|
|
1293
|
+
this.cadence = cadence;
|
|
1294
|
+
this.timezone = timezone;
|
|
1295
|
+
}
|
|
1296
|
+
toRequestBody() {
|
|
1297
|
+
let cadenceBody;
|
|
1298
|
+
if (this.cadence instanceof Daily) {
|
|
1299
|
+
cadenceBody = { daily: {} };
|
|
1300
|
+
} else if (this.cadence instanceof Weekly) {
|
|
1301
|
+
cadenceBody = { weekly: { days: [...this.cadence.days] } };
|
|
1302
|
+
} else {
|
|
1303
|
+
cadenceBody = { monthly: { days: [...this.cadence.days] } };
|
|
1304
|
+
}
|
|
1305
|
+
return {
|
|
1306
|
+
calendar: { times_of_day: [...this.timesOfDay], cadence: cadenceBody },
|
|
1307
|
+
timezone: this.timezone
|
|
1308
|
+
};
|
|
1309
|
+
}
|
|
1310
|
+
};
|
|
1311
|
+
|
|
55
1312
|
// src/index.ts
|
|
56
1313
|
var API_URL = "https://api.zenrows.com/v1/";
|
|
1314
|
+
async function isAuth010(response) {
|
|
1315
|
+
try {
|
|
1316
|
+
const body = await response.clone().json();
|
|
1317
|
+
return typeof body?.code === "string" && body.code.toUpperCase() === "AUTH010";
|
|
1318
|
+
} catch {
|
|
1319
|
+
return false;
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
57
1322
|
var ZenRows = class {
|
|
58
1323
|
apiKey;
|
|
59
1324
|
clientConfig;
|
|
60
1325
|
queue;
|
|
61
1326
|
fetchWithRetry;
|
|
1327
|
+
/** Client for the Batch API (async job/run/task model) — see `./batch.ts`. */
|
|
1328
|
+
batch;
|
|
62
1329
|
constructor(apiKey, clientConfig = {}) {
|
|
63
1330
|
this.apiKey = apiKey;
|
|
64
1331
|
this.clientConfig = clientConfig;
|
|
1332
|
+
this.batch = new ZenRowsBatchClient(apiKey);
|
|
65
1333
|
const retries = this.clientConfig.retries ?? 0;
|
|
66
1334
|
this.queue = fastq.promise(this, this.worker, this.clientConfig.concurrency ?? 5);
|
|
67
1335
|
this.fetchWithRetry = fetchRetry(fetch, {
|
|
@@ -78,9 +1346,47 @@ var ZenRows = class {
|
|
|
78
1346
|
}
|
|
79
1347
|
});
|
|
80
1348
|
}
|
|
81
|
-
|
|
1349
|
+
/**
|
|
1350
|
+
* Fetch a URL through Zenrows — the main page-scraping product. This is the primary entry
|
|
1351
|
+
* point; `get()` remains as a deprecated alias for existing callers.
|
|
1352
|
+
*/
|
|
1353
|
+
fetch(url, config, { headers = {} } = {}) {
|
|
82
1354
|
return this.queue.push({ url, config, headers });
|
|
83
1355
|
}
|
|
1356
|
+
/** @deprecated Use `fetch()` instead. Kept for backward compatibility. */
|
|
1357
|
+
get(url, config, opts = {}) {
|
|
1358
|
+
return this.fetch(url, config, opts);
|
|
1359
|
+
}
|
|
1360
|
+
/**
|
|
1361
|
+
* Fetch a URL and run it through Extract — Zenrows' AI-powered structured extraction
|
|
1362
|
+
* (beta). `mode` defaults to `"auto"`; pass `"native"` or `"standard"` to pick a
|
|
1363
|
+
* different extraction contract. This is a thin, typed wrapper over `fetch()` with the
|
|
1364
|
+
* `extract` param set — no separate endpoint or auth.
|
|
1365
|
+
*
|
|
1366
|
+
* Also sends Adaptive Stealth Mode (`mode: "auto"`) by default, so a target that needs
|
|
1367
|
+
* `js_render`/`premium_proxy` gets escalated automatically instead of failing with
|
|
1368
|
+
* REQS002 — pass `adaptiveStealth: false` to disable that and set `js_render`/
|
|
1369
|
+
* `premium_proxy` yourself.
|
|
1370
|
+
*
|
|
1371
|
+
* `extract=auto` is a domain-gated open beta: when the target domain isn't enabled yet,
|
|
1372
|
+
* the API returns a 402 with `code: "AUTH010"`. By default this method catches that and
|
|
1373
|
+
* retries once with `autoparse: true` instead of returning the error response — pass
|
|
1374
|
+
* `fallbackToAutoparse: false` to disable that and get the raw AUTH010 response back.
|
|
1375
|
+
*/
|
|
1376
|
+
async extract(url, config, opts = {}) {
|
|
1377
|
+
const {
|
|
1378
|
+
extract = "auto",
|
|
1379
|
+
fallbackToAutoparse = true,
|
|
1380
|
+
adaptiveStealth = true,
|
|
1381
|
+
...rest
|
|
1382
|
+
} = config ?? {};
|
|
1383
|
+
const mode = adaptiveStealth ? "auto" : void 0;
|
|
1384
|
+
const response = await this.fetch(url, { ...rest, extract, mode }, opts);
|
|
1385
|
+
if (response.status === 402 && extract === "auto" && fallbackToAutoparse && await isAuth010(response)) {
|
|
1386
|
+
return this.fetch(url, { ...rest, autoparse: true, mode }, opts);
|
|
1387
|
+
}
|
|
1388
|
+
return response;
|
|
1389
|
+
}
|
|
84
1390
|
post(url, config, { headers = {}, data = {} } = {
|
|
85
1391
|
headers: { "Content-Type": "application/x-www-form-urlencoded" }
|
|
86
1392
|
}) {
|
|
@@ -150,5 +1456,39 @@ var ZenRows = class {
|
|
|
150
1456
|
}
|
|
151
1457
|
};
|
|
152
1458
|
export {
|
|
153
|
-
|
|
1459
|
+
AUTO_MAX_CREDITS,
|
|
1460
|
+
AUTO_MIN_CREDITS,
|
|
1461
|
+
At,
|
|
1462
|
+
BASE_CREDITS,
|
|
1463
|
+
Calendar,
|
|
1464
|
+
CurrentRun,
|
|
1465
|
+
DEFAULT_MAX_BYTES_PER_FILE,
|
|
1466
|
+
DEFAULT_MAX_COUNT_IN_MEMORY,
|
|
1467
|
+
DEFAULT_MAX_FILES,
|
|
1468
|
+
DEFAULT_MAX_TOTAL_BYTES_IN_MEMORY,
|
|
1469
|
+
Daily,
|
|
1470
|
+
ExportHandle,
|
|
1471
|
+
ExportRef,
|
|
1472
|
+
JS_AND_PROXY_CREDITS,
|
|
1473
|
+
JS_CREDITS,
|
|
1474
|
+
JobHandle,
|
|
1475
|
+
JobRef,
|
|
1476
|
+
Monthly,
|
|
1477
|
+
PREMIUM_PROXY_CREDITS,
|
|
1478
|
+
Rate,
|
|
1479
|
+
RunHandle,
|
|
1480
|
+
RunRef,
|
|
1481
|
+
WaiterFailureError,
|
|
1482
|
+
WaiterTimeoutError,
|
|
1483
|
+
Weekly,
|
|
1484
|
+
ZenRows,
|
|
1485
|
+
ZenRowsBatchClient,
|
|
1486
|
+
ZenRowsBatchError,
|
|
1487
|
+
downloadTaskToFile,
|
|
1488
|
+
downloadTaskToMemory,
|
|
1489
|
+
downloadToDir,
|
|
1490
|
+
downloadToMemory,
|
|
1491
|
+
estimateCost,
|
|
1492
|
+
parseProblem,
|
|
1493
|
+
pollUntil
|
|
154
1494
|
};
|