zenrows 2.0.3 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/index.cjs +13 -4
- package/dist/index.d.cts +45 -17
- package/dist/index.d.ts +45 -17
- package/dist/index.js +13 -4
- package/package.json +6 -2
- package/tsconfig.test.json +9 -0
- package/vitest.config.ts +2 -0
package/README.md
CHANGED
|
@@ -17,6 +17,7 @@ SDK to access [Zenrows](https://www.zenrows.com/) API directly from Node.js. Zen
|
|
|
17
17
|
- [POST Requests](#post-requests)
|
|
18
18
|
- [Extract](#extract)
|
|
19
19
|
- [Batch](#batch)
|
|
20
|
+
- [Failed runs and extensible values](#failed-runs-and-extensible-values)
|
|
20
21
|
- [Extract in a batch](#extract-in-a-batch)
|
|
21
22
|
- [Concurrency](#concurrency)
|
|
22
23
|
- [An important note about Promise.allSettled() on TypeScript](#an-important-note-about-promiseallsettled-on-typescript)
|
|
@@ -226,6 +227,17 @@ const apiKey = "YOUR-API-KEY";
|
|
|
226
227
|
|
|
227
228
|
`client.batch` also exposes `listJobs()`, `deleteJob()`, `stopRun()`, `rerun()`, `listRuns()`, `getRun()`, `deleteRun()`, and `getTaskContent()` (returns the scraped page's raw content as a string, not JSON — the endpoint can return HTML or plain text depending on what the target page served). Scheduling, webhook config, HMAC key rotation, CSV task uploads, and results exports aren't wrapped yet — call the [Batch API](https://docs.zenrows.com) directly for those.
|
|
228
229
|
|
|
230
|
+
#### Failed runs and extensible values
|
|
231
|
+
|
|
232
|
+
A run the server auto-fails carries `failure_reason` and a human-readable `failure_detail`. Starting, rerunning or resuming a job whose API key hit one of its credit caps throws a `ZenRowsBatchError` with `status` 402 and `code` `"api_key_cap_reached"`; `error.problem.detail` names the cap and when it resets (UTC).
|
|
233
|
+
|
|
234
|
+
```js
|
|
235
|
+
const run = await client.batch.getRun(jobId, runId);
|
|
236
|
+
if (run.data.failure_reason === "api_key_cap_reached") console.log(run.data.failure_detail);
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
Enum-valued fields on responses (run, job, task and export `status`, `ingest_status`, `failure_reason`) are typed `Extensible<...>`: the known values plus any `string`, because the server may add values. The exported named types (`RunStatus`, `JobStatus`, `TaskStatus`, `ExportStatus`, `IngestStatus`, `RunFailureReason`) stay closed, so they still work as request input. Give switches on these fields a default branch; an exhaustive `switch` that assigns the leftover to `never` no longer compiles.
|
|
240
|
+
|
|
229
241
|
#### Extract in a batch
|
|
230
242
|
|
|
231
243
|
Set `extract` in the batch params to run tasks through Extract — structured data instead of raw HTML. It works job-wide or per task, and per-task values win on collision.
|
package/dist/index.cjs
CHANGED
|
@@ -73,7 +73,7 @@ var import_fetch_retry = __toESM(require("fetch-retry"), 1);
|
|
|
73
73
|
// package.json
|
|
74
74
|
var package_default = {
|
|
75
75
|
name: "zenrows",
|
|
76
|
-
version: "2.
|
|
76
|
+
version: "2.1.1",
|
|
77
77
|
description: "ZenRows Node SDK",
|
|
78
78
|
main: "./dist/index.cjs",
|
|
79
79
|
module: "./dist/index.js",
|
|
@@ -95,7 +95,15 @@ var package_default = {
|
|
|
95
95
|
format: "biome format --write .",
|
|
96
96
|
check: "biome check ."
|
|
97
97
|
},
|
|
98
|
-
keywords: [
|
|
98
|
+
keywords: [
|
|
99
|
+
"sdk",
|
|
100
|
+
"zenrows",
|
|
101
|
+
"scraping",
|
|
102
|
+
"proxy",
|
|
103
|
+
"data extraction",
|
|
104
|
+
"web scraping",
|
|
105
|
+
"web scraping api"
|
|
106
|
+
],
|
|
99
107
|
author: "ZenRows",
|
|
100
108
|
repository: {
|
|
101
109
|
type: "git",
|
|
@@ -839,7 +847,7 @@ async function pollUntil(fetch2, {
|
|
|
839
847
|
|
|
840
848
|
// src/batch/client.ts
|
|
841
849
|
var DEFAULT_BATCH_API_URL = "https://async.api.zenrows.com/v1";
|
|
842
|
-
var TERMINAL_RUN_STATUSES = /* @__PURE__ */ new Set(["completed", "stopped", "deleted"]);
|
|
850
|
+
var TERMINAL_RUN_STATUSES = /* @__PURE__ */ new Set(["completed", "stopped", "failed", "deleted"]);
|
|
843
851
|
var TERMINAL_EXPORT_STATUSES = /* @__PURE__ */ new Set(["completed", "failed"]);
|
|
844
852
|
function coerceUrl(item) {
|
|
845
853
|
return typeof item === "string" ? { url: item } : item;
|
|
@@ -1202,7 +1210,8 @@ var ZenRowsBatchClient = class {
|
|
|
1202
1210
|
return this._getRunData(jobId, options.runId);
|
|
1203
1211
|
};
|
|
1204
1212
|
return pollUntil(fetchRun, {
|
|
1205
|
-
|
|
1213
|
+
// A status the caller named as a failure throws, even when it is also a target.
|
|
1214
|
+
isDone: (run) => run !== void 0 && target.has(run.status) && !options.failureStatuses?.has(run.status),
|
|
1206
1215
|
isFailure: (run) => run !== void 0 && Boolean(options.failureStatuses?.has(run.status)),
|
|
1207
1216
|
timeout: options.timeout ?? 300,
|
|
1208
1217
|
initialInterval: options.pollInterval ?? 2,
|
package/dist/index.d.cts
CHANGED
|
@@ -1,14 +1,29 @@
|
|
|
1
1
|
import * as fetch_retry from 'fetch-retry';
|
|
2
2
|
import fastq from 'fastq';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* A server-extensible string enum: the listed values autocomplete, but the server may
|
|
6
|
+
* return values not listed (the Batch API marks these `x-extensible-enum`). Response fields
|
|
7
|
+
* use `Extensible<...>`, so callers must handle an unknown value. The named types stay closed
|
|
8
|
+
* so they keep working as request input.
|
|
9
|
+
*/
|
|
10
|
+
type Extensible<T extends string> = T | (string & {});
|
|
4
11
|
type JobType = "regular" | "scheduled";
|
|
5
12
|
type JobStatus = "open" | "closed" | "deleted";
|
|
6
13
|
type ScheduleState = "active" | "paused";
|
|
7
14
|
type RunStatus = "pending" | "running" | "completed" | "failed" | "stopped" | "deleted";
|
|
8
|
-
type TaskStatus = "pending" | "successful" | "failed";
|
|
15
|
+
type TaskStatus = "pending" | "processing" | "successful" | "failed";
|
|
9
16
|
type PauseState = "active" | "paused";
|
|
10
17
|
type IngestStatus = "pending" | "done";
|
|
11
18
|
type ExportStatus = "pending" | "running" | "completed" | "failed";
|
|
19
|
+
/**
|
|
20
|
+
* Why a run auto-failed (`status == "failed"`).
|
|
21
|
+
* - `insufficient_credits` — the account is out of credits.
|
|
22
|
+
* - `subscription_inactive` — the subscription is not active.
|
|
23
|
+
* - `api_key_cap_reached` — the job's API key reached one of its credit caps; resolves when
|
|
24
|
+
* that cap's window resets or the cap is raised. `Run.failure_detail` says which.
|
|
25
|
+
*/
|
|
26
|
+
type RunFailureReason = "insufficient_credits" | "subscription_inactive" | "api_key_cap_reached";
|
|
12
27
|
interface Spend {
|
|
13
28
|
credits: number;
|
|
14
29
|
cost: number;
|
|
@@ -53,19 +68,21 @@ interface Run {
|
|
|
53
68
|
run_id: string;
|
|
54
69
|
job_id: string;
|
|
55
70
|
run_sequence: number;
|
|
56
|
-
status: RunStatus
|
|
71
|
+
status: Extensible<RunStatus>;
|
|
57
72
|
stats: RunStats;
|
|
58
73
|
last_batch_received?: boolean;
|
|
59
74
|
pause_state?: PauseState;
|
|
60
|
-
ingest_status?: IngestStatus
|
|
61
|
-
failure_reason?:
|
|
75
|
+
ingest_status?: Extensible<IngestStatus>;
|
|
76
|
+
failure_reason?: Extensible<RunFailureReason>;
|
|
77
|
+
/** Human-readable explanation of `failure_reason` (e.g. which cap, and when it resets, UTC). */
|
|
78
|
+
failure_detail?: string | null;
|
|
62
79
|
created_at?: string;
|
|
63
80
|
updated_at?: string;
|
|
64
81
|
}
|
|
65
82
|
interface Job {
|
|
66
83
|
job_id: string;
|
|
67
84
|
type: JobType;
|
|
68
|
-
status: JobStatus
|
|
85
|
+
status: Extensible<JobStatus>;
|
|
69
86
|
format?: string;
|
|
70
87
|
zenrows_params?: Record<string, string>;
|
|
71
88
|
external_id?: string;
|
|
@@ -89,19 +106,19 @@ interface ListJobRunsResponse {
|
|
|
89
106
|
}
|
|
90
107
|
interface SubmitJobResponse {
|
|
91
108
|
job_id: string;
|
|
92
|
-
status: JobStatus
|
|
109
|
+
status: Extensible<JobStatus>;
|
|
93
110
|
latest_run?: Run;
|
|
94
111
|
accepted_tasks: number;
|
|
95
112
|
webhook?: WebhookConfig;
|
|
96
113
|
}
|
|
97
114
|
interface AddTasksResponse {
|
|
98
115
|
accepted_tasks: number;
|
|
99
|
-
job_status: JobStatus
|
|
116
|
+
job_status: Extensible<JobStatus>;
|
|
100
117
|
latest_run: Run;
|
|
101
118
|
}
|
|
102
119
|
interface RerunJobResponse {
|
|
103
120
|
job_id: string;
|
|
104
|
-
status: JobStatus
|
|
121
|
+
status: Extensible<JobStatus>;
|
|
105
122
|
latest_run: Run;
|
|
106
123
|
rerun_of?: string;
|
|
107
124
|
retried_tasks: number;
|
|
@@ -122,7 +139,7 @@ interface TaskResult {
|
|
|
122
139
|
url: string;
|
|
123
140
|
metadata?: Record<string, string>;
|
|
124
141
|
method?: "GET" | "POST";
|
|
125
|
-
status: TaskStatus
|
|
142
|
+
status: Extensible<TaskStatus>;
|
|
126
143
|
type?: string;
|
|
127
144
|
result_url?: string;
|
|
128
145
|
error?: ProblemJson;
|
|
@@ -145,13 +162,13 @@ interface TaskHistoryResponse {
|
|
|
145
162
|
}
|
|
146
163
|
interface StartExportResponse {
|
|
147
164
|
export_id: string;
|
|
148
|
-
status: ExportStatus
|
|
165
|
+
status: Extensible<ExportStatus>;
|
|
149
166
|
created_at: string;
|
|
150
167
|
expires_at: string;
|
|
151
168
|
}
|
|
152
169
|
interface Export {
|
|
153
170
|
export_id: string;
|
|
154
|
-
status: ExportStatus
|
|
171
|
+
status: Extensible<ExportStatus>;
|
|
155
172
|
error?: string;
|
|
156
173
|
download_url?: string;
|
|
157
174
|
created_at: string;
|
|
@@ -350,7 +367,7 @@ declare class RunRef {
|
|
|
350
367
|
declare class RunHandle extends RunRef {
|
|
351
368
|
readonly data: Run;
|
|
352
369
|
constructor(client: ZenRowsBatchClient, jobId: string, runId: string, data: Run);
|
|
353
|
-
get status(): RunStatus
|
|
370
|
+
get status(): Extensible<RunStatus>;
|
|
354
371
|
get stats(): RunStats;
|
|
355
372
|
}
|
|
356
373
|
declare class ExportRef {
|
|
@@ -372,7 +389,7 @@ declare class ExportRef {
|
|
|
372
389
|
declare class ExportHandle extends ExportRef {
|
|
373
390
|
readonly data: Export;
|
|
374
391
|
constructor(client: ZenRowsBatchClient, jobId: string, runId: string, exportId: string, data: Export, startResponse?: StartExportResponse);
|
|
375
|
-
get status(): ExportStatus
|
|
392
|
+
get status(): Extensible<ExportStatus>;
|
|
376
393
|
}
|
|
377
394
|
/** Operations on a job's CURRENT run, reached via `job.run`. */
|
|
378
395
|
declare class CurrentRun {
|
|
@@ -431,7 +448,7 @@ declare class JobRef {
|
|
|
431
448
|
/** Operations on the schedule — pause/resume/update (scheduled jobs only). */
|
|
432
449
|
get schedule(): ScheduleControls;
|
|
433
450
|
/** The job status from the submit response — only known on refs from a `submit*` call. */
|
|
434
|
-
get status(): JobStatus | undefined;
|
|
451
|
+
get status(): Extensible<JobStatus> | undefined;
|
|
435
452
|
get acceptedTasks(): number | undefined;
|
|
436
453
|
load(): Promise<JobHandle>;
|
|
437
454
|
close(): Promise<JobHandle>;
|
|
@@ -449,7 +466,7 @@ declare class JobRef {
|
|
|
449
466
|
lastBatch?: boolean;
|
|
450
467
|
}): Promise<{
|
|
451
468
|
accepted_tasks: number;
|
|
452
|
-
job_status: JobStatus
|
|
469
|
+
job_status: Extensible<JobStatus>;
|
|
453
470
|
latest_run: Run;
|
|
454
471
|
}>;
|
|
455
472
|
runs(pageSize?: number): AsyncGenerator<RunHandle>;
|
|
@@ -475,7 +492,7 @@ declare class JobRef {
|
|
|
475
492
|
declare class JobHandle extends JobRef {
|
|
476
493
|
readonly data: Job;
|
|
477
494
|
constructor(client: ZenRowsBatchClient, jobId: string, data: Job, submitResponse?: SubmitJobResponse);
|
|
478
|
-
get status(): JobStatus
|
|
495
|
+
get status(): Extensible<JobStatus>;
|
|
479
496
|
}
|
|
480
497
|
|
|
481
498
|
interface BatchClientConfig {
|
|
@@ -531,7 +548,16 @@ interface GetResultsOptions {
|
|
|
531
548
|
}
|
|
532
549
|
interface WaitForRunOptions {
|
|
533
550
|
runId?: string;
|
|
551
|
+
/**
|
|
552
|
+
* Statuses that resolve the wait. Defaults to every terminal status (`completed`,
|
|
553
|
+
* `stopped`, `failed`, `deleted`), so an auto-failed run resolves with its
|
|
554
|
+
* `failure_reason` / `failure_detail` instead of polling until `timeout`.
|
|
555
|
+
*/
|
|
534
556
|
targetStatuses?: Set<string>;
|
|
557
|
+
/**
|
|
558
|
+
* Statuses that reject with `WaiterFailureError`. Takes precedence over `targetStatuses`,
|
|
559
|
+
* so `new Set(["failed"])` makes a failed run throw. Empty by default.
|
|
560
|
+
*/
|
|
535
561
|
failureStatuses?: Set<string>;
|
|
536
562
|
timeout?: number;
|
|
537
563
|
pollInterval?: number;
|
|
@@ -541,6 +567,8 @@ type TaskInput = string | {
|
|
|
541
567
|
url: string;
|
|
542
568
|
external_id?: string;
|
|
543
569
|
metadata?: Record<string, unknown>;
|
|
570
|
+
/** Per-task scrape params, merged over the job-level `zenrowsParams` (task wins). */
|
|
571
|
+
zenrows_params?: Record<string, unknown>;
|
|
544
572
|
};
|
|
545
573
|
/**
|
|
546
574
|
* Client for the Zenrows Batch API (async, job/run/task model). Usable standalone
|
|
@@ -816,4 +844,4 @@ declare class ZenRows {
|
|
|
816
844
|
private worker;
|
|
817
845
|
}
|
|
818
846
|
|
|
819
|
-
export { AUTO_MAX_CREDITS, AUTO_MIN_CREDITS, type AddTasksResponse, At, BASE_CREDITS, type BatchClientConfig, type BatchTask, type BatchWebhookConfig, type Cadence, Calendar, type CostEstimate, type CostLine, type CreateJobInputResponse, CurrentRun, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_MAX_COUNT_IN_MEMORY, DEFAULT_MAX_FILES, DEFAULT_MAX_TOTAL_BYTES_IN_MEMORY, Daily, type DownloadToDirOptions, type DownloadToMemoryOptions, type DownloadedResult, type EstimateTask, type Export, ExportHandle, ExportRef, type ExportStatus, type ExtractMode, type FileInputUploadTarget, type GetResultsOptions, type HMACKeyCreated, type HMACKeyFinalized, type HMACKeyList, type HMACKeyMeta, type IngestStatus, JS_AND_PROXY_CREDITS, JS_CREDITS, type Job, JobHandle, JobRef, type JobSchedule, type JobStatus, type JobType, type ListJobRunsResponse, type ListJobsOptions, type ListJobsResponse, type ListResultsResponse, Monthly, PREMIUM_PROXY_CREDITS, type ParamMap, type ParamValue, type PauseState, type PollUntilOptions, type ProblemJson, Rate, type RerunJobResponse, type Run, RunHandle, RunRef, type RunStats, type RunStatus, type Schedule, type ScheduleState, type Spend, type StartExportResponse, type SubmitJobOptions, type SubmitJobResponse, type SubmitTypedOptions, type TaskHistoryEvent, type TaskHistoryResponse, type TaskLike, type TaskResult, type TaskSpend, type TaskStatus, type TestWebhookResponse, type Tier, type WaitForRunOptions, WaiterFailureError, WaiterTimeoutError, type WebhookConfig, Weekly, ZenRows, ZenRowsBatchClient, ZenRowsBatchError, type ZenRowsConfig, downloadTaskToFile, downloadTaskToMemory, downloadToDir, downloadToMemory, estimateCost, parseProblem, pollUntil };
|
|
847
|
+
export { AUTO_MAX_CREDITS, AUTO_MIN_CREDITS, type AddTasksResponse, At, BASE_CREDITS, type BatchClientConfig, type BatchTask, type BatchWebhookConfig, type Cadence, Calendar, type CostEstimate, type CostLine, type CreateJobInputResponse, CurrentRun, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_MAX_COUNT_IN_MEMORY, DEFAULT_MAX_FILES, DEFAULT_MAX_TOTAL_BYTES_IN_MEMORY, Daily, type DownloadToDirOptions, type DownloadToMemoryOptions, type DownloadedResult, type EstimateTask, type Export, ExportHandle, ExportRef, type ExportStatus, type Extensible, type ExtractMode, type FileInputUploadTarget, type GetResultsOptions, type HMACKeyCreated, type HMACKeyFinalized, type HMACKeyList, type HMACKeyMeta, type IngestStatus, JS_AND_PROXY_CREDITS, JS_CREDITS, type Job, JobHandle, JobRef, type JobSchedule, type JobStatus, type JobType, type ListJobRunsResponse, type ListJobsOptions, type ListJobsResponse, type ListResultsResponse, Monthly, PREMIUM_PROXY_CREDITS, type ParamMap, type ParamValue, type PauseState, type PollUntilOptions, type ProblemJson, Rate, type RerunJobResponse, type Run, type RunFailureReason, RunHandle, RunRef, type RunStats, type RunStatus, type Schedule, type ScheduleState, type Spend, type StartExportResponse, type SubmitJobOptions, type SubmitJobResponse, type SubmitTypedOptions, type TaskHistoryEvent, type TaskHistoryResponse, type TaskLike, type TaskResult, type TaskSpend, type TaskStatus, type TestWebhookResponse, type Tier, type WaitForRunOptions, WaiterFailureError, WaiterTimeoutError, type WebhookConfig, Weekly, ZenRows, ZenRowsBatchClient, ZenRowsBatchError, type ZenRowsConfig, downloadTaskToFile, downloadTaskToMemory, downloadToDir, downloadToMemory, estimateCost, parseProblem, pollUntil };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,14 +1,29 @@
|
|
|
1
1
|
import * as fetch_retry from 'fetch-retry';
|
|
2
2
|
import fastq from 'fastq';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* A server-extensible string enum: the listed values autocomplete, but the server may
|
|
6
|
+
* return values not listed (the Batch API marks these `x-extensible-enum`). Response fields
|
|
7
|
+
* use `Extensible<...>`, so callers must handle an unknown value. The named types stay closed
|
|
8
|
+
* so they keep working as request input.
|
|
9
|
+
*/
|
|
10
|
+
type Extensible<T extends string> = T | (string & {});
|
|
4
11
|
type JobType = "regular" | "scheduled";
|
|
5
12
|
type JobStatus = "open" | "closed" | "deleted";
|
|
6
13
|
type ScheduleState = "active" | "paused";
|
|
7
14
|
type RunStatus = "pending" | "running" | "completed" | "failed" | "stopped" | "deleted";
|
|
8
|
-
type TaskStatus = "pending" | "successful" | "failed";
|
|
15
|
+
type TaskStatus = "pending" | "processing" | "successful" | "failed";
|
|
9
16
|
type PauseState = "active" | "paused";
|
|
10
17
|
type IngestStatus = "pending" | "done";
|
|
11
18
|
type ExportStatus = "pending" | "running" | "completed" | "failed";
|
|
19
|
+
/**
|
|
20
|
+
* Why a run auto-failed (`status == "failed"`).
|
|
21
|
+
* - `insufficient_credits` — the account is out of credits.
|
|
22
|
+
* - `subscription_inactive` — the subscription is not active.
|
|
23
|
+
* - `api_key_cap_reached` — the job's API key reached one of its credit caps; resolves when
|
|
24
|
+
* that cap's window resets or the cap is raised. `Run.failure_detail` says which.
|
|
25
|
+
*/
|
|
26
|
+
type RunFailureReason = "insufficient_credits" | "subscription_inactive" | "api_key_cap_reached";
|
|
12
27
|
interface Spend {
|
|
13
28
|
credits: number;
|
|
14
29
|
cost: number;
|
|
@@ -53,19 +68,21 @@ interface Run {
|
|
|
53
68
|
run_id: string;
|
|
54
69
|
job_id: string;
|
|
55
70
|
run_sequence: number;
|
|
56
|
-
status: RunStatus
|
|
71
|
+
status: Extensible<RunStatus>;
|
|
57
72
|
stats: RunStats;
|
|
58
73
|
last_batch_received?: boolean;
|
|
59
74
|
pause_state?: PauseState;
|
|
60
|
-
ingest_status?: IngestStatus
|
|
61
|
-
failure_reason?:
|
|
75
|
+
ingest_status?: Extensible<IngestStatus>;
|
|
76
|
+
failure_reason?: Extensible<RunFailureReason>;
|
|
77
|
+
/** Human-readable explanation of `failure_reason` (e.g. which cap, and when it resets, UTC). */
|
|
78
|
+
failure_detail?: string | null;
|
|
62
79
|
created_at?: string;
|
|
63
80
|
updated_at?: string;
|
|
64
81
|
}
|
|
65
82
|
interface Job {
|
|
66
83
|
job_id: string;
|
|
67
84
|
type: JobType;
|
|
68
|
-
status: JobStatus
|
|
85
|
+
status: Extensible<JobStatus>;
|
|
69
86
|
format?: string;
|
|
70
87
|
zenrows_params?: Record<string, string>;
|
|
71
88
|
external_id?: string;
|
|
@@ -89,19 +106,19 @@ interface ListJobRunsResponse {
|
|
|
89
106
|
}
|
|
90
107
|
interface SubmitJobResponse {
|
|
91
108
|
job_id: string;
|
|
92
|
-
status: JobStatus
|
|
109
|
+
status: Extensible<JobStatus>;
|
|
93
110
|
latest_run?: Run;
|
|
94
111
|
accepted_tasks: number;
|
|
95
112
|
webhook?: WebhookConfig;
|
|
96
113
|
}
|
|
97
114
|
interface AddTasksResponse {
|
|
98
115
|
accepted_tasks: number;
|
|
99
|
-
job_status: JobStatus
|
|
116
|
+
job_status: Extensible<JobStatus>;
|
|
100
117
|
latest_run: Run;
|
|
101
118
|
}
|
|
102
119
|
interface RerunJobResponse {
|
|
103
120
|
job_id: string;
|
|
104
|
-
status: JobStatus
|
|
121
|
+
status: Extensible<JobStatus>;
|
|
105
122
|
latest_run: Run;
|
|
106
123
|
rerun_of?: string;
|
|
107
124
|
retried_tasks: number;
|
|
@@ -122,7 +139,7 @@ interface TaskResult {
|
|
|
122
139
|
url: string;
|
|
123
140
|
metadata?: Record<string, string>;
|
|
124
141
|
method?: "GET" | "POST";
|
|
125
|
-
status: TaskStatus
|
|
142
|
+
status: Extensible<TaskStatus>;
|
|
126
143
|
type?: string;
|
|
127
144
|
result_url?: string;
|
|
128
145
|
error?: ProblemJson;
|
|
@@ -145,13 +162,13 @@ interface TaskHistoryResponse {
|
|
|
145
162
|
}
|
|
146
163
|
interface StartExportResponse {
|
|
147
164
|
export_id: string;
|
|
148
|
-
status: ExportStatus
|
|
165
|
+
status: Extensible<ExportStatus>;
|
|
149
166
|
created_at: string;
|
|
150
167
|
expires_at: string;
|
|
151
168
|
}
|
|
152
169
|
interface Export {
|
|
153
170
|
export_id: string;
|
|
154
|
-
status: ExportStatus
|
|
171
|
+
status: Extensible<ExportStatus>;
|
|
155
172
|
error?: string;
|
|
156
173
|
download_url?: string;
|
|
157
174
|
created_at: string;
|
|
@@ -350,7 +367,7 @@ declare class RunRef {
|
|
|
350
367
|
declare class RunHandle extends RunRef {
|
|
351
368
|
readonly data: Run;
|
|
352
369
|
constructor(client: ZenRowsBatchClient, jobId: string, runId: string, data: Run);
|
|
353
|
-
get status(): RunStatus
|
|
370
|
+
get status(): Extensible<RunStatus>;
|
|
354
371
|
get stats(): RunStats;
|
|
355
372
|
}
|
|
356
373
|
declare class ExportRef {
|
|
@@ -372,7 +389,7 @@ declare class ExportRef {
|
|
|
372
389
|
declare class ExportHandle extends ExportRef {
|
|
373
390
|
readonly data: Export;
|
|
374
391
|
constructor(client: ZenRowsBatchClient, jobId: string, runId: string, exportId: string, data: Export, startResponse?: StartExportResponse);
|
|
375
|
-
get status(): ExportStatus
|
|
392
|
+
get status(): Extensible<ExportStatus>;
|
|
376
393
|
}
|
|
377
394
|
/** Operations on a job's CURRENT run, reached via `job.run`. */
|
|
378
395
|
declare class CurrentRun {
|
|
@@ -431,7 +448,7 @@ declare class JobRef {
|
|
|
431
448
|
/** Operations on the schedule — pause/resume/update (scheduled jobs only). */
|
|
432
449
|
get schedule(): ScheduleControls;
|
|
433
450
|
/** The job status from the submit response — only known on refs from a `submit*` call. */
|
|
434
|
-
get status(): JobStatus | undefined;
|
|
451
|
+
get status(): Extensible<JobStatus> | undefined;
|
|
435
452
|
get acceptedTasks(): number | undefined;
|
|
436
453
|
load(): Promise<JobHandle>;
|
|
437
454
|
close(): Promise<JobHandle>;
|
|
@@ -449,7 +466,7 @@ declare class JobRef {
|
|
|
449
466
|
lastBatch?: boolean;
|
|
450
467
|
}): Promise<{
|
|
451
468
|
accepted_tasks: number;
|
|
452
|
-
job_status: JobStatus
|
|
469
|
+
job_status: Extensible<JobStatus>;
|
|
453
470
|
latest_run: Run;
|
|
454
471
|
}>;
|
|
455
472
|
runs(pageSize?: number): AsyncGenerator<RunHandle>;
|
|
@@ -475,7 +492,7 @@ declare class JobRef {
|
|
|
475
492
|
declare class JobHandle extends JobRef {
|
|
476
493
|
readonly data: Job;
|
|
477
494
|
constructor(client: ZenRowsBatchClient, jobId: string, data: Job, submitResponse?: SubmitJobResponse);
|
|
478
|
-
get status(): JobStatus
|
|
495
|
+
get status(): Extensible<JobStatus>;
|
|
479
496
|
}
|
|
480
497
|
|
|
481
498
|
interface BatchClientConfig {
|
|
@@ -531,7 +548,16 @@ interface GetResultsOptions {
|
|
|
531
548
|
}
|
|
532
549
|
interface WaitForRunOptions {
|
|
533
550
|
runId?: string;
|
|
551
|
+
/**
|
|
552
|
+
* Statuses that resolve the wait. Defaults to every terminal status (`completed`,
|
|
553
|
+
* `stopped`, `failed`, `deleted`), so an auto-failed run resolves with its
|
|
554
|
+
* `failure_reason` / `failure_detail` instead of polling until `timeout`.
|
|
555
|
+
*/
|
|
534
556
|
targetStatuses?: Set<string>;
|
|
557
|
+
/**
|
|
558
|
+
* Statuses that reject with `WaiterFailureError`. Takes precedence over `targetStatuses`,
|
|
559
|
+
* so `new Set(["failed"])` makes a failed run throw. Empty by default.
|
|
560
|
+
*/
|
|
535
561
|
failureStatuses?: Set<string>;
|
|
536
562
|
timeout?: number;
|
|
537
563
|
pollInterval?: number;
|
|
@@ -541,6 +567,8 @@ type TaskInput = string | {
|
|
|
541
567
|
url: string;
|
|
542
568
|
external_id?: string;
|
|
543
569
|
metadata?: Record<string, unknown>;
|
|
570
|
+
/** Per-task scrape params, merged over the job-level `zenrowsParams` (task wins). */
|
|
571
|
+
zenrows_params?: Record<string, unknown>;
|
|
544
572
|
};
|
|
545
573
|
/**
|
|
546
574
|
* Client for the Zenrows Batch API (async, job/run/task model). Usable standalone
|
|
@@ -816,4 +844,4 @@ declare class ZenRows {
|
|
|
816
844
|
private worker;
|
|
817
845
|
}
|
|
818
846
|
|
|
819
|
-
export { AUTO_MAX_CREDITS, AUTO_MIN_CREDITS, type AddTasksResponse, At, BASE_CREDITS, type BatchClientConfig, type BatchTask, type BatchWebhookConfig, type Cadence, Calendar, type CostEstimate, type CostLine, type CreateJobInputResponse, CurrentRun, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_MAX_COUNT_IN_MEMORY, DEFAULT_MAX_FILES, DEFAULT_MAX_TOTAL_BYTES_IN_MEMORY, Daily, type DownloadToDirOptions, type DownloadToMemoryOptions, type DownloadedResult, type EstimateTask, type Export, ExportHandle, ExportRef, type ExportStatus, type ExtractMode, type FileInputUploadTarget, type GetResultsOptions, type HMACKeyCreated, type HMACKeyFinalized, type HMACKeyList, type HMACKeyMeta, type IngestStatus, JS_AND_PROXY_CREDITS, JS_CREDITS, type Job, JobHandle, JobRef, type JobSchedule, type JobStatus, type JobType, type ListJobRunsResponse, type ListJobsOptions, type ListJobsResponse, type ListResultsResponse, Monthly, PREMIUM_PROXY_CREDITS, type ParamMap, type ParamValue, type PauseState, type PollUntilOptions, type ProblemJson, Rate, type RerunJobResponse, type Run, RunHandle, RunRef, type RunStats, type RunStatus, type Schedule, type ScheduleState, type Spend, type StartExportResponse, type SubmitJobOptions, type SubmitJobResponse, type SubmitTypedOptions, type TaskHistoryEvent, type TaskHistoryResponse, type TaskLike, type TaskResult, type TaskSpend, type TaskStatus, type TestWebhookResponse, type Tier, type WaitForRunOptions, WaiterFailureError, WaiterTimeoutError, type WebhookConfig, Weekly, ZenRows, ZenRowsBatchClient, ZenRowsBatchError, type ZenRowsConfig, downloadTaskToFile, downloadTaskToMemory, downloadToDir, downloadToMemory, estimateCost, parseProblem, pollUntil };
|
|
847
|
+
export { AUTO_MAX_CREDITS, AUTO_MIN_CREDITS, type AddTasksResponse, At, BASE_CREDITS, type BatchClientConfig, type BatchTask, type BatchWebhookConfig, type Cadence, Calendar, type CostEstimate, type CostLine, type CreateJobInputResponse, CurrentRun, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_MAX_COUNT_IN_MEMORY, DEFAULT_MAX_FILES, DEFAULT_MAX_TOTAL_BYTES_IN_MEMORY, Daily, type DownloadToDirOptions, type DownloadToMemoryOptions, type DownloadedResult, type EstimateTask, type Export, ExportHandle, ExportRef, type ExportStatus, type Extensible, type ExtractMode, type FileInputUploadTarget, type GetResultsOptions, type HMACKeyCreated, type HMACKeyFinalized, type HMACKeyList, type HMACKeyMeta, type IngestStatus, JS_AND_PROXY_CREDITS, JS_CREDITS, type Job, JobHandle, JobRef, type JobSchedule, type JobStatus, type JobType, type ListJobRunsResponse, type ListJobsOptions, type ListJobsResponse, type ListResultsResponse, Monthly, PREMIUM_PROXY_CREDITS, type ParamMap, type ParamValue, type PauseState, type PollUntilOptions, type ProblemJson, Rate, type RerunJobResponse, type Run, type RunFailureReason, RunHandle, RunRef, type RunStats, type RunStatus, type Schedule, type ScheduleState, type Spend, type StartExportResponse, type SubmitJobOptions, type SubmitJobResponse, type SubmitTypedOptions, type TaskHistoryEvent, type TaskHistoryResponse, type TaskLike, type TaskResult, type TaskSpend, type TaskStatus, type TestWebhookResponse, type Tier, type WaitForRunOptions, WaiterFailureError, WaiterTimeoutError, type WebhookConfig, Weekly, ZenRows, ZenRowsBatchClient, ZenRowsBatchError, type ZenRowsConfig, downloadTaskToFile, downloadTaskToMemory, downloadToDir, downloadToMemory, estimateCost, parseProblem, pollUntil };
|
package/dist/index.js
CHANGED
|
@@ -5,7 +5,7 @@ import fetchRetry from "fetch-retry";
|
|
|
5
5
|
// package.json
|
|
6
6
|
var package_default = {
|
|
7
7
|
name: "zenrows",
|
|
8
|
-
version: "2.
|
|
8
|
+
version: "2.1.1",
|
|
9
9
|
description: "ZenRows Node SDK",
|
|
10
10
|
main: "./dist/index.cjs",
|
|
11
11
|
module: "./dist/index.js",
|
|
@@ -27,7 +27,15 @@ var package_default = {
|
|
|
27
27
|
format: "biome format --write .",
|
|
28
28
|
check: "biome check ."
|
|
29
29
|
},
|
|
30
|
-
keywords: [
|
|
30
|
+
keywords: [
|
|
31
|
+
"sdk",
|
|
32
|
+
"zenrows",
|
|
33
|
+
"scraping",
|
|
34
|
+
"proxy",
|
|
35
|
+
"data extraction",
|
|
36
|
+
"web scraping",
|
|
37
|
+
"web scraping api"
|
|
38
|
+
],
|
|
31
39
|
author: "ZenRows",
|
|
32
40
|
repository: {
|
|
33
41
|
type: "git",
|
|
@@ -771,7 +779,7 @@ async function pollUntil(fetch2, {
|
|
|
771
779
|
|
|
772
780
|
// src/batch/client.ts
|
|
773
781
|
var DEFAULT_BATCH_API_URL = "https://async.api.zenrows.com/v1";
|
|
774
|
-
var TERMINAL_RUN_STATUSES = /* @__PURE__ */ new Set(["completed", "stopped", "deleted"]);
|
|
782
|
+
var TERMINAL_RUN_STATUSES = /* @__PURE__ */ new Set(["completed", "stopped", "failed", "deleted"]);
|
|
775
783
|
var TERMINAL_EXPORT_STATUSES = /* @__PURE__ */ new Set(["completed", "failed"]);
|
|
776
784
|
function coerceUrl(item) {
|
|
777
785
|
return typeof item === "string" ? { url: item } : item;
|
|
@@ -1134,7 +1142,8 @@ var ZenRowsBatchClient = class {
|
|
|
1134
1142
|
return this._getRunData(jobId, options.runId);
|
|
1135
1143
|
};
|
|
1136
1144
|
return pollUntil(fetchRun, {
|
|
1137
|
-
|
|
1145
|
+
// A status the caller named as a failure throws, even when it is also a target.
|
|
1146
|
+
isDone: (run) => run !== void 0 && target.has(run.status) && !options.failureStatuses?.has(run.status),
|
|
1138
1147
|
isFailure: (run) => run !== void 0 && Boolean(options.failureStatuses?.has(run.status)),
|
|
1139
1148
|
timeout: options.timeout ?? 300,
|
|
1140
1149
|
initialInterval: options.pollInterval ?? 2,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "zenrows",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.1.1",
|
|
4
4
|
"description": "ZenRows Node SDK",
|
|
5
5
|
"main": "./dist/index.cjs",
|
|
6
6
|
"module": "./dist/index.js",
|
|
@@ -25,7 +25,11 @@
|
|
|
25
25
|
"keywords": [
|
|
26
26
|
"sdk",
|
|
27
27
|
"zenrows",
|
|
28
|
-
"scraping"
|
|
28
|
+
"scraping",
|
|
29
|
+
"proxy",
|
|
30
|
+
"data extraction",
|
|
31
|
+
"web scraping",
|
|
32
|
+
"web scraping api"
|
|
29
33
|
],
|
|
30
34
|
"author": "ZenRows",
|
|
31
35
|
"repository": {
|
package/vitest.config.ts
CHANGED
|
@@ -6,6 +6,8 @@ export default defineConfig({
|
|
|
6
6
|
globals: true,
|
|
7
7
|
include: ["tests/**/*.test.ts"],
|
|
8
8
|
setupFiles: ["tests/_setup.ts"],
|
|
9
|
+
// Type-level assertions (expectTypeOf) only run under typecheck.
|
|
10
|
+
typecheck: { enabled: true, tsconfig: "./tsconfig.test.json", include: ["tests/**/*.test.ts"] },
|
|
9
11
|
coverage: {
|
|
10
12
|
include: ["src/**"],
|
|
11
13
|
exclude: ["examples/**"],
|