scavio 0.23.0 → 0.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -6
- package/dist/index.cjs +10 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +16 -6
- package/dist/index.d.ts +16 -6
- package/dist/index.js +10 -3
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -126,7 +126,7 @@ map.data.truncated; // true when more URLs exist than `limit`
|
|
|
126
126
|
// Crawl: start a job, then poll it (polling and cancelling are free).
|
|
127
127
|
const job = await client.extract.crawl(
|
|
128
128
|
{ url: "https://example.com/docs", max_pages: 50, max_depth: 3, format: "markdown" },
|
|
129
|
-
{ idempotencyKey: "docs-crawl-2026-10-12" }, // optional
|
|
129
|
+
{ idempotencyKey: "docs-crawl-2026-10-12" }, // optional; one is generated when omitted
|
|
130
130
|
);
|
|
131
131
|
job.data.id; // "crawl_..."
|
|
132
132
|
job.data.status_url;
|
|
@@ -185,16 +185,21 @@ call. Results are kept for 3 days after the job finishes (`expires_at`), up to
|
|
|
185
185
|
**`wait: true`.** `crawl()` and `batch()` poll the status endpoint every
|
|
186
186
|
`pollInterval` ms (default 2,000) until the job is completed, failed or
|
|
187
187
|
cancelled, then page through every cursor and return
|
|
188
|
-
`{ data: { ...summary, results: [all], next_cursor: null } }
|
|
188
|
+
`{ data: { ...summary, results: [all], next_cursor: null }, created: {...} }`,
|
|
189
|
+
where `created` is the 202 job handle (including `webhook_secret`, which is
|
|
190
|
+
only returned on creation), as in the Python SDK (since 0.23.1). If the job is
|
|
189
191
|
still running after `waitTimeout` ms (default 10 minutes) they throw
|
|
190
192
|
`ScavioJobTimeoutError`; the job keeps running and the error carries
|
|
191
193
|
`jobId`, `jobType` and `statusUrl` so you can keep polling or cancel it. For
|
|
192
194
|
big crawls, prefer a webhook or your own polling loop.
|
|
193
195
|
|
|
194
|
-
**Retries and idempotency.** The SDK retries 429/5xx and network errors.
|
|
195
|
-
`
|
|
196
|
-
|
|
197
|
-
|
|
196
|
+
**Retries and idempotency.** The SDK retries 429/5xx and network errors.
|
|
197
|
+
`crawl()` and `batch()` always send an `Idempotency-Key`: yours when you pass
|
|
198
|
+
`idempotencyKey`, otherwise a random UUID generated for that call and reused on
|
|
199
|
+
every retry of it (since 0.23.1). Repeating a POST with the same key within 24
|
|
200
|
+
hours returns the original job, so a retried start never begins a second,
|
|
201
|
+
billed one. Pass your own key to dedupe across calls or processes. Plain
|
|
202
|
+
`extract()` and `map()` are not jobs and send no key.
|
|
198
203
|
|
|
199
204
|
#### Completion webhooks
|
|
200
205
|
|
package/dist/index.cjs
CHANGED
|
@@ -232,7 +232,7 @@ function parseRetryAfter(header) {
|
|
|
232
232
|
var BASE_URL = "https://api.scavio.dev";
|
|
233
233
|
var DEFAULT_TIMEOUT = 3e4;
|
|
234
234
|
var DEFAULT_MAX_RETRIES = 2;
|
|
235
|
-
var SDK_VERSION = true ? "0.23.
|
|
235
|
+
var SDK_VERSION = true ? "0.23.1" : "0.0.0";
|
|
236
236
|
var USER_AGENT = `scavio-js/${SDK_VERSION}`;
|
|
237
237
|
function isServerRuntime() {
|
|
238
238
|
const g = globalThis;
|
|
@@ -2999,6 +2999,12 @@ var COLLECT_PAGE_SIZE = 100;
|
|
|
2999
2999
|
function sleep2(ms) {
|
|
3000
3000
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3001
3001
|
}
|
|
3002
|
+
async function randomUUID() {
|
|
3003
|
+
const g = globalThis;
|
|
3004
|
+
if (typeof g.crypto?.randomUUID === "function") return g.crypto.randomUUID();
|
|
3005
|
+
const nodeCrypto = await import("crypto");
|
|
3006
|
+
return nodeCrypto.randomUUID();
|
|
3007
|
+
}
|
|
3002
3008
|
var ExtractNamespace = class {
|
|
3003
3009
|
constructor(client) {
|
|
3004
3010
|
this.client = client;
|
|
@@ -3038,14 +3044,15 @@ var ExtractNamespace = class {
|
|
|
3038
3044
|
return this.cancel("batch", id);
|
|
3039
3045
|
}
|
|
3040
3046
|
async start(type, body, request2) {
|
|
3041
|
-
const headers =
|
|
3047
|
+
const headers = { "Idempotency-Key": request2.idempotencyKey ?? await randomUUID() };
|
|
3042
3048
|
const created = await this.client._post(
|
|
3043
3049
|
`/api/v1/extract/${type}`,
|
|
3044
3050
|
body,
|
|
3045
3051
|
headers
|
|
3046
3052
|
);
|
|
3047
3053
|
if (!request2.wait) return created;
|
|
3048
|
-
|
|
3054
|
+
const done = await this.waitFor(type, created.data.id, created.data.status_url, request2);
|
|
3055
|
+
return { ...done, created: created.data };
|
|
3049
3056
|
}
|
|
3050
3057
|
async status(type, id, query = {}) {
|
|
3051
3058
|
return await this.client._get(`/api/v1/extract/${type}/${encodeURIComponent(id)}`, {
|