scavio 0.22.0 → 0.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -2
- package/dist/index.cjs +252 -32
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +374 -28
- package/dist/index.d.ts +374 -28
- package/dist/index.js +238 -31
- package/dist/index.js.map +1 -1
- package/package.json +5 -2
package/README.md
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
TypeScript SDK for the [Scavio API](https://scavio.dev?utm_source=npm-scavio-js&utm_medium=package&utm_campaign=readme) — real-time structured web
|
|
4
4
|
data across 36 platforms on one API key, plus `extract()` to read
|
|
5
|
-
any URL as clean Markdown.
|
|
5
|
+
any URL as clean Markdown, and `extract.map()` / `extract.crawl()` /
|
|
6
|
+
`extract.batch()` to map, crawl and batch-extract whole websites.
|
|
6
7
|
|
|
7
8
|
Structured JSON in, structured JSON out. No per-site parsers to maintain.
|
|
8
9
|
|
|
@@ -14,6 +15,7 @@ Structured JSON in, structured JSON out. No per-site parsers to maintain.
|
|
|
14
15
|
- **Ads transparency** — Google Ads Transparency Center, Meta Ad Library
|
|
15
16
|
- **Social and video** — YouTube, TikTok, Instagram, Threads, X, LinkedIn, Reddit, Kuaishou
|
|
16
17
|
- **Any other page** — `extract()` turns a URL into Markdown, plain text or raw HTML
|
|
18
|
+
- **Whole websites** — `extract.map()` lists a site's URLs, `extract.crawl()` crawls a site, `extract.batch()` extracts a list of URLs in one job
|
|
17
19
|
|
|
18
20
|
## Install
|
|
19
21
|
|
|
@@ -103,6 +105,134 @@ boilerplate stripped. `text` is that markdown flattened to plain text. URLs are
|
|
|
103
105
|
http(s) only, a bare host is upgraded to https, and loopback, private,
|
|
104
106
|
link-local and cloud-metadata hosts are rejected with a 400.
|
|
105
107
|
|
|
108
|
+
### Website map, crawl and batch extract (new in 0.23.0)
|
|
109
|
+
|
|
110
|
+
`extract` is also a namespace: calling it reads one page, exactly as before
|
|
111
|
+
(`client.extract({ url })` sends the same request it did in 0.22), and its
|
|
112
|
+
methods work on whole sites and URL lists, with each page returned exactly
|
|
113
|
+
like [Extract](https://scavio.dev/docs/extract?utm_source=npm-scavio-js&utm_medium=package&utm_campaign=readme).
|
|
114
|
+
|
|
115
|
+
```typescript
|
|
116
|
+
// Map: every URL on a site, from its sitemaps and start page. 1 credit.
|
|
117
|
+
const map = await client.extract.map({
|
|
118
|
+
url: "https://example.com",
|
|
119
|
+
limit: 500, // 1-5,000, default 1,000 (plan caps apply)
|
|
120
|
+
sitemap: "include", // "include" | "only" | "skip"
|
|
121
|
+
include_paths: ["/blog/**"], // optional path globs
|
|
122
|
+
});
|
|
123
|
+
map.data.links; // [{ url, source: "sitemap" | "page", lastmod? }]
|
|
124
|
+
map.data.truncated; // true when more URLs exist than `limit`
|
|
125
|
+
|
|
126
|
+
// Crawl: start a job, then poll it (polling and cancelling are free).
|
|
127
|
+
const job = await client.extract.crawl(
|
|
128
|
+
{ url: "https://example.com/docs", max_pages: 50, max_depth: 3, format: "markdown" },
|
|
129
|
+
{ idempotencyKey: "docs-crawl-2026-10-12" }, // optional; one is generated when omitted
|
|
130
|
+
);
|
|
131
|
+
job.data.id; // "crawl_..."
|
|
132
|
+
job.data.status_url;
|
|
133
|
+
|
|
134
|
+
const page = await client.extract.crawlStatus(job.data.id, { limit: 25 });
|
|
135
|
+
page.data.status; // "queued" | "running" | "completed" | "failed" | "cancelled"
|
|
136
|
+
page.data.pages; // { discovered, queued, succeeded, failed, skipped }
|
|
137
|
+
page.data.results; // [{ url, depth, status, status_code, content, ... }]
|
|
138
|
+
page.data.next_cursor; // pass as { cursor } for the next page, null at the end
|
|
139
|
+
|
|
140
|
+
await client.extract.cancelCrawl(job.data.id); // 409 once the crawl has finished
|
|
141
|
+
|
|
142
|
+
// Or wait for it: polls until done and returns every result in one response.
|
|
143
|
+
const done = await client.extract.crawl(
|
|
144
|
+
{ url: "https://example.com/docs", max_pages: 20 },
|
|
145
|
+
{ wait: true, pollInterval: 2000, waitTimeout: 600_000 },
|
|
146
|
+
);
|
|
147
|
+
done.data.results.length;
|
|
148
|
+
|
|
149
|
+
// Batch: extract a list of URLs in one job (results in input order).
|
|
150
|
+
const batch = await client.extract.batch({
|
|
151
|
+
urls: ["https://example.com/a", "https://example.com/b"],
|
|
152
|
+
mode: "normal",
|
|
153
|
+
});
|
|
154
|
+
batch.data.invalid_urls; // rejected up front, never billed
|
|
155
|
+
await client.extract.batchStatus(batch.data.id);
|
|
156
|
+
await client.extract.cancelBatch(batch.data.id);
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
**Pricing.** Map costs 1 credit. Crawl and batch bill per **successful** page
|
|
160
|
+
at Extract's price for the tier: `normal` 1, `advanced` 1, `ultra` 2. Failed
|
|
161
|
+
and skipped pages are free, starting, polling and cancelling a job are free,
|
|
162
|
+
and credits are taken page by page: if your balance runs out the job stops
|
|
163
|
+
cleanly with `stopped_reason: "insufficient_credits"`.
|
|
164
|
+
|
|
165
|
+
**Plan limits.**
|
|
166
|
+
|
|
167
|
+
| Plan | Max pages per crawl | Active jobs | URLs per batch |
|
|
168
|
+
|------|--------------------:|------------:|---------------:|
|
|
169
|
+
| Free | 10 | 1 | 10 |
|
|
170
|
+
| PAYG | 100 | 1 | 50 |
|
|
171
|
+
| Project | 500 | 2 | 100 |
|
|
172
|
+
| Bootstrap | 2,000 | 3 | 500 |
|
|
173
|
+
| Startup | 5,000 | 5 | 1,000 |
|
|
174
|
+
| Growth | 10,000 | 10 | 1,000 |
|
|
175
|
+
|
|
176
|
+
Starting a job above your plan's active-job limit is a 429 (`RateLimitError`)
|
|
177
|
+
with `Retry-After`.
|
|
178
|
+
|
|
179
|
+
**Behavior.** Crawl and map always obey the site's robots.txt. A crawl stays on
|
|
180
|
+
the start URL's site (`allow_subdomains` opts into subdomains) and visits URLs
|
|
181
|
+
under the start path first. Batch treats each URL like a single `extract()`
|
|
182
|
+
call. Results are kept for 3 days after the job finishes (`expires_at`), up to
|
|
183
|
+
2 MB per page and 250 MB per job, and a job runs for at most 6 hours.
|
|
184
|
+
|
|
185
|
+
**`wait: true`.** `crawl()` and `batch()` poll the status endpoint every
|
|
186
|
+
`pollInterval` ms (default 2,000) until the job is completed, failed or
|
|
187
|
+
cancelled, then page through every cursor and return
|
|
188
|
+
`{ data: { ...summary, results: [all], next_cursor: null }, created: {...} }`,
|
|
189
|
+
where `created` is the 202 job handle (including `webhook_secret`, which is
|
|
190
|
+
only returned on creation), as in the Python SDK (since 0.23.1). If the job is
|
|
191
|
+
still running after `waitTimeout` ms (default 10 minutes) they throw
|
|
192
|
+
`ScavioJobTimeoutError`; the job keeps running and the error carries
|
|
193
|
+
`jobId`, `jobType` and `statusUrl` so you can keep polling or cancel it. For
|
|
194
|
+
big crawls, prefer a webhook or your own polling loop.
|
|
195
|
+
|
|
196
|
+
**Retries and idempotency.** The SDK retries 429/5xx and network errors.
|
|
197
|
+
`crawl()` and `batch()` always send an `Idempotency-Key`: yours when you pass
|
|
198
|
+
`idempotencyKey`, otherwise a random UUID generated for that call and reused on
|
|
199
|
+
every retry of it (since 0.23.1). Repeating a POST with the same key within 24
|
|
200
|
+
hours returns the original job, so a retried start never begins a second,
|
|
201
|
+
billed one. Pass your own key to dedupe across calls or processes. Plain
|
|
202
|
+
`extract()` and `map()` are not jobs and send no key.
|
|
203
|
+
|
|
204
|
+
#### Completion webhooks
|
|
205
|
+
|
|
206
|
+
Pass `webhook_url` (https) to `crawl()` or `batch()` and Scavio POSTs the job
|
|
207
|
+
summary there when it completes, fails or is cancelled (retried at 0 s, 30 s
|
|
208
|
+
and 5 min). The signing secret comes back once, as `job.data.webhook_secret`.
|
|
209
|
+
Each delivery is signed with
|
|
210
|
+
`Scavio-Signature: t=<unix>,v1=<hex HMAC-SHA256 of "<t>.<raw body>">`; verify
|
|
211
|
+
it against the **raw** body with `verifyWebhook` (async, Web Crypto, so it runs
|
|
212
|
+
on Node 18+, Deno, Bun and edge runtimes):
|
|
213
|
+
|
|
214
|
+
```typescript
|
|
215
|
+
import { verifyWebhook } from "scavio";
|
|
216
|
+
|
|
217
|
+
// e.g. a Next.js / Hono / Workers route handler
|
|
218
|
+
export async function POST(req: Request) {
|
|
219
|
+
const raw = await req.text();
|
|
220
|
+
const ok = await verifyWebhook(
|
|
221
|
+
process.env.SCAVIO_WEBHOOK_SECRET!, // the job's webhook_secret
|
|
222
|
+
req.headers.get("Scavio-Signature"),
|
|
223
|
+
raw,
|
|
224
|
+
300, // max signature age in seconds (default)
|
|
225
|
+
);
|
|
226
|
+
if (!ok) return new Response("bad signature", { status: 401 });
|
|
227
|
+
const event = JSON.parse(raw);
|
|
228
|
+
// event.id, event.status, ...
|
|
229
|
+
return new Response("ok");
|
|
230
|
+
}
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
`verifyWebhook` returns `false` (never throws) for a missing, malformed,
|
|
234
|
+
tampered or stale signature.
|
|
235
|
+
|
|
106
236
|
### Google
|
|
107
237
|
|
|
108
238
|
Each method hits its own `/api/v2/google*` endpoint and returns Google's full
|
|
@@ -534,6 +664,10 @@ await client.tiktok.userPosts({ sec_user_id: "abc123", count: 30 });
|
|
|
534
664
|
// Video details
|
|
535
665
|
await client.tiktok.video({ video_id: "vid123" });
|
|
536
666
|
|
|
667
|
+
// Video transcript: timestamped segments and full text (video_id or url;
|
|
668
|
+
// optional 2-letter language). Caption-less videos return available: false.
|
|
669
|
+
await client.tiktok.videoTranscript({ video_id: "7609177768793787679" });
|
|
670
|
+
|
|
537
671
|
// Video comments
|
|
538
672
|
await client.tiktok.videoComments({ video_id: "vid123", count: 20 });
|
|
539
673
|
|
|
@@ -1225,7 +1359,8 @@ All error classes:
|
|
|
1225
1359
|
| `InsufficientCreditsError` | 402 | No credits remaining |
|
|
1226
1360
|
| `NotFoundError` | 404 | No data for that id (see TikTok Shop above) |
|
|
1227
1361
|
| `RateLimitError` | 429 | Rate limit exceeded |
|
|
1228
|
-
| `ScavioAPIError` | other | Catch-all (has `.statusCode
|
|
1362
|
+
| `ScavioAPIError` | other | Catch-all (has `.statusCode`; a 409 from cancelling a finished job lands here) |
|
|
1363
|
+
| `ScavioJobTimeoutError` | — | `crawl()`/`batch()` with `wait: true` outlived `waitTimeout`; carries `jobId` |
|
|
1229
1364
|
|
|
1230
1365
|
Threads and Kuaishou answer a missing or conflicting identifier with **422**, not
|
|
1231
1366
|
400 — those routes have no 400 at all. Both map to `BadRequestError`, so one
|
package/dist/index.cjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
2
3
|
var __defProp = Object.defineProperty;
|
|
3
4
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
5
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
5
7
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
8
|
var __export = (target, all) => {
|
|
7
9
|
for (var name in all)
|
|
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
15
17
|
}
|
|
16
18
|
return to;
|
|
17
19
|
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
18
28
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
29
|
|
|
20
30
|
// src/index.ts
|
|
@@ -30,6 +40,7 @@ __export(index_exports, {
|
|
|
30
40
|
CostcoNamespace: () => CostcoNamespace,
|
|
31
41
|
DEFAULT_MAX_REQUESTS_PER_SECOND: () => DEFAULT_MAX_REQUESTS_PER_SECOND,
|
|
32
42
|
EbayNamespace: () => EbayNamespace,
|
|
43
|
+
ExtractNamespace: () => ExtractNamespace,
|
|
33
44
|
G2Namespace: () => G2Namespace,
|
|
34
45
|
GlassdoorNamespace: () => GlassdoorNamespace,
|
|
35
46
|
GoogleAdsNamespace: () => GoogleAdsNamespace,
|
|
@@ -57,6 +68,7 @@ __export(index_exports, {
|
|
|
57
68
|
ScavioAPIError: () => ScavioAPIError,
|
|
58
69
|
ScavioConnectionError: () => ScavioConnectionError,
|
|
59
70
|
ScavioError: () => ScavioError,
|
|
71
|
+
ScavioJobTimeoutError: () => ScavioJobTimeoutError,
|
|
60
72
|
ScavioTimeoutError: () => ScavioTimeoutError,
|
|
61
73
|
TargetNamespace: () => TargetNamespace,
|
|
62
74
|
ThreadsNamespace: () => ThreadsNamespace,
|
|
@@ -68,7 +80,8 @@ __export(index_exports, {
|
|
|
68
80
|
XNamespace: () => XNamespace,
|
|
69
81
|
YelpNamespace: () => YelpNamespace,
|
|
70
82
|
YouTubeNamespace: () => YouTubeNamespace,
|
|
71
|
-
ZillowNamespace: () => ZillowNamespace
|
|
83
|
+
ZillowNamespace: () => ZillowNamespace,
|
|
84
|
+
verifyWebhook: () => verifyWebhook
|
|
72
85
|
});
|
|
73
86
|
module.exports = __toCommonJS(index_exports);
|
|
74
87
|
|
|
@@ -155,6 +168,22 @@ var ScavioAPIError = class extends ScavioError {
|
|
|
155
168
|
this.responseBody = responseBody;
|
|
156
169
|
}
|
|
157
170
|
};
|
|
171
|
+
var ScavioJobTimeoutError = class extends ScavioError {
|
|
172
|
+
constructor(jobType, jobId, statusUrl, lastStatus) {
|
|
173
|
+
super(
|
|
174
|
+
`${jobType} ${jobId} was still ${lastStatus} when the wait timed out. It keeps running: poll extract.${jobType}Status("${jobId}") or cancel it.`
|
|
175
|
+
);
|
|
176
|
+
this.jobType = jobType;
|
|
177
|
+
this.jobId = jobId;
|
|
178
|
+
this.statusUrl = statusUrl;
|
|
179
|
+
this.lastStatus = lastStatus;
|
|
180
|
+
this.name = "ScavioJobTimeoutError";
|
|
181
|
+
}
|
|
182
|
+
jobType;
|
|
183
|
+
jobId;
|
|
184
|
+
statusUrl;
|
|
185
|
+
lastStatus;
|
|
186
|
+
};
|
|
158
187
|
|
|
159
188
|
// src/retry.ts
|
|
160
189
|
var DEFAULT_RETRY_STATUSES = /* @__PURE__ */ new Set([
|
|
@@ -203,7 +232,7 @@ function parseRetryAfter(header) {
|
|
|
203
232
|
var BASE_URL = "https://api.scavio.dev";
|
|
204
233
|
var DEFAULT_TIMEOUT = 3e4;
|
|
205
234
|
var DEFAULT_MAX_RETRIES = 2;
|
|
206
|
-
var SDK_VERSION = true ? "0.
|
|
235
|
+
var SDK_VERSION = true ? "0.23.1" : "0.0.0";
|
|
207
236
|
var USER_AGENT = `scavio-js/${SDK_VERSION}`;
|
|
208
237
|
function isServerRuntime() {
|
|
209
238
|
const g = globalThis;
|
|
@@ -261,9 +290,18 @@ function getHeader(response, name) {
|
|
|
261
290
|
}
|
|
262
291
|
return null;
|
|
263
292
|
}
|
|
293
|
+
function buildQuery(query) {
|
|
294
|
+
if (!query) return "";
|
|
295
|
+
const params = new URLSearchParams();
|
|
296
|
+
for (const [key, value] of Object.entries(query)) {
|
|
297
|
+
if (value !== void 0) params.append(key, String(value));
|
|
298
|
+
}
|
|
299
|
+
const qs = params.toString();
|
|
300
|
+
return qs ? `?${qs}` : "";
|
|
301
|
+
}
|
|
264
302
|
async function request(options) {
|
|
265
|
-
const url = `${options.baseUrl}${options.path}`;
|
|
266
|
-
const headers = buildHeaders(options.apiKey);
|
|
303
|
+
const url = `${options.baseUrl}${options.path}${buildQuery(options.query)}`;
|
|
304
|
+
const headers = { ...buildHeaders(options.apiKey), ...options.headers };
|
|
267
305
|
const retry = makeRetryConfig(
|
|
268
306
|
options.maxRetries ?? DEFAULT_MAX_RETRIES
|
|
269
307
|
);
|
|
@@ -525,6 +563,14 @@ var TikTokNamespace = class {
|
|
|
525
563
|
async video(options) {
|
|
526
564
|
return this.client._post("/api/v1/tiktok/video", options);
|
|
527
565
|
}
|
|
566
|
+
/**
|
|
567
|
+
* Spoken-word transcript of a TikTok video, as timestamped segments and
|
|
568
|
+
* full text. Pass `video_id` or `url`. Videos with no spoken captions
|
|
569
|
+
* (music, dance, muted clips) return `available: false`. 1 credit.
|
|
570
|
+
*/
|
|
571
|
+
async videoTranscript(options) {
|
|
572
|
+
return this.client._post("/api/v1/tiktok/video/transcript", options);
|
|
573
|
+
}
|
|
528
574
|
async videoComments(options) {
|
|
529
575
|
return this.client._post("/api/v1/tiktok/video/comments", options);
|
|
530
576
|
}
|
|
@@ -2945,10 +2991,135 @@ var MercadoLibreNamespace = class {
|
|
|
2945
2991
|
}
|
|
2946
2992
|
};
|
|
2947
2993
|
|
|
2994
|
+
// src/namespaces/extract.ts
|
|
2995
|
+
var TERMINAL = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
|
|
2996
|
+
var DEFAULT_POLL_INTERVAL = 2e3;
|
|
2997
|
+
var DEFAULT_WAIT_TIMEOUT = 6e5;
|
|
2998
|
+
var COLLECT_PAGE_SIZE = 100;
|
|
2999
|
+
function sleep2(ms) {
|
|
3000
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3001
|
+
}
|
|
3002
|
+
async function randomUUID() {
|
|
3003
|
+
const g = globalThis;
|
|
3004
|
+
if (typeof g.crypto?.randomUUID === "function") return g.crypto.randomUUID();
|
|
3005
|
+
const nodeCrypto = await import("crypto");
|
|
3006
|
+
return nodeCrypto.randomUUID();
|
|
3007
|
+
}
|
|
3008
|
+
var ExtractNamespace = class {
|
|
3009
|
+
constructor(client) {
|
|
3010
|
+
this.client = client;
|
|
3011
|
+
}
|
|
3012
|
+
client;
|
|
3013
|
+
/**
|
|
3014
|
+
* List a site's page URLs, gathered from its sitemaps and its start page,
|
|
3015
|
+
* deduplicated, filtered by path and limited. Pages disallowed by the
|
|
3016
|
+
* site's robots.txt are left out. 1 credit on the normal tier.
|
|
3017
|
+
*/
|
|
3018
|
+
async map(options) {
|
|
3019
|
+
return await this.client._post("/api/v1/extract/map", options);
|
|
3020
|
+
}
|
|
3021
|
+
async crawl(options, request2 = {}) {
|
|
3022
|
+
return this.start("crawl", options, request2);
|
|
3023
|
+
}
|
|
3024
|
+
/** A crawl's progress and its page results, paged by cursor. Free. */
|
|
3025
|
+
async crawlStatus(id, query) {
|
|
3026
|
+
return this.status("crawl", id, query);
|
|
3027
|
+
}
|
|
3028
|
+
/**
|
|
3029
|
+
* Stop a queued or running crawl. No further pages are fetched or billed;
|
|
3030
|
+
* results so far stay available. A finished crawl answers 409. Free.
|
|
3031
|
+
*/
|
|
3032
|
+
async cancelCrawl(id) {
|
|
3033
|
+
return this.cancel("crawl", id);
|
|
3034
|
+
}
|
|
3035
|
+
async batch(options, request2 = {}) {
|
|
3036
|
+
return this.start("batch", options, request2);
|
|
3037
|
+
}
|
|
3038
|
+
/** A batch's progress and its page results in input order, paged by cursor. Free. */
|
|
3039
|
+
async batchStatus(id, query) {
|
|
3040
|
+
return this.status("batch", id, query);
|
|
3041
|
+
}
|
|
3042
|
+
/** Stop a queued or running batch. A finished batch answers 409. Free. */
|
|
3043
|
+
async cancelBatch(id) {
|
|
3044
|
+
return this.cancel("batch", id);
|
|
3045
|
+
}
|
|
3046
|
+
async start(type, body, request2) {
|
|
3047
|
+
const headers = { "Idempotency-Key": request2.idempotencyKey ?? await randomUUID() };
|
|
3048
|
+
const created = await this.client._post(
|
|
3049
|
+
`/api/v1/extract/${type}`,
|
|
3050
|
+
body,
|
|
3051
|
+
headers
|
|
3052
|
+
);
|
|
3053
|
+
if (!request2.wait) return created;
|
|
3054
|
+
const done = await this.waitFor(type, created.data.id, created.data.status_url, request2);
|
|
3055
|
+
return { ...done, created: created.data };
|
|
3056
|
+
}
|
|
3057
|
+
async status(type, id, query = {}) {
|
|
3058
|
+
return await this.client._get(`/api/v1/extract/${type}/${encodeURIComponent(id)}`, {
|
|
3059
|
+
cursor: query.cursor,
|
|
3060
|
+
limit: query.limit,
|
|
3061
|
+
status: query.status,
|
|
3062
|
+
include_content: query.include_content
|
|
3063
|
+
});
|
|
3064
|
+
}
|
|
3065
|
+
async cancel(type, id) {
|
|
3066
|
+
return await this.client._delete(
|
|
3067
|
+
`/api/v1/extract/${type}/${encodeURIComponent(id)}`
|
|
3068
|
+
);
|
|
3069
|
+
}
|
|
3070
|
+
async waitFor(type, id, statusUrl, request2) {
|
|
3071
|
+
const interval = request2.pollInterval ?? DEFAULT_POLL_INTERVAL;
|
|
3072
|
+
const deadline = Date.now() + (request2.waitTimeout ?? DEFAULT_WAIT_TIMEOUT);
|
|
3073
|
+
for (; ; ) {
|
|
3074
|
+
const probe = await this.status(type, id, { limit: 1, include_content: false });
|
|
3075
|
+
if (TERMINAL.has(probe.data.status)) break;
|
|
3076
|
+
if (Date.now() + interval > deadline) {
|
|
3077
|
+
throw new ScavioJobTimeoutError(type, id, statusUrl, probe.data.status);
|
|
3078
|
+
}
|
|
3079
|
+
await sleep2(interval);
|
|
3080
|
+
}
|
|
3081
|
+
const results = [];
|
|
3082
|
+
let cursor;
|
|
3083
|
+
let last;
|
|
3084
|
+
do {
|
|
3085
|
+
last = await this.status(type, id, { cursor, limit: COLLECT_PAGE_SIZE });
|
|
3086
|
+
results.push(...last.data.results);
|
|
3087
|
+
cursor = last.data.next_cursor ?? void 0;
|
|
3088
|
+
} while (cursor);
|
|
3089
|
+
return { data: { ...last.data, results, next_cursor: null } };
|
|
3090
|
+
}
|
|
3091
|
+
};
|
|
3092
|
+
|
|
2948
3093
|
// src/client.ts
|
|
2949
3094
|
var DEFAULT_MAX_REQUESTS_PER_SECOND = 1;
|
|
2950
3095
|
var MAX_REQUESTS_PER_SECOND = 50;
|
|
2951
|
-
var
|
|
3096
|
+
var ScavioCore = class {
|
|
3097
|
+
/**
|
|
3098
|
+
* Read ANY web page and get it back as readability Markdown (the default),
|
|
3099
|
+
* plain text, or raw HTML. Returns `{ url, format, mode, content,
|
|
3100
|
+
* content_length }`.
|
|
3101
|
+
*
|
|
3102
|
+
* Extract is a core endpoint, not a platform: `scavio.extract({ url })`
|
|
3103
|
+
* reads one page, and `scavio.extract.map()` / `.crawl()` / `.batch()` work
|
|
3104
|
+
* on whole sites and URL lists.
|
|
3105
|
+
*
|
|
3106
|
+
* Credits are a function of `mode`, not a flat per-call constant:
|
|
3107
|
+
* "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
|
|
3108
|
+
* only on a successful extraction - a dead link, bot wall or timeout costs
|
|
3109
|
+
* nothing.
|
|
3110
|
+
*
|
|
3111
|
+
* Start on "normal". Move to "advanced" when the page builds its content in
|
|
3112
|
+
* the browser, and to "ultra" only when a bot wall blocks the other two.
|
|
3113
|
+
*
|
|
3114
|
+
* @example
|
|
3115
|
+
* const page = await scavio.extract({ url: "https://example.com/pricing" });
|
|
3116
|
+
* console.log(page.content);
|
|
3117
|
+
*/
|
|
3118
|
+
async extract(options) {
|
|
3119
|
+
return this._post("/api/v1/extract", options);
|
|
3120
|
+
}
|
|
3121
|
+
};
|
|
3122
|
+
var Scavio = class extends ScavioCore {
|
|
2952
3123
|
google;
|
|
2953
3124
|
amazon;
|
|
2954
3125
|
walmart;
|
|
@@ -2991,6 +3162,7 @@ var Scavio = class {
|
|
|
2991
3162
|
maxRetries;
|
|
2992
3163
|
rateLimiter;
|
|
2993
3164
|
constructor(config) {
|
|
3165
|
+
super();
|
|
2994
3166
|
this.apiKey = config?.apiKey ?? process.env.SCAVIO_API_KEY ?? "";
|
|
2995
3167
|
if (!this.apiKey) {
|
|
2996
3168
|
throw new MissingAPIKeyError();
|
|
@@ -3041,9 +3213,19 @@ var Scavio = class {
|
|
|
3041
3213
|
this.realtor = new RealtorNamespace(this);
|
|
3042
3214
|
this.rightmove = new RightmoveNamespace(this);
|
|
3043
3215
|
this.mercadolibre = new MercadoLibreNamespace(this);
|
|
3216
|
+
const extractNs = new ExtractNamespace(this);
|
|
3217
|
+
this.extract = Object.assign(ScavioCore.prototype.extract.bind(this), {
|
|
3218
|
+
map: extractNs.map.bind(extractNs),
|
|
3219
|
+
crawl: extractNs.crawl.bind(extractNs),
|
|
3220
|
+
crawlStatus: extractNs.crawlStatus.bind(extractNs),
|
|
3221
|
+
cancelCrawl: extractNs.cancelCrawl.bind(extractNs),
|
|
3222
|
+
batch: extractNs.batch.bind(extractNs),
|
|
3223
|
+
batchStatus: extractNs.batchStatus.bind(extractNs),
|
|
3224
|
+
cancelBatch: extractNs.cancelBatch.bind(extractNs)
|
|
3225
|
+
});
|
|
3044
3226
|
}
|
|
3045
3227
|
/** @internal */
|
|
3046
|
-
async _post(path, body) {
|
|
3228
|
+
async _post(path, body, headers) {
|
|
3047
3229
|
return request({
|
|
3048
3230
|
method: "POST",
|
|
3049
3231
|
path,
|
|
@@ -3052,11 +3234,12 @@ var Scavio = class {
|
|
|
3052
3234
|
timeout: this.timeout,
|
|
3053
3235
|
maxRetries: this.maxRetries,
|
|
3054
3236
|
rateLimiter: this.rateLimiter,
|
|
3055
|
-
body
|
|
3237
|
+
body,
|
|
3238
|
+
headers
|
|
3056
3239
|
});
|
|
3057
3240
|
}
|
|
3058
3241
|
/** @internal */
|
|
3059
|
-
async _get(path) {
|
|
3242
|
+
async _get(path, query) {
|
|
3060
3243
|
return request({
|
|
3061
3244
|
method: "GET",
|
|
3062
3245
|
path,
|
|
@@ -3064,39 +3247,73 @@ var Scavio = class {
|
|
|
3064
3247
|
baseUrl: this.baseUrl,
|
|
3065
3248
|
timeout: this.timeout,
|
|
3066
3249
|
maxRetries: this.maxRetries,
|
|
3250
|
+
rateLimiter: this.rateLimiter,
|
|
3251
|
+
query
|
|
3252
|
+
});
|
|
3253
|
+
}
|
|
3254
|
+
/** @internal */
|
|
3255
|
+
async _delete(path) {
|
|
3256
|
+
return request({
|
|
3257
|
+
method: "DELETE",
|
|
3258
|
+
path,
|
|
3259
|
+
apiKey: this.apiKey,
|
|
3260
|
+
baseUrl: this.baseUrl,
|
|
3261
|
+
timeout: this.timeout,
|
|
3262
|
+
maxRetries: this.maxRetries,
|
|
3067
3263
|
rateLimiter: this.rateLimiter
|
|
3068
3264
|
});
|
|
3069
3265
|
}
|
|
3070
3266
|
async search(options) {
|
|
3071
3267
|
return this.google.search(options);
|
|
3072
3268
|
}
|
|
3073
|
-
/**
|
|
3074
|
-
* Read ANY web page and get it back as readability Markdown (the default),
|
|
3075
|
-
* plain text, or raw HTML. Returns `{ url, format, mode, content,
|
|
3076
|
-
* content_length }`.
|
|
3077
|
-
*
|
|
3078
|
-
* This is a core endpoint, not a platform, so it lives on the client itself:
|
|
3079
|
-
* `scavio.extract({ url })`, never `scavio.extract.extract()`.
|
|
3080
|
-
*
|
|
3081
|
-
* Credits are a function of `mode`, not a flat per-call constant:
|
|
3082
|
-
* "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
|
|
3083
|
-
* only on a successful extraction - a dead link, bot wall or timeout costs
|
|
3084
|
-
* nothing.
|
|
3085
|
-
*
|
|
3086
|
-
* Start on "normal". Move to "advanced" when the page builds its content in
|
|
3087
|
-
* the browser, and to "ultra" only when a bot wall blocks the other two.
|
|
3088
|
-
*
|
|
3089
|
-
* @example
|
|
3090
|
-
* const page = await scavio.extract({ url: "https://example.com/pricing" });
|
|
3091
|
-
* console.log(page.content);
|
|
3092
|
-
*/
|
|
3093
|
-
async extract(options) {
|
|
3094
|
-
return this._post("/api/v1/extract", options);
|
|
3095
|
-
}
|
|
3096
3269
|
async getUsage() {
|
|
3097
3270
|
return this._get("/api/v1/usage");
|
|
3098
3271
|
}
|
|
3099
3272
|
};
|
|
3273
|
+
|
|
3274
|
+
// src/webhook.ts
|
|
3275
|
+
async function verifyWebhook(secret, header, body, toleranceSec = 300) {
|
|
3276
|
+
if (!secret || !header) return false;
|
|
3277
|
+
const parts = {};
|
|
3278
|
+
for (const piece of header.split(",")) {
|
|
3279
|
+
const i = piece.indexOf("=");
|
|
3280
|
+
if (i > 0) parts[piece.slice(0, i).trim()] = piece.slice(i + 1).trim();
|
|
3281
|
+
}
|
|
3282
|
+
const t = Number(parts.t);
|
|
3283
|
+
const v1 = parts.v1;
|
|
3284
|
+
if (!parts.t || !Number.isFinite(t) || !v1 || !/^[0-9a-f]+$/i.test(v1)) return false;
|
|
3285
|
+
if (Math.abs(Math.floor(Date.now() / 1e3) - t) > toleranceSec) return false;
|
|
3286
|
+
const subtle = await getSubtle();
|
|
3287
|
+
const enc = new TextEncoder();
|
|
3288
|
+
const key = await subtle.importKey(
|
|
3289
|
+
"raw",
|
|
3290
|
+
enc.encode(secret),
|
|
3291
|
+
{ name: "HMAC", hash: "SHA-256" },
|
|
3292
|
+
false,
|
|
3293
|
+
["sign"]
|
|
3294
|
+
);
|
|
3295
|
+
const bodyBytes = typeof body === "string" ? enc.encode(body) : body;
|
|
3296
|
+
const prefix = enc.encode(`${t}.`);
|
|
3297
|
+
const message = new Uint8Array(prefix.length + bodyBytes.length);
|
|
3298
|
+
message.set(prefix, 0);
|
|
3299
|
+
message.set(bodyBytes, prefix.length);
|
|
3300
|
+
const mac = new Uint8Array(await subtle.sign("HMAC", key, message));
|
|
3301
|
+
let expected = "";
|
|
3302
|
+
for (const b of mac) expected += b.toString(16).padStart(2, "0");
|
|
3303
|
+
return timingSafeEqualHex(expected, v1.toLowerCase());
|
|
3304
|
+
}
|
|
3305
|
+
function timingSafeEqualHex(a, b) {
|
|
3306
|
+
if (a.length !== b.length) return false;
|
|
3307
|
+
let diff = 0;
|
|
3308
|
+
for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
3309
|
+
return diff === 0;
|
|
3310
|
+
}
|
|
3311
|
+
async function getSubtle() {
|
|
3312
|
+
const g = globalThis;
|
|
3313
|
+
if (g.crypto?.subtle) return g.crypto.subtle;
|
|
3314
|
+
const nodeCrypto = await import("crypto");
|
|
3315
|
+
return nodeCrypto.webcrypto.subtle;
|
|
3316
|
+
}
|
|
3100
3317
|
// Annotate the CommonJS export names for ESM import in node:
|
|
3101
3318
|
0 && (module.exports = {
|
|
3102
3319
|
AirbnbNamespace,
|
|
@@ -3109,6 +3326,7 @@ var Scavio = class {
|
|
|
3109
3326
|
CostcoNamespace,
|
|
3110
3327
|
DEFAULT_MAX_REQUESTS_PER_SECOND,
|
|
3111
3328
|
EbayNamespace,
|
|
3329
|
+
ExtractNamespace,
|
|
3112
3330
|
G2Namespace,
|
|
3113
3331
|
GlassdoorNamespace,
|
|
3114
3332
|
GoogleAdsNamespace,
|
|
@@ -3136,6 +3354,7 @@ var Scavio = class {
|
|
|
3136
3354
|
ScavioAPIError,
|
|
3137
3355
|
ScavioConnectionError,
|
|
3138
3356
|
ScavioError,
|
|
3357
|
+
ScavioJobTimeoutError,
|
|
3139
3358
|
ScavioTimeoutError,
|
|
3140
3359
|
TargetNamespace,
|
|
3141
3360
|
ThreadsNamespace,
|
|
@@ -3147,6 +3366,7 @@ var Scavio = class {
|
|
|
3147
3366
|
XNamespace,
|
|
3148
3367
|
YelpNamespace,
|
|
3149
3368
|
YouTubeNamespace,
|
|
3150
|
-
ZillowNamespace
|
|
3369
|
+
ZillowNamespace,
|
|
3370
|
+
verifyWebhook
|
|
3151
3371
|
});
|
|
3152
3372
|
//# sourceMappingURL=index.cjs.map
|