scavio 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -2
- package/dist/index.cjs +245 -32
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +364 -28
- package/dist/index.d.ts +364 -28
- package/dist/index.js +231 -31
- package/dist/index.js.map +1 -1
- package/package.json +5 -2
package/README.md
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
TypeScript SDK for the [Scavio API](https://scavio.dev?utm_source=npm-scavio-js&utm_medium=package&utm_campaign=readme) — real-time structured web
|
|
4
4
|
data across 36 platforms on one API key, plus `extract()` to read
|
|
5
|
-
any URL as clean Markdown.
|
|
5
|
+
any URL as clean Markdown, and `extract.map()` / `extract.crawl()` /
|
|
6
|
+
`extract.batch()` to map, crawl and batch-extract whole websites.
|
|
6
7
|
|
|
7
8
|
Structured JSON in, structured JSON out. No per-site parsers to maintain.
|
|
8
9
|
|
|
@@ -14,6 +15,7 @@ Structured JSON in, structured JSON out. No per-site parsers to maintain.
|
|
|
14
15
|
- **Ads transparency** — Google Ads Transparency Center, Meta Ad Library
|
|
15
16
|
- **Social and video** — YouTube, TikTok, Instagram, Threads, X, LinkedIn, Reddit, Kuaishou
|
|
16
17
|
- **Any other page** — `extract()` turns a URL into Markdown, plain text or raw HTML
|
|
18
|
+
- **Whole websites** — `extract.map()` lists a site's URLs, `extract.crawl()` crawls a site, `extract.batch()` extracts a list of URLs in one job
|
|
17
19
|
|
|
18
20
|
## Install
|
|
19
21
|
|
|
@@ -103,6 +105,129 @@ boilerplate stripped. `text` is that markdown flattened to plain text. URLs are
|
|
|
103
105
|
http(s) only, a bare host is upgraded to https, and loopback, private,
|
|
104
106
|
link-local and cloud-metadata hosts are rejected with a 400.
|
|
105
107
|
|
|
108
|
+
### Website map, crawl and batch extract (new in 0.23.0)
|
|
109
|
+
|
|
110
|
+
`extract` is also a namespace: calling it reads one page, exactly as before
|
|
111
|
+
(`client.extract({ url })` sends the same request it did in 0.22), and its
|
|
112
|
+
methods work on whole sites and URL lists, with each page returned exactly
|
|
113
|
+
like [Extract](https://scavio.dev/docs/extract?utm_source=npm-scavio-js&utm_medium=package&utm_campaign=readme).
|
|
114
|
+
|
|
115
|
+
```typescript
|
|
116
|
+
// Map: every URL on a site, from its sitemaps and start page. 1 credit.
|
|
117
|
+
const map = await client.extract.map({
|
|
118
|
+
url: "https://example.com",
|
|
119
|
+
limit: 500, // 1-5,000, default 1,000 (plan caps apply)
|
|
120
|
+
sitemap: "include", // "include" | "only" | "skip"
|
|
121
|
+
include_paths: ["/blog/**"], // optional path globs
|
|
122
|
+
});
|
|
123
|
+
map.data.links; // [{ url, source: "sitemap" | "page", lastmod? }]
|
|
124
|
+
map.data.truncated; // true when more URLs exist than `limit`
|
|
125
|
+
|
|
126
|
+
// Crawl: start a job, then poll it (polling and cancelling are free).
|
|
127
|
+
const job = await client.extract.crawl(
|
|
128
|
+
{ url: "https://example.com/docs", max_pages: 50, max_depth: 3, format: "markdown" },
|
|
129
|
+
{ idempotencyKey: "docs-crawl-2026-10-12" }, // optional, sent as Idempotency-Key
|
|
130
|
+
);
|
|
131
|
+
job.data.id; // "crawl_..."
|
|
132
|
+
job.data.status_url;
|
|
133
|
+
|
|
134
|
+
const page = await client.extract.crawlStatus(job.data.id, { limit: 25 });
|
|
135
|
+
page.data.status; // "queued" | "running" | "completed" | "failed" | "cancelled"
|
|
136
|
+
page.data.pages; // { discovered, queued, succeeded, failed, skipped }
|
|
137
|
+
page.data.results; // [{ url, depth, status, status_code, content, ... }]
|
|
138
|
+
page.data.next_cursor; // pass as { cursor } for the next page, null at the end
|
|
139
|
+
|
|
140
|
+
await client.extract.cancelCrawl(job.data.id); // 409 once the crawl has finished
|
|
141
|
+
|
|
142
|
+
// Or wait for it: polls until done and returns every result in one response.
|
|
143
|
+
const done = await client.extract.crawl(
|
|
144
|
+
{ url: "https://example.com/docs", max_pages: 20 },
|
|
145
|
+
{ wait: true, pollInterval: 2000, waitTimeout: 600_000 },
|
|
146
|
+
);
|
|
147
|
+
done.data.results.length;
|
|
148
|
+
|
|
149
|
+
// Batch: extract a list of URLs in one job (results in input order).
|
|
150
|
+
const batch = await client.extract.batch({
|
|
151
|
+
urls: ["https://example.com/a", "https://example.com/b"],
|
|
152
|
+
mode: "normal",
|
|
153
|
+
});
|
|
154
|
+
batch.data.invalid_urls; // rejected up front, never billed
|
|
155
|
+
await client.extract.batchStatus(batch.data.id);
|
|
156
|
+
await client.extract.cancelBatch(batch.data.id);
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
**Pricing.** Map costs 1 credit. Crawl and batch bill per **successful** page
|
|
160
|
+
at Extract's price for the tier: `normal` 1, `advanced` 1, `ultra` 2. Failed
|
|
161
|
+
and skipped pages are free, starting, polling and cancelling a job are free,
|
|
162
|
+
and credits are taken page by page: if your balance runs out the job stops
|
|
163
|
+
cleanly with `stopped_reason: "insufficient_credits"`.
|
|
164
|
+
|
|
165
|
+
**Plan limits.**
|
|
166
|
+
|
|
167
|
+
| Plan | Max pages per crawl | Active jobs | URLs per batch |
|
|
168
|
+
|------|--------------------:|------------:|---------------:|
|
|
169
|
+
| Free | 10 | 1 | 10 |
|
|
170
|
+
| PAYG | 100 | 1 | 50 |
|
|
171
|
+
| Project | 500 | 2 | 100 |
|
|
172
|
+
| Bootstrap | 2,000 | 3 | 500 |
|
|
173
|
+
| Startup | 5,000 | 5 | 1,000 |
|
|
174
|
+
| Growth | 10,000 | 10 | 1,000 |
|
|
175
|
+
|
|
176
|
+
Starting a job above your plan's active-job limit is a 429 (`RateLimitError`)
|
|
177
|
+
with `Retry-After`.
|
|
178
|
+
|
|
179
|
+
**Behavior.** Crawl and map always obey the site's robots.txt. A crawl stays on
|
|
180
|
+
the start URL's site (`allow_subdomains` opts into subdomains) and visits URLs
|
|
181
|
+
under the start path first. Batch treats each URL like a single `extract()`
|
|
182
|
+
call. Results are kept for 3 days after the job finishes (`expires_at`), up to
|
|
183
|
+
2 MB per page and 250 MB per job, and a job runs for at most 6 hours.
|
|
184
|
+
|
|
185
|
+
**`wait: true`.** `crawl()` and `batch()` poll the status endpoint every
|
|
186
|
+
`pollInterval` ms (default 2,000) until the job is completed, failed or
|
|
187
|
+
cancelled, then page through every cursor and return
|
|
188
|
+
`{ data: { ...summary, results: [all], next_cursor: null } }`. If the job is
|
|
189
|
+
still running after `waitTimeout` ms (default 10 minutes) they throw
|
|
190
|
+
`ScavioJobTimeoutError`; the job keeps running and the error carries
|
|
191
|
+
`jobId`, `jobType` and `statusUrl` so you can keep polling or cancel it. For
|
|
192
|
+
big crawls, prefer a webhook or your own polling loop.
|
|
193
|
+
|
|
194
|
+
**Retries and idempotency.** The SDK retries 429/5xx and network errors. Pass
|
|
195
|
+
`idempotencyKey` when starting a job: repeating a POST with the same key within
|
|
196
|
+
24 hours returns the original job, so a retried request never starts a second
|
|
197
|
+
one.
|
|
198
|
+
|
|
199
|
+
#### Completion webhooks
|
|
200
|
+
|
|
201
|
+
Pass `webhook_url` (https) to `crawl()` or `batch()` and Scavio POSTs the job
|
|
202
|
+
summary there when it completes, fails or is cancelled (retried at 0 s, 30 s
|
|
203
|
+
and 5 min). The signing secret comes back once, as `job.data.webhook_secret`.
|
|
204
|
+
Each delivery is signed with
|
|
205
|
+
`Scavio-Signature: t=<unix>,v1=<hex HMAC-SHA256 of "<t>.<raw body>">`; verify
|
|
206
|
+
it against the **raw** body with `verifyWebhook` (async, Web Crypto, so it runs
|
|
207
|
+
on Node 18+, Deno, Bun and edge runtimes):
|
|
208
|
+
|
|
209
|
+
```typescript
|
|
210
|
+
import { verifyWebhook } from "scavio";
|
|
211
|
+
|
|
212
|
+
// e.g. a Next.js / Hono / Workers route handler
|
|
213
|
+
export async function POST(req: Request) {
|
|
214
|
+
const raw = await req.text();
|
|
215
|
+
const ok = await verifyWebhook(
|
|
216
|
+
process.env.SCAVIO_WEBHOOK_SECRET!, // the job's webhook_secret
|
|
217
|
+
req.headers.get("Scavio-Signature"),
|
|
218
|
+
raw,
|
|
219
|
+
300, // max signature age in seconds (default)
|
|
220
|
+
);
|
|
221
|
+
if (!ok) return new Response("bad signature", { status: 401 });
|
|
222
|
+
const event = JSON.parse(raw);
|
|
223
|
+
// event.id, event.status, ...
|
|
224
|
+
return new Response("ok");
|
|
225
|
+
}
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
`verifyWebhook` returns `false` (never throws) for a missing, malformed,
|
|
229
|
+
tampered or stale signature.
|
|
230
|
+
|
|
106
231
|
### Google
|
|
107
232
|
|
|
108
233
|
Each method hits its own `/api/v2/google*` endpoint and returns Google's full
|
|
@@ -534,6 +659,10 @@ await client.tiktok.userPosts({ sec_user_id: "abc123", count: 30 });
|
|
|
534
659
|
// Video details
|
|
535
660
|
await client.tiktok.video({ video_id: "vid123" });
|
|
536
661
|
|
|
662
|
+
// Video transcript: timestamped segments and full text (video_id or url;
|
|
663
|
+
// optional 2-letter language). Caption-less videos return available: false.
|
|
664
|
+
await client.tiktok.videoTranscript({ video_id: "7609177768793787679" });
|
|
665
|
+
|
|
537
666
|
// Video comments
|
|
538
667
|
await client.tiktok.videoComments({ video_id: "vid123", count: 20 });
|
|
539
668
|
|
|
@@ -1225,7 +1354,8 @@ All error classes:
|
|
|
1225
1354
|
| `InsufficientCreditsError` | 402 | No credits remaining |
|
|
1226
1355
|
| `NotFoundError` | 404 | No data for that id (see TikTok Shop above) |
|
|
1227
1356
|
| `RateLimitError` | 429 | Rate limit exceeded |
|
|
1228
|
-
| `ScavioAPIError` | other | Catch-all (has `.statusCode
|
|
1357
|
+
| `ScavioAPIError` | other | Catch-all (has `.statusCode`; a 409 from cancelling a finished job lands here) |
|
|
1358
|
+
| `ScavioJobTimeoutError` | — | `crawl()`/`batch()` with `wait: true` outlived `waitTimeout`; carries `jobId` |
|
|
1229
1359
|
|
|
1230
1360
|
Threads and Kuaishou answer a missing or conflicting identifier with **422**, not
|
|
1231
1361
|
400 — those routes have no 400 at all. Both map to `BadRequestError`, so one
|
package/dist/index.cjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
2
3
|
var __defProp = Object.defineProperty;
|
|
3
4
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
5
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
5
7
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
8
|
var __export = (target, all) => {
|
|
7
9
|
for (var name in all)
|
|
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
15
17
|
}
|
|
16
18
|
return to;
|
|
17
19
|
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
18
28
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
29
|
|
|
20
30
|
// src/index.ts
|
|
@@ -30,6 +40,7 @@ __export(index_exports, {
|
|
|
30
40
|
CostcoNamespace: () => CostcoNamespace,
|
|
31
41
|
DEFAULT_MAX_REQUESTS_PER_SECOND: () => DEFAULT_MAX_REQUESTS_PER_SECOND,
|
|
32
42
|
EbayNamespace: () => EbayNamespace,
|
|
43
|
+
ExtractNamespace: () => ExtractNamespace,
|
|
33
44
|
G2Namespace: () => G2Namespace,
|
|
34
45
|
GlassdoorNamespace: () => GlassdoorNamespace,
|
|
35
46
|
GoogleAdsNamespace: () => GoogleAdsNamespace,
|
|
@@ -57,6 +68,7 @@ __export(index_exports, {
|
|
|
57
68
|
ScavioAPIError: () => ScavioAPIError,
|
|
58
69
|
ScavioConnectionError: () => ScavioConnectionError,
|
|
59
70
|
ScavioError: () => ScavioError,
|
|
71
|
+
ScavioJobTimeoutError: () => ScavioJobTimeoutError,
|
|
60
72
|
ScavioTimeoutError: () => ScavioTimeoutError,
|
|
61
73
|
TargetNamespace: () => TargetNamespace,
|
|
62
74
|
ThreadsNamespace: () => ThreadsNamespace,
|
|
@@ -68,7 +80,8 @@ __export(index_exports, {
|
|
|
68
80
|
XNamespace: () => XNamespace,
|
|
69
81
|
YelpNamespace: () => YelpNamespace,
|
|
70
82
|
YouTubeNamespace: () => YouTubeNamespace,
|
|
71
|
-
ZillowNamespace: () => ZillowNamespace
|
|
83
|
+
ZillowNamespace: () => ZillowNamespace,
|
|
84
|
+
verifyWebhook: () => verifyWebhook
|
|
72
85
|
});
|
|
73
86
|
module.exports = __toCommonJS(index_exports);
|
|
74
87
|
|
|
@@ -155,6 +168,22 @@ var ScavioAPIError = class extends ScavioError {
|
|
|
155
168
|
this.responseBody = responseBody;
|
|
156
169
|
}
|
|
157
170
|
};
|
|
171
|
+
var ScavioJobTimeoutError = class extends ScavioError {
|
|
172
|
+
constructor(jobType, jobId, statusUrl, lastStatus) {
|
|
173
|
+
super(
|
|
174
|
+
`${jobType} ${jobId} was still ${lastStatus} when the wait timed out. It keeps running: poll extract.${jobType}Status("${jobId}") or cancel it.`
|
|
175
|
+
);
|
|
176
|
+
this.jobType = jobType;
|
|
177
|
+
this.jobId = jobId;
|
|
178
|
+
this.statusUrl = statusUrl;
|
|
179
|
+
this.lastStatus = lastStatus;
|
|
180
|
+
this.name = "ScavioJobTimeoutError";
|
|
181
|
+
}
|
|
182
|
+
jobType;
|
|
183
|
+
jobId;
|
|
184
|
+
statusUrl;
|
|
185
|
+
lastStatus;
|
|
186
|
+
};
|
|
158
187
|
|
|
159
188
|
// src/retry.ts
|
|
160
189
|
var DEFAULT_RETRY_STATUSES = /* @__PURE__ */ new Set([
|
|
@@ -203,7 +232,7 @@ function parseRetryAfter(header) {
|
|
|
203
232
|
var BASE_URL = "https://api.scavio.dev";
|
|
204
233
|
var DEFAULT_TIMEOUT = 3e4;
|
|
205
234
|
var DEFAULT_MAX_RETRIES = 2;
|
|
206
|
-
var SDK_VERSION = true ? "0.
|
|
235
|
+
var SDK_VERSION = true ? "0.23.0" : "0.0.0";
|
|
207
236
|
var USER_AGENT = `scavio-js/${SDK_VERSION}`;
|
|
208
237
|
function isServerRuntime() {
|
|
209
238
|
const g = globalThis;
|
|
@@ -261,9 +290,18 @@ function getHeader(response, name) {
|
|
|
261
290
|
}
|
|
262
291
|
return null;
|
|
263
292
|
}
|
|
293
|
+
function buildQuery(query) {
|
|
294
|
+
if (!query) return "";
|
|
295
|
+
const params = new URLSearchParams();
|
|
296
|
+
for (const [key, value] of Object.entries(query)) {
|
|
297
|
+
if (value !== void 0) params.append(key, String(value));
|
|
298
|
+
}
|
|
299
|
+
const qs = params.toString();
|
|
300
|
+
return qs ? `?${qs}` : "";
|
|
301
|
+
}
|
|
264
302
|
async function request(options) {
|
|
265
|
-
const url = `${options.baseUrl}${options.path}`;
|
|
266
|
-
const headers = buildHeaders(options.apiKey);
|
|
303
|
+
const url = `${options.baseUrl}${options.path}${buildQuery(options.query)}`;
|
|
304
|
+
const headers = { ...buildHeaders(options.apiKey), ...options.headers };
|
|
267
305
|
const retry = makeRetryConfig(
|
|
268
306
|
options.maxRetries ?? DEFAULT_MAX_RETRIES
|
|
269
307
|
);
|
|
@@ -525,6 +563,14 @@ var TikTokNamespace = class {
|
|
|
525
563
|
async video(options) {
|
|
526
564
|
return this.client._post("/api/v1/tiktok/video", options);
|
|
527
565
|
}
|
|
566
|
+
/**
|
|
567
|
+
* Spoken-word transcript of a TikTok video, as timestamped segments and
|
|
568
|
+
* full text. Pass `video_id` or `url`. Videos with no spoken captions
|
|
569
|
+
* (music, dance, muted clips) return `available: false`. 1 credit.
|
|
570
|
+
*/
|
|
571
|
+
async videoTranscript(options) {
|
|
572
|
+
return this.client._post("/api/v1/tiktok/video/transcript", options);
|
|
573
|
+
}
|
|
528
574
|
async videoComments(options) {
|
|
529
575
|
return this.client._post("/api/v1/tiktok/video/comments", options);
|
|
530
576
|
}
|
|
@@ -2945,10 +2991,128 @@ var MercadoLibreNamespace = class {
|
|
|
2945
2991
|
}
|
|
2946
2992
|
};
|
|
2947
2993
|
|
|
2994
|
+
// src/namespaces/extract.ts
|
|
2995
|
+
var TERMINAL = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
|
|
2996
|
+
var DEFAULT_POLL_INTERVAL = 2e3;
|
|
2997
|
+
var DEFAULT_WAIT_TIMEOUT = 6e5;
|
|
2998
|
+
var COLLECT_PAGE_SIZE = 100;
|
|
2999
|
+
function sleep2(ms) {
|
|
3000
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3001
|
+
}
|
|
3002
|
+
var ExtractNamespace = class {
|
|
3003
|
+
constructor(client) {
|
|
3004
|
+
this.client = client;
|
|
3005
|
+
}
|
|
3006
|
+
client;
|
|
3007
|
+
/**
|
|
3008
|
+
* List a site's page URLs, gathered from its sitemaps and its start page,
|
|
3009
|
+
* deduplicated, filtered by path and limited. Pages disallowed by the
|
|
3010
|
+
* site's robots.txt are left out. 1 credit on the normal tier.
|
|
3011
|
+
*/
|
|
3012
|
+
async map(options) {
|
|
3013
|
+
return await this.client._post("/api/v1/extract/map", options);
|
|
3014
|
+
}
|
|
3015
|
+
async crawl(options, request2 = {}) {
|
|
3016
|
+
return this.start("crawl", options, request2);
|
|
3017
|
+
}
|
|
3018
|
+
/** A crawl's progress and its page results, paged by cursor. Free. */
|
|
3019
|
+
async crawlStatus(id, query) {
|
|
3020
|
+
return this.status("crawl", id, query);
|
|
3021
|
+
}
|
|
3022
|
+
/**
|
|
3023
|
+
* Stop a queued or running crawl. No further pages are fetched or billed;
|
|
3024
|
+
* results so far stay available. A finished crawl answers 409. Free.
|
|
3025
|
+
*/
|
|
3026
|
+
async cancelCrawl(id) {
|
|
3027
|
+
return this.cancel("crawl", id);
|
|
3028
|
+
}
|
|
3029
|
+
async batch(options, request2 = {}) {
|
|
3030
|
+
return this.start("batch", options, request2);
|
|
3031
|
+
}
|
|
3032
|
+
/** A batch's progress and its page results in input order, paged by cursor. Free. */
|
|
3033
|
+
async batchStatus(id, query) {
|
|
3034
|
+
return this.status("batch", id, query);
|
|
3035
|
+
}
|
|
3036
|
+
/** Stop a queued or running batch. A finished batch answers 409. Free. */
|
|
3037
|
+
async cancelBatch(id) {
|
|
3038
|
+
return this.cancel("batch", id);
|
|
3039
|
+
}
|
|
3040
|
+
async start(type, body, request2) {
|
|
3041
|
+
const headers = request2.idempotencyKey ? { "Idempotency-Key": request2.idempotencyKey } : void 0;
|
|
3042
|
+
const created = await this.client._post(
|
|
3043
|
+
`/api/v1/extract/${type}`,
|
|
3044
|
+
body,
|
|
3045
|
+
headers
|
|
3046
|
+
);
|
|
3047
|
+
if (!request2.wait) return created;
|
|
3048
|
+
return this.waitFor(type, created.data.id, created.data.status_url, request2);
|
|
3049
|
+
}
|
|
3050
|
+
async status(type, id, query = {}) {
|
|
3051
|
+
return await this.client._get(`/api/v1/extract/${type}/${encodeURIComponent(id)}`, {
|
|
3052
|
+
cursor: query.cursor,
|
|
3053
|
+
limit: query.limit,
|
|
3054
|
+
status: query.status,
|
|
3055
|
+
include_content: query.include_content
|
|
3056
|
+
});
|
|
3057
|
+
}
|
|
3058
|
+
async cancel(type, id) {
|
|
3059
|
+
return await this.client._delete(
|
|
3060
|
+
`/api/v1/extract/${type}/${encodeURIComponent(id)}`
|
|
3061
|
+
);
|
|
3062
|
+
}
|
|
3063
|
+
async waitFor(type, id, statusUrl, request2) {
|
|
3064
|
+
const interval = request2.pollInterval ?? DEFAULT_POLL_INTERVAL;
|
|
3065
|
+
const deadline = Date.now() + (request2.waitTimeout ?? DEFAULT_WAIT_TIMEOUT);
|
|
3066
|
+
for (; ; ) {
|
|
3067
|
+
const probe = await this.status(type, id, { limit: 1, include_content: false });
|
|
3068
|
+
if (TERMINAL.has(probe.data.status)) break;
|
|
3069
|
+
if (Date.now() + interval > deadline) {
|
|
3070
|
+
throw new ScavioJobTimeoutError(type, id, statusUrl, probe.data.status);
|
|
3071
|
+
}
|
|
3072
|
+
await sleep2(interval);
|
|
3073
|
+
}
|
|
3074
|
+
const results = [];
|
|
3075
|
+
let cursor;
|
|
3076
|
+
let last;
|
|
3077
|
+
do {
|
|
3078
|
+
last = await this.status(type, id, { cursor, limit: COLLECT_PAGE_SIZE });
|
|
3079
|
+
results.push(...last.data.results);
|
|
3080
|
+
cursor = last.data.next_cursor ?? void 0;
|
|
3081
|
+
} while (cursor);
|
|
3082
|
+
return { data: { ...last.data, results, next_cursor: null } };
|
|
3083
|
+
}
|
|
3084
|
+
};
|
|
3085
|
+
|
|
2948
3086
|
// src/client.ts
|
|
2949
3087
|
var DEFAULT_MAX_REQUESTS_PER_SECOND = 1;
|
|
2950
3088
|
var MAX_REQUESTS_PER_SECOND = 50;
|
|
2951
|
-
var
|
|
3089
|
+
var ScavioCore = class {
|
|
3090
|
+
/**
|
|
3091
|
+
* Read ANY web page and get it back as readability Markdown (the default),
|
|
3092
|
+
* plain text, or raw HTML. Returns `{ url, format, mode, content,
|
|
3093
|
+
* content_length }`.
|
|
3094
|
+
*
|
|
3095
|
+
* Extract is a core endpoint, not a platform: `scavio.extract({ url })`
|
|
3096
|
+
* reads one page, and `scavio.extract.map()` / `.crawl()` / `.batch()` work
|
|
3097
|
+
* on whole sites and URL lists.
|
|
3098
|
+
*
|
|
3099
|
+
* Credits are a function of `mode`, not a flat per-call constant:
|
|
3100
|
+
* "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
|
|
3101
|
+
* only on a successful extraction - a dead link, bot wall or timeout costs
|
|
3102
|
+
* nothing.
|
|
3103
|
+
*
|
|
3104
|
+
* Start on "normal". Move to "advanced" when the page builds its content in
|
|
3105
|
+
* the browser, and to "ultra" only when a bot wall blocks the other two.
|
|
3106
|
+
*
|
|
3107
|
+
* @example
|
|
3108
|
+
* const page = await scavio.extract({ url: "https://example.com/pricing" });
|
|
3109
|
+
* console.log(page.content);
|
|
3110
|
+
*/
|
|
3111
|
+
async extract(options) {
|
|
3112
|
+
return this._post("/api/v1/extract", options);
|
|
3113
|
+
}
|
|
3114
|
+
};
|
|
3115
|
+
var Scavio = class extends ScavioCore {
|
|
2952
3116
|
google;
|
|
2953
3117
|
amazon;
|
|
2954
3118
|
walmart;
|
|
@@ -2991,6 +3155,7 @@ var Scavio = class {
|
|
|
2991
3155
|
maxRetries;
|
|
2992
3156
|
rateLimiter;
|
|
2993
3157
|
constructor(config) {
|
|
3158
|
+
super();
|
|
2994
3159
|
this.apiKey = config?.apiKey ?? process.env.SCAVIO_API_KEY ?? "";
|
|
2995
3160
|
if (!this.apiKey) {
|
|
2996
3161
|
throw new MissingAPIKeyError();
|
|
@@ -3041,9 +3206,19 @@ var Scavio = class {
|
|
|
3041
3206
|
this.realtor = new RealtorNamespace(this);
|
|
3042
3207
|
this.rightmove = new RightmoveNamespace(this);
|
|
3043
3208
|
this.mercadolibre = new MercadoLibreNamespace(this);
|
|
3209
|
+
const extractNs = new ExtractNamespace(this);
|
|
3210
|
+
this.extract = Object.assign(ScavioCore.prototype.extract.bind(this), {
|
|
3211
|
+
map: extractNs.map.bind(extractNs),
|
|
3212
|
+
crawl: extractNs.crawl.bind(extractNs),
|
|
3213
|
+
crawlStatus: extractNs.crawlStatus.bind(extractNs),
|
|
3214
|
+
cancelCrawl: extractNs.cancelCrawl.bind(extractNs),
|
|
3215
|
+
batch: extractNs.batch.bind(extractNs),
|
|
3216
|
+
batchStatus: extractNs.batchStatus.bind(extractNs),
|
|
3217
|
+
cancelBatch: extractNs.cancelBatch.bind(extractNs)
|
|
3218
|
+
});
|
|
3044
3219
|
}
|
|
3045
3220
|
/** @internal */
|
|
3046
|
-
async _post(path, body) {
|
|
3221
|
+
async _post(path, body, headers) {
|
|
3047
3222
|
return request({
|
|
3048
3223
|
method: "POST",
|
|
3049
3224
|
path,
|
|
@@ -3052,11 +3227,12 @@ var Scavio = class {
|
|
|
3052
3227
|
timeout: this.timeout,
|
|
3053
3228
|
maxRetries: this.maxRetries,
|
|
3054
3229
|
rateLimiter: this.rateLimiter,
|
|
3055
|
-
body
|
|
3230
|
+
body,
|
|
3231
|
+
headers
|
|
3056
3232
|
});
|
|
3057
3233
|
}
|
|
3058
3234
|
/** @internal */
|
|
3059
|
-
async _get(path) {
|
|
3235
|
+
async _get(path, query) {
|
|
3060
3236
|
return request({
|
|
3061
3237
|
method: "GET",
|
|
3062
3238
|
path,
|
|
@@ -3064,39 +3240,73 @@ var Scavio = class {
|
|
|
3064
3240
|
baseUrl: this.baseUrl,
|
|
3065
3241
|
timeout: this.timeout,
|
|
3066
3242
|
maxRetries: this.maxRetries,
|
|
3243
|
+
rateLimiter: this.rateLimiter,
|
|
3244
|
+
query
|
|
3245
|
+
});
|
|
3246
|
+
}
|
|
3247
|
+
/** @internal */
|
|
3248
|
+
async _delete(path) {
|
|
3249
|
+
return request({
|
|
3250
|
+
method: "DELETE",
|
|
3251
|
+
path,
|
|
3252
|
+
apiKey: this.apiKey,
|
|
3253
|
+
baseUrl: this.baseUrl,
|
|
3254
|
+
timeout: this.timeout,
|
|
3255
|
+
maxRetries: this.maxRetries,
|
|
3067
3256
|
rateLimiter: this.rateLimiter
|
|
3068
3257
|
});
|
|
3069
3258
|
}
|
|
3070
3259
|
async search(options) {
|
|
3071
3260
|
return this.google.search(options);
|
|
3072
3261
|
}
|
|
3073
|
-
/**
|
|
3074
|
-
* Read ANY web page and get it back as readability Markdown (the default),
|
|
3075
|
-
* plain text, or raw HTML. Returns `{ url, format, mode, content,
|
|
3076
|
-
* content_length }`.
|
|
3077
|
-
*
|
|
3078
|
-
* This is a core endpoint, not a platform, so it lives on the client itself:
|
|
3079
|
-
* `scavio.extract({ url })`, never `scavio.extract.extract()`.
|
|
3080
|
-
*
|
|
3081
|
-
* Credits are a function of `mode`, not a flat per-call constant:
|
|
3082
|
-
* "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
|
|
3083
|
-
* only on a successful extraction - a dead link, bot wall or timeout costs
|
|
3084
|
-
* nothing.
|
|
3085
|
-
*
|
|
3086
|
-
* Start on "normal". Move to "advanced" when the page builds its content in
|
|
3087
|
-
* the browser, and to "ultra" only when a bot wall blocks the other two.
|
|
3088
|
-
*
|
|
3089
|
-
* @example
|
|
3090
|
-
* const page = await scavio.extract({ url: "https://example.com/pricing" });
|
|
3091
|
-
* console.log(page.content);
|
|
3092
|
-
*/
|
|
3093
|
-
async extract(options) {
|
|
3094
|
-
return this._post("/api/v1/extract", options);
|
|
3095
|
-
}
|
|
3096
3262
|
async getUsage() {
|
|
3097
3263
|
return this._get("/api/v1/usage");
|
|
3098
3264
|
}
|
|
3099
3265
|
};
|
|
3266
|
+
|
|
3267
|
+
// src/webhook.ts
|
|
3268
|
+
async function verifyWebhook(secret, header, body, toleranceSec = 300) {
|
|
3269
|
+
if (!secret || !header) return false;
|
|
3270
|
+
const parts = {};
|
|
3271
|
+
for (const piece of header.split(",")) {
|
|
3272
|
+
const i = piece.indexOf("=");
|
|
3273
|
+
if (i > 0) parts[piece.slice(0, i).trim()] = piece.slice(i + 1).trim();
|
|
3274
|
+
}
|
|
3275
|
+
const t = Number(parts.t);
|
|
3276
|
+
const v1 = parts.v1;
|
|
3277
|
+
if (!parts.t || !Number.isFinite(t) || !v1 || !/^[0-9a-f]+$/i.test(v1)) return false;
|
|
3278
|
+
if (Math.abs(Math.floor(Date.now() / 1e3) - t) > toleranceSec) return false;
|
|
3279
|
+
const subtle = await getSubtle();
|
|
3280
|
+
const enc = new TextEncoder();
|
|
3281
|
+
const key = await subtle.importKey(
|
|
3282
|
+
"raw",
|
|
3283
|
+
enc.encode(secret),
|
|
3284
|
+
{ name: "HMAC", hash: "SHA-256" },
|
|
3285
|
+
false,
|
|
3286
|
+
["sign"]
|
|
3287
|
+
);
|
|
3288
|
+
const bodyBytes = typeof body === "string" ? enc.encode(body) : body;
|
|
3289
|
+
const prefix = enc.encode(`${t}.`);
|
|
3290
|
+
const message = new Uint8Array(prefix.length + bodyBytes.length);
|
|
3291
|
+
message.set(prefix, 0);
|
|
3292
|
+
message.set(bodyBytes, prefix.length);
|
|
3293
|
+
const mac = new Uint8Array(await subtle.sign("HMAC", key, message));
|
|
3294
|
+
let expected = "";
|
|
3295
|
+
for (const b of mac) expected += b.toString(16).padStart(2, "0");
|
|
3296
|
+
return timingSafeEqualHex(expected, v1.toLowerCase());
|
|
3297
|
+
}
|
|
3298
|
+
function timingSafeEqualHex(a, b) {
|
|
3299
|
+
if (a.length !== b.length) return false;
|
|
3300
|
+
let diff = 0;
|
|
3301
|
+
for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
3302
|
+
return diff === 0;
|
|
3303
|
+
}
|
|
3304
|
+
async function getSubtle() {
|
|
3305
|
+
const g = globalThis;
|
|
3306
|
+
if (g.crypto?.subtle) return g.crypto.subtle;
|
|
3307
|
+
const nodeCrypto = await import("crypto");
|
|
3308
|
+
return nodeCrypto.webcrypto.subtle;
|
|
3309
|
+
}
|
|
3100
3310
|
// Annotate the CommonJS export names for ESM import in node:
|
|
3101
3311
|
0 && (module.exports = {
|
|
3102
3312
|
AirbnbNamespace,
|
|
@@ -3109,6 +3319,7 @@ var Scavio = class {
|
|
|
3109
3319
|
CostcoNamespace,
|
|
3110
3320
|
DEFAULT_MAX_REQUESTS_PER_SECOND,
|
|
3111
3321
|
EbayNamespace,
|
|
3322
|
+
ExtractNamespace,
|
|
3112
3323
|
G2Namespace,
|
|
3113
3324
|
GlassdoorNamespace,
|
|
3114
3325
|
GoogleAdsNamespace,
|
|
@@ -3136,6 +3347,7 @@ var Scavio = class {
|
|
|
3136
3347
|
ScavioAPIError,
|
|
3137
3348
|
ScavioConnectionError,
|
|
3138
3349
|
ScavioError,
|
|
3350
|
+
ScavioJobTimeoutError,
|
|
3139
3351
|
ScavioTimeoutError,
|
|
3140
3352
|
TargetNamespace,
|
|
3141
3353
|
ThreadsNamespace,
|
|
@@ -3147,6 +3359,7 @@ var Scavio = class {
|
|
|
3147
3359
|
XNamespace,
|
|
3148
3360
|
YelpNamespace,
|
|
3149
3361
|
YouTubeNamespace,
|
|
3150
|
-
ZillowNamespace
|
|
3362
|
+
ZillowNamespace,
|
|
3363
|
+
verifyWebhook
|
|
3151
3364
|
});
|
|
3152
3365
|
//# sourceMappingURL=index.cjs.map
|