scavio 0.22.0 → 0.23.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -2,7 +2,8 @@
2
2
 
3
3
  TypeScript SDK for the [Scavio API](https://scavio.dev?utm_source=npm-scavio-js&utm_medium=package&utm_campaign=readme) — real-time structured web
4
4
  data across 36 platforms on one API key, plus `extract()` to read
5
- any URL as clean Markdown.
5
+ any URL as clean Markdown, and `extract.map()` / `extract.crawl()` /
6
+ `extract.batch()` to map, crawl and batch-extract whole websites.
6
7
 
7
8
  Structured JSON in, structured JSON out. No per-site parsers to maintain.
8
9
 
@@ -14,6 +15,7 @@ Structured JSON in, structured JSON out. No per-site parsers to maintain.
14
15
  - **Ads transparency** — Google Ads Transparency Center, Meta Ad Library
15
16
  - **Social and video** — YouTube, TikTok, Instagram, Threads, X, LinkedIn, Reddit, Kuaishou
16
17
  - **Any other page** — `extract()` turns a URL into Markdown, plain text or raw HTML
18
+ - **Whole websites** — `extract.map()` lists a site's URLs, `extract.crawl()` crawls a site, `extract.batch()` extracts a list of URLs in one job
17
19
 
18
20
  ## Install
19
21
 
@@ -103,6 +105,134 @@ boilerplate stripped. `text` is that markdown flattened to plain text. URLs are
103
105
  http(s) only, a bare host is upgraded to https, and loopback, private,
104
106
  link-local and cloud-metadata hosts are rejected with a 400.
105
107
 
108
+ ### Website map, crawl and batch extract (new in 0.23.0)
109
+
110
+ `extract` is also a namespace: calling it reads one page, exactly as before
111
+ (`client.extract({ url })` sends the same request it did in 0.22), and its
112
+ methods work on whole sites and URL lists, with each page returned exactly
113
+ like [Extract](https://scavio.dev/docs/extract?utm_source=npm-scavio-js&utm_medium=package&utm_campaign=readme).
114
+
115
+ ```typescript
116
+ // Map: every URL on a site, from its sitemaps and start page. 1 credit.
117
+ const map = await client.extract.map({
118
+ url: "https://example.com",
119
+ limit: 500, // 1-5,000, default 1,000 (plan caps apply)
120
+ sitemap: "include", // "include" | "only" | "skip"
121
+ include_paths: ["/blog/**"], // optional path globs
122
+ });
123
+ map.data.links; // [{ url, source: "sitemap" | "page", lastmod? }]
124
+ map.data.truncated; // true when more URLs exist than `limit`
125
+
126
+ // Crawl: start a job, then poll it (polling and cancelling are free).
127
+ const job = await client.extract.crawl(
128
+ { url: "https://example.com/docs", max_pages: 50, max_depth: 3, format: "markdown" },
129
+ { idempotencyKey: "docs-crawl-2026-10-12" }, // optional; one is generated when omitted
130
+ );
131
+ job.data.id; // "crawl_..."
132
+ job.data.status_url;
133
+
134
+ const page = await client.extract.crawlStatus(job.data.id, { limit: 25 });
135
+ page.data.status; // "queued" | "running" | "completed" | "failed" | "cancelled"
136
+ page.data.pages; // { discovered, queued, succeeded, failed, skipped }
137
+ page.data.results; // [{ url, depth, status, status_code, content, ... }]
138
+ page.data.next_cursor; // pass as { cursor } for the next page, null at the end
139
+
140
+ await client.extract.cancelCrawl(job.data.id); // 409 once the crawl has finished
141
+
142
+ // Or wait for it: polls until done and returns every result in one response.
143
+ const done = await client.extract.crawl(
144
+ { url: "https://example.com/docs", max_pages: 20 },
145
+ { wait: true, pollInterval: 2000, waitTimeout: 600_000 },
146
+ );
147
+ done.data.results.length;
148
+
149
+ // Batch: extract a list of URLs in one job (results in input order).
150
+ const batch = await client.extract.batch({
151
+ urls: ["https://example.com/a", "https://example.com/b"],
152
+ mode: "normal",
153
+ });
154
+ batch.data.invalid_urls; // rejected up front, never billed
155
+ await client.extract.batchStatus(batch.data.id);
156
+ await client.extract.cancelBatch(batch.data.id);
157
+ ```
158
+
159
+ **Pricing.** Map costs 1 credit. Crawl and batch bill per **successful** page
160
+ at Extract's price for the tier: `normal` 1, `advanced` 1, `ultra` 2. Failed
161
+ and skipped pages are free, starting, polling and cancelling a job are free,
162
+ and credits are taken page by page: if your balance runs out the job stops
163
+ cleanly with `stopped_reason: "insufficient_credits"`.
164
+
165
+ **Plan limits.**
166
+
167
+ | Plan | Max pages per crawl | Active jobs | URLs per batch |
168
+ |------|--------------------:|------------:|---------------:|
169
+ | Free | 10 | 1 | 10 |
170
+ | PAYG | 100 | 1 | 50 |
171
+ | Project | 500 | 2 | 100 |
172
+ | Bootstrap | 2,000 | 3 | 500 |
173
+ | Startup | 5,000 | 5 | 1,000 |
174
+ | Growth | 10,000 | 10 | 1,000 |
175
+
176
+ Starting a job above your plan's active-job limit is a 429 (`RateLimitError`)
177
+ with `Retry-After`.
178
+
179
+ **Behavior.** Crawl and map always obey the site's robots.txt. A crawl stays on
180
+ the start URL's site (`allow_subdomains` opts into subdomains) and visits URLs
181
+ under the start path first. Batch treats each URL like a single `extract()`
182
+ call. Results are kept for 3 days after the job finishes (`expires_at`), up to
183
+ 2 MB per page and 250 MB per job, and a job runs for at most 6 hours.
184
+
185
+ **`wait: true`.** `crawl()` and `batch()` poll the status endpoint every
186
+ `pollInterval` ms (default 2,000) until the job is completed, failed or
187
+ cancelled, then page through every cursor and return
188
+ `{ data: { ...summary, results: [all], next_cursor: null }, created: {...} }`,
189
+ where `created` is the 202 job handle (including `webhook_secret`, which is
190
+ only returned on creation), as in the Python SDK (since 0.23.1). If the job is
191
+ still running after `waitTimeout` ms (default 10 minutes) they throw
192
+ `ScavioJobTimeoutError`; the job keeps running and the error carries
193
+ `jobId`, `jobType` and `statusUrl` so you can keep polling or cancel it. For
194
+ big crawls, prefer a webhook or your own polling loop.
195
+
196
+ **Retries and idempotency.** The SDK retries 429/5xx and network errors.
197
+ `crawl()` and `batch()` always send an `Idempotency-Key`: yours when you pass
198
+ `idempotencyKey`, otherwise a random UUID generated for that call and reused on
199
+ every retry of it (since 0.23.1). Repeating a POST with the same key within 24
200
+ hours returns the original job, so a retried start never begins a second,
201
+ billed one. Pass your own key to dedupe across calls or processes. Plain
202
+ `extract()` and `map()` are not jobs and send no key.
203
+
204
+ #### Completion webhooks
205
+
206
+ Pass `webhook_url` (https) to `crawl()` or `batch()` and Scavio POSTs the job
207
+ summary there when it completes, fails or is cancelled (retried at 0 s, 30 s
208
+ and 5 min). The signing secret comes back once, as `job.data.webhook_secret`.
209
+ Each delivery is signed with
210
+ `Scavio-Signature: t=<unix>,v1=<hex HMAC-SHA256 of "<t>.<raw body>">`; verify
211
+ it against the **raw** body with `verifyWebhook` (async, Web Crypto, so it runs
212
+ on Node 18+, Deno, Bun and edge runtimes):
213
+
214
+ ```typescript
215
+ import { verifyWebhook } from "scavio";
216
+
217
+ // e.g. a Next.js / Hono / Workers route handler
218
+ export async function POST(req: Request) {
219
+ const raw = await req.text();
220
+ const ok = await verifyWebhook(
221
+ process.env.SCAVIO_WEBHOOK_SECRET!, // the job's webhook_secret
222
+ req.headers.get("Scavio-Signature"),
223
+ raw,
224
+ 300, // max signature age in seconds (default)
225
+ );
226
+ if (!ok) return new Response("bad signature", { status: 401 });
227
+ const event = JSON.parse(raw);
228
+ // event.id, event.status, ...
229
+ return new Response("ok");
230
+ }
231
+ ```
232
+
233
+ `verifyWebhook` returns `false` (never throws) for a missing, malformed,
234
+ tampered or stale signature.
235
+
106
236
  ### Google
107
237
 
108
238
  Each method hits its own `/api/v2/google*` endpoint and returns Google's full
@@ -534,6 +664,10 @@ await client.tiktok.userPosts({ sec_user_id: "abc123", count: 30 });
534
664
  // Video details
535
665
  await client.tiktok.video({ video_id: "vid123" });
536
666
 
667
+ // Video transcript: timestamped segments and full text (video_id or url;
668
+ // optional 2-letter language). Caption-less videos return available: false.
669
+ await client.tiktok.videoTranscript({ video_id: "7609177768793787679" });
670
+
537
671
  // Video comments
538
672
  await client.tiktok.videoComments({ video_id: "vid123", count: 20 });
539
673
 
@@ -1225,7 +1359,8 @@ All error classes:
1225
1359
  | `InsufficientCreditsError` | 402 | No credits remaining |
1226
1360
  | `NotFoundError` | 404 | No data for that id (see TikTok Shop above) |
1227
1361
  | `RateLimitError` | 429 | Rate limit exceeded |
1228
- | `ScavioAPIError` | other | Catch-all (has `.statusCode`) |
1362
+ | `ScavioAPIError` | other | Catch-all (has `.statusCode`; a 409 from cancelling a finished job lands here) |
1363
+ | `ScavioJobTimeoutError` | — | `crawl()`/`batch()` with `wait: true` outlived `waitTimeout`; carries `jobId` |
1229
1364
 
1230
1365
  Threads and Kuaishou answer a missing or conflicting identifier with **422**, not
1231
1366
  400 — those routes have no 400 at all. Both map to `BadRequestError`, so one
package/dist/index.cjs CHANGED
@@ -1,7 +1,9 @@
1
1
  "use strict";
2
+ var __create = Object.create;
2
3
  var __defProp = Object.defineProperty;
3
4
  var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
5
  var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
5
7
  var __hasOwnProp = Object.prototype.hasOwnProperty;
6
8
  var __export = (target, all) => {
7
9
  for (var name in all)
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
15
17
  }
16
18
  return to;
17
19
  };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
18
28
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
29
 
20
30
  // src/index.ts
@@ -30,6 +40,7 @@ __export(index_exports, {
30
40
  CostcoNamespace: () => CostcoNamespace,
31
41
  DEFAULT_MAX_REQUESTS_PER_SECOND: () => DEFAULT_MAX_REQUESTS_PER_SECOND,
32
42
  EbayNamespace: () => EbayNamespace,
43
+ ExtractNamespace: () => ExtractNamespace,
33
44
  G2Namespace: () => G2Namespace,
34
45
  GlassdoorNamespace: () => GlassdoorNamespace,
35
46
  GoogleAdsNamespace: () => GoogleAdsNamespace,
@@ -57,6 +68,7 @@ __export(index_exports, {
57
68
  ScavioAPIError: () => ScavioAPIError,
58
69
  ScavioConnectionError: () => ScavioConnectionError,
59
70
  ScavioError: () => ScavioError,
71
+ ScavioJobTimeoutError: () => ScavioJobTimeoutError,
60
72
  ScavioTimeoutError: () => ScavioTimeoutError,
61
73
  TargetNamespace: () => TargetNamespace,
62
74
  ThreadsNamespace: () => ThreadsNamespace,
@@ -68,7 +80,8 @@ __export(index_exports, {
68
80
  XNamespace: () => XNamespace,
69
81
  YelpNamespace: () => YelpNamespace,
70
82
  YouTubeNamespace: () => YouTubeNamespace,
71
- ZillowNamespace: () => ZillowNamespace
83
+ ZillowNamespace: () => ZillowNamespace,
84
+ verifyWebhook: () => verifyWebhook
72
85
  });
73
86
  module.exports = __toCommonJS(index_exports);
74
87
 
@@ -155,6 +168,22 @@ var ScavioAPIError = class extends ScavioError {
155
168
  this.responseBody = responseBody;
156
169
  }
157
170
  };
171
+ var ScavioJobTimeoutError = class extends ScavioError {
172
+ constructor(jobType, jobId, statusUrl, lastStatus) {
173
+ super(
174
+ `${jobType} ${jobId} was still ${lastStatus} when the wait timed out. It keeps running: poll extract.${jobType}Status("${jobId}") or cancel it.`
175
+ );
176
+ this.jobType = jobType;
177
+ this.jobId = jobId;
178
+ this.statusUrl = statusUrl;
179
+ this.lastStatus = lastStatus;
180
+ this.name = "ScavioJobTimeoutError";
181
+ }
182
+ jobType;
183
+ jobId;
184
+ statusUrl;
185
+ lastStatus;
186
+ };
158
187
 
159
188
  // src/retry.ts
160
189
  var DEFAULT_RETRY_STATUSES = /* @__PURE__ */ new Set([
@@ -203,7 +232,7 @@ function parseRetryAfter(header) {
203
232
  var BASE_URL = "https://api.scavio.dev";
204
233
  var DEFAULT_TIMEOUT = 3e4;
205
234
  var DEFAULT_MAX_RETRIES = 2;
206
- var SDK_VERSION = true ? "0.22.0" : "0.0.0";
235
+ var SDK_VERSION = true ? "0.23.1" : "0.0.0";
207
236
  var USER_AGENT = `scavio-js/${SDK_VERSION}`;
208
237
  function isServerRuntime() {
209
238
  const g = globalThis;
@@ -261,9 +290,18 @@ function getHeader(response, name) {
261
290
  }
262
291
  return null;
263
292
  }
293
+ function buildQuery(query) {
294
+ if (!query) return "";
295
+ const params = new URLSearchParams();
296
+ for (const [key, value] of Object.entries(query)) {
297
+ if (value !== void 0) params.append(key, String(value));
298
+ }
299
+ const qs = params.toString();
300
+ return qs ? `?${qs}` : "";
301
+ }
264
302
  async function request(options) {
265
- const url = `${options.baseUrl}${options.path}`;
266
- const headers = buildHeaders(options.apiKey);
303
+ const url = `${options.baseUrl}${options.path}${buildQuery(options.query)}`;
304
+ const headers = { ...buildHeaders(options.apiKey), ...options.headers };
267
305
  const retry = makeRetryConfig(
268
306
  options.maxRetries ?? DEFAULT_MAX_RETRIES
269
307
  );
@@ -525,6 +563,14 @@ var TikTokNamespace = class {
525
563
  async video(options) {
526
564
  return this.client._post("/api/v1/tiktok/video", options);
527
565
  }
566
+ /**
567
+ * Spoken-word transcript of a TikTok video, as timestamped segments and
568
+ * full text. Pass `video_id` or `url`. Videos with no spoken captions
569
+ * (music, dance, muted clips) return `available: false`. 1 credit.
570
+ */
571
+ async videoTranscript(options) {
572
+ return this.client._post("/api/v1/tiktok/video/transcript", options);
573
+ }
528
574
  async videoComments(options) {
529
575
  return this.client._post("/api/v1/tiktok/video/comments", options);
530
576
  }
@@ -2945,10 +2991,135 @@ var MercadoLibreNamespace = class {
2945
2991
  }
2946
2992
  };
2947
2993
 
2994
+ // src/namespaces/extract.ts
2995
+ var TERMINAL = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
2996
+ var DEFAULT_POLL_INTERVAL = 2e3;
2997
+ var DEFAULT_WAIT_TIMEOUT = 6e5;
2998
+ var COLLECT_PAGE_SIZE = 100;
2999
+ function sleep2(ms) {
3000
+ return new Promise((resolve) => setTimeout(resolve, ms));
3001
+ }
3002
+ async function randomUUID() {
3003
+ const g = globalThis;
3004
+ if (typeof g.crypto?.randomUUID === "function") return g.crypto.randomUUID();
3005
+ const nodeCrypto = await import("crypto");
3006
+ return nodeCrypto.randomUUID();
3007
+ }
3008
+ var ExtractNamespace = class {
3009
+ constructor(client) {
3010
+ this.client = client;
3011
+ }
3012
+ client;
3013
+ /**
3014
+ * List a site's page URLs, gathered from its sitemaps and its start page,
3015
+ * deduplicated, filtered by path and limited. Pages disallowed by the
3016
+ * site's robots.txt are left out. 1 credit on the normal tier.
3017
+ */
3018
+ async map(options) {
3019
+ return await this.client._post("/api/v1/extract/map", options);
3020
+ }
3021
+ async crawl(options, request2 = {}) {
3022
+ return this.start("crawl", options, request2);
3023
+ }
3024
+ /** A crawl's progress and its page results, paged by cursor. Free. */
3025
+ async crawlStatus(id, query) {
3026
+ return this.status("crawl", id, query);
3027
+ }
3028
+ /**
3029
+ * Stop a queued or running crawl. No further pages are fetched or billed;
3030
+ * results so far stay available. A finished crawl answers 409. Free.
3031
+ */
3032
+ async cancelCrawl(id) {
3033
+ return this.cancel("crawl", id);
3034
+ }
3035
+ async batch(options, request2 = {}) {
3036
+ return this.start("batch", options, request2);
3037
+ }
3038
+ /** A batch's progress and its page results in input order, paged by cursor. Free. */
3039
+ async batchStatus(id, query) {
3040
+ return this.status("batch", id, query);
3041
+ }
3042
+ /** Stop a queued or running batch. A finished batch answers 409. Free. */
3043
+ async cancelBatch(id) {
3044
+ return this.cancel("batch", id);
3045
+ }
3046
+ async start(type, body, request2) {
3047
+ const headers = { "Idempotency-Key": request2.idempotencyKey ?? await randomUUID() };
3048
+ const created = await this.client._post(
3049
+ `/api/v1/extract/${type}`,
3050
+ body,
3051
+ headers
3052
+ );
3053
+ if (!request2.wait) return created;
3054
+ const done = await this.waitFor(type, created.data.id, created.data.status_url, request2);
3055
+ return { ...done, created: created.data };
3056
+ }
3057
+ async status(type, id, query = {}) {
3058
+ return await this.client._get(`/api/v1/extract/${type}/${encodeURIComponent(id)}`, {
3059
+ cursor: query.cursor,
3060
+ limit: query.limit,
3061
+ status: query.status,
3062
+ include_content: query.include_content
3063
+ });
3064
+ }
3065
+ async cancel(type, id) {
3066
+ return await this.client._delete(
3067
+ `/api/v1/extract/${type}/${encodeURIComponent(id)}`
3068
+ );
3069
+ }
3070
+ async waitFor(type, id, statusUrl, request2) {
3071
+ const interval = request2.pollInterval ?? DEFAULT_POLL_INTERVAL;
3072
+ const deadline = Date.now() + (request2.waitTimeout ?? DEFAULT_WAIT_TIMEOUT);
3073
+ for (; ; ) {
3074
+ const probe = await this.status(type, id, { limit: 1, include_content: false });
3075
+ if (TERMINAL.has(probe.data.status)) break;
3076
+ if (Date.now() + interval > deadline) {
3077
+ throw new ScavioJobTimeoutError(type, id, statusUrl, probe.data.status);
3078
+ }
3079
+ await sleep2(interval);
3080
+ }
3081
+ const results = [];
3082
+ let cursor;
3083
+ let last;
3084
+ do {
3085
+ last = await this.status(type, id, { cursor, limit: COLLECT_PAGE_SIZE });
3086
+ results.push(...last.data.results);
3087
+ cursor = last.data.next_cursor ?? void 0;
3088
+ } while (cursor);
3089
+ return { data: { ...last.data, results, next_cursor: null } };
3090
+ }
3091
+ };
3092
+
2948
3093
  // src/client.ts
2949
3094
  var DEFAULT_MAX_REQUESTS_PER_SECOND = 1;
2950
3095
  var MAX_REQUESTS_PER_SECOND = 50;
2951
- var Scavio = class {
3096
+ var ScavioCore = class {
3097
+ /**
3098
+ * Read ANY web page and get it back as readability Markdown (the default),
3099
+ * plain text, or raw HTML. Returns `{ url, format, mode, content,
3100
+ * content_length }`.
3101
+ *
3102
+ * Extract is a core endpoint, not a platform: `scavio.extract({ url })`
3103
+ * reads one page, and `scavio.extract.map()` / `.crawl()` / `.batch()` work
3104
+ * on whole sites and URL lists.
3105
+ *
3106
+ * Credits are a function of `mode`, not a flat per-call constant:
3107
+ * "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
3108
+ * only on a successful extraction - a dead link, bot wall or timeout costs
3109
+ * nothing.
3110
+ *
3111
+ * Start on "normal". Move to "advanced" when the page builds its content in
3112
+ * the browser, and to "ultra" only when a bot wall blocks the other two.
3113
+ *
3114
+ * @example
3115
+ * const page = await scavio.extract({ url: "https://example.com/pricing" });
3116
+ * console.log(page.content);
3117
+ */
3118
+ async extract(options) {
3119
+ return this._post("/api/v1/extract", options);
3120
+ }
3121
+ };
3122
+ var Scavio = class extends ScavioCore {
2952
3123
  google;
2953
3124
  amazon;
2954
3125
  walmart;
@@ -2991,6 +3162,7 @@ var Scavio = class {
2991
3162
  maxRetries;
2992
3163
  rateLimiter;
2993
3164
  constructor(config) {
3165
+ super();
2994
3166
  this.apiKey = config?.apiKey ?? process.env.SCAVIO_API_KEY ?? "";
2995
3167
  if (!this.apiKey) {
2996
3168
  throw new MissingAPIKeyError();
@@ -3041,9 +3213,19 @@ var Scavio = class {
3041
3213
  this.realtor = new RealtorNamespace(this);
3042
3214
  this.rightmove = new RightmoveNamespace(this);
3043
3215
  this.mercadolibre = new MercadoLibreNamespace(this);
3216
+ const extractNs = new ExtractNamespace(this);
3217
+ this.extract = Object.assign(ScavioCore.prototype.extract.bind(this), {
3218
+ map: extractNs.map.bind(extractNs),
3219
+ crawl: extractNs.crawl.bind(extractNs),
3220
+ crawlStatus: extractNs.crawlStatus.bind(extractNs),
3221
+ cancelCrawl: extractNs.cancelCrawl.bind(extractNs),
3222
+ batch: extractNs.batch.bind(extractNs),
3223
+ batchStatus: extractNs.batchStatus.bind(extractNs),
3224
+ cancelBatch: extractNs.cancelBatch.bind(extractNs)
3225
+ });
3044
3226
  }
3045
3227
  /** @internal */
3046
- async _post(path, body) {
3228
+ async _post(path, body, headers) {
3047
3229
  return request({
3048
3230
  method: "POST",
3049
3231
  path,
@@ -3052,11 +3234,12 @@ var Scavio = class {
3052
3234
  timeout: this.timeout,
3053
3235
  maxRetries: this.maxRetries,
3054
3236
  rateLimiter: this.rateLimiter,
3055
- body
3237
+ body,
3238
+ headers
3056
3239
  });
3057
3240
  }
3058
3241
  /** @internal */
3059
- async _get(path) {
3242
+ async _get(path, query) {
3060
3243
  return request({
3061
3244
  method: "GET",
3062
3245
  path,
@@ -3064,39 +3247,73 @@ var Scavio = class {
3064
3247
  baseUrl: this.baseUrl,
3065
3248
  timeout: this.timeout,
3066
3249
  maxRetries: this.maxRetries,
3250
+ rateLimiter: this.rateLimiter,
3251
+ query
3252
+ });
3253
+ }
3254
+ /** @internal */
3255
+ async _delete(path) {
3256
+ return request({
3257
+ method: "DELETE",
3258
+ path,
3259
+ apiKey: this.apiKey,
3260
+ baseUrl: this.baseUrl,
3261
+ timeout: this.timeout,
3262
+ maxRetries: this.maxRetries,
3067
3263
  rateLimiter: this.rateLimiter
3068
3264
  });
3069
3265
  }
3070
3266
  async search(options) {
3071
3267
  return this.google.search(options);
3072
3268
  }
3073
- /**
3074
- * Read ANY web page and get it back as readability Markdown (the default),
3075
- * plain text, or raw HTML. Returns `{ url, format, mode, content,
3076
- * content_length }`.
3077
- *
3078
- * This is a core endpoint, not a platform, so it lives on the client itself:
3079
- * `scavio.extract({ url })`, never `scavio.extract.extract()`.
3080
- *
3081
- * Credits are a function of `mode`, not a flat per-call constant:
3082
- * "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
3083
- * only on a successful extraction - a dead link, bot wall or timeout costs
3084
- * nothing.
3085
- *
3086
- * Start on "normal". Move to "advanced" when the page builds its content in
3087
- * the browser, and to "ultra" only when a bot wall blocks the other two.
3088
- *
3089
- * @example
3090
- * const page = await scavio.extract({ url: "https://example.com/pricing" });
3091
- * console.log(page.content);
3092
- */
3093
- async extract(options) {
3094
- return this._post("/api/v1/extract", options);
3095
- }
3096
3269
  async getUsage() {
3097
3270
  return this._get("/api/v1/usage");
3098
3271
  }
3099
3272
  };
3273
+
3274
+ // src/webhook.ts
3275
+ async function verifyWebhook(secret, header, body, toleranceSec = 300) {
3276
+ if (!secret || !header) return false;
3277
+ const parts = {};
3278
+ for (const piece of header.split(",")) {
3279
+ const i = piece.indexOf("=");
3280
+ if (i > 0) parts[piece.slice(0, i).trim()] = piece.slice(i + 1).trim();
3281
+ }
3282
+ const t = Number(parts.t);
3283
+ const v1 = parts.v1;
3284
+ if (!parts.t || !Number.isFinite(t) || !v1 || !/^[0-9a-f]+$/i.test(v1)) return false;
3285
+ if (Math.abs(Math.floor(Date.now() / 1e3) - t) > toleranceSec) return false;
3286
+ const subtle = await getSubtle();
3287
+ const enc = new TextEncoder();
3288
+ const key = await subtle.importKey(
3289
+ "raw",
3290
+ enc.encode(secret),
3291
+ { name: "HMAC", hash: "SHA-256" },
3292
+ false,
3293
+ ["sign"]
3294
+ );
3295
+ const bodyBytes = typeof body === "string" ? enc.encode(body) : body;
3296
+ const prefix = enc.encode(`${t}.`);
3297
+ const message = new Uint8Array(prefix.length + bodyBytes.length);
3298
+ message.set(prefix, 0);
3299
+ message.set(bodyBytes, prefix.length);
3300
+ const mac = new Uint8Array(await subtle.sign("HMAC", key, message));
3301
+ let expected = "";
3302
+ for (const b of mac) expected += b.toString(16).padStart(2, "0");
3303
+ return timingSafeEqualHex(expected, v1.toLowerCase());
3304
+ }
3305
+ function timingSafeEqualHex(a, b) {
3306
+ if (a.length !== b.length) return false;
3307
+ let diff = 0;
3308
+ for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
3309
+ return diff === 0;
3310
+ }
3311
+ async function getSubtle() {
3312
+ const g = globalThis;
3313
+ if (g.crypto?.subtle) return g.crypto.subtle;
3314
+ const nodeCrypto = await import("crypto");
3315
+ return nodeCrypto.webcrypto.subtle;
3316
+ }
3100
3317
  // Annotate the CommonJS export names for ESM import in node:
3101
3318
  0 && (module.exports = {
3102
3319
  AirbnbNamespace,
@@ -3109,6 +3326,7 @@ var Scavio = class {
3109
3326
  CostcoNamespace,
3110
3327
  DEFAULT_MAX_REQUESTS_PER_SECOND,
3111
3328
  EbayNamespace,
3329
+ ExtractNamespace,
3112
3330
  G2Namespace,
3113
3331
  GlassdoorNamespace,
3114
3332
  GoogleAdsNamespace,
@@ -3136,6 +3354,7 @@ var Scavio = class {
3136
3354
  ScavioAPIError,
3137
3355
  ScavioConnectionError,
3138
3356
  ScavioError,
3357
+ ScavioJobTimeoutError,
3139
3358
  ScavioTimeoutError,
3140
3359
  TargetNamespace,
3141
3360
  ThreadsNamespace,
@@ -3147,6 +3366,7 @@ var Scavio = class {
3147
3366
  XNamespace,
3148
3367
  YelpNamespace,
3149
3368
  YouTubeNamespace,
3150
- ZillowNamespace
3369
+ ZillowNamespace,
3370
+ verifyWebhook
3151
3371
  });
3152
3372
  //# sourceMappingURL=index.cjs.map