scavio 0.22.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -2,7 +2,8 @@
2
2
 
3
3
  TypeScript SDK for the [Scavio API](https://scavio.dev?utm_source=npm-scavio-js&utm_medium=package&utm_campaign=readme) — real-time structured web
4
4
  data across 36 platforms on one API key, plus `extract()` to read
5
- any URL as clean Markdown.
5
+ any URL as clean Markdown, and `extract.map()` / `extract.crawl()` /
6
+ `extract.batch()` to map, crawl and batch-extract whole websites.
6
7
 
7
8
  Structured JSON in, structured JSON out. No per-site parsers to maintain.
8
9
 
@@ -14,6 +15,7 @@ Structured JSON in, structured JSON out. No per-site parsers to maintain.
14
15
  - **Ads transparency** — Google Ads Transparency Center, Meta Ad Library
15
16
  - **Social and video** — YouTube, TikTok, Instagram, Threads, X, LinkedIn, Reddit, Kuaishou
16
17
  - **Any other page** — `extract()` turns a URL into Markdown, plain text or raw HTML
18
+ - **Whole websites** — `extract.map()` lists a site's URLs, `extract.crawl()` crawls a site, `extract.batch()` extracts a list of URLs in one job
17
19
 
18
20
  ## Install
19
21
 
@@ -103,6 +105,129 @@ boilerplate stripped. `text` is that markdown flattened to plain text. URLs are
103
105
  http(s) only, a bare host is upgraded to https, and loopback, private,
104
106
  link-local and cloud-metadata hosts are rejected with a 400.
105
107
 
108
+ ### Website map, crawl and batch extract (new in 0.23.0)
109
+
110
+ `extract` is also a namespace: calling it reads one page, exactly as before
111
+ (`client.extract({ url })` sends the same request it did in 0.22), and its
112
+ methods work on whole sites and URL lists, with each page returned exactly
113
+ like [Extract](https://scavio.dev/docs/extract?utm_source=npm-scavio-js&utm_medium=package&utm_campaign=readme).
114
+
115
+ ```typescript
116
+ // Map: every URL on a site, from its sitemaps and start page. 1 credit.
117
+ const map = await client.extract.map({
118
+ url: "https://example.com",
119
+ limit: 500, // 1-5,000, default 1,000 (plan caps apply)
120
+ sitemap: "include", // "include" | "only" | "skip"
121
+ include_paths: ["/blog/**"], // optional path globs
122
+ });
123
+ map.data.links; // [{ url, source: "sitemap" | "page", lastmod? }]
124
+ map.data.truncated; // true when more URLs exist than `limit`
125
+
126
+ // Crawl: start a job, then poll it (polling and cancelling are free).
127
+ const job = await client.extract.crawl(
128
+ { url: "https://example.com/docs", max_pages: 50, max_depth: 3, format: "markdown" },
129
+ { idempotencyKey: "docs-crawl-2026-10-12" }, // optional, sent as Idempotency-Key
130
+ );
131
+ job.data.id; // "crawl_..."
132
+ job.data.status_url;
133
+
134
+ const page = await client.extract.crawlStatus(job.data.id, { limit: 25 });
135
+ page.data.status; // "queued" | "running" | "completed" | "failed" | "cancelled"
136
+ page.data.pages; // { discovered, queued, succeeded, failed, skipped }
137
+ page.data.results; // [{ url, depth, status, status_code, content, ... }]
138
+ page.data.next_cursor; // pass as { cursor } for the next page, null at the end
139
+
140
+ await client.extract.cancelCrawl(job.data.id); // 409 once the crawl has finished
141
+
142
+ // Or wait for it: polls until done and returns every result in one response.
143
+ const done = await client.extract.crawl(
144
+ { url: "https://example.com/docs", max_pages: 20 },
145
+ { wait: true, pollInterval: 2000, waitTimeout: 600_000 },
146
+ );
147
+ done.data.results.length;
148
+
149
+ // Batch: extract a list of URLs in one job (results in input order).
150
+ const batch = await client.extract.batch({
151
+ urls: ["https://example.com/a", "https://example.com/b"],
152
+ mode: "normal",
153
+ });
154
+ batch.data.invalid_urls; // rejected up front, never billed
155
+ await client.extract.batchStatus(batch.data.id);
156
+ await client.extract.cancelBatch(batch.data.id);
157
+ ```
158
+
159
+ **Pricing.** Map costs 1 credit. Crawl and batch bill per **successful** page
160
+ at Extract's price for the tier: `normal` 1, `advanced` 1, `ultra` 2. Failed
161
+ and skipped pages are free, starting, polling and cancelling a job are free,
162
+ and credits are taken page by page: if your balance runs out the job stops
163
+ cleanly with `stopped_reason: "insufficient_credits"`.
164
+
165
+ **Plan limits.**
166
+
167
+ | Plan | Max pages per crawl | Active jobs | URLs per batch |
168
+ |------|--------------------:|------------:|---------------:|
169
+ | Free | 10 | 1 | 10 |
170
+ | PAYG | 100 | 1 | 50 |
171
+ | Project | 500 | 2 | 100 |
172
+ | Bootstrap | 2,000 | 3 | 500 |
173
+ | Startup | 5,000 | 5 | 1,000 |
174
+ | Growth | 10,000 | 10 | 1,000 |
175
+
176
+ Starting a job above your plan's active-job limit is a 429 (`RateLimitError`)
177
+ with `Retry-After`.
178
+
179
+ **Behavior.** Crawl and map always obey the site's robots.txt. A crawl stays on
180
+ the start URL's site (`allow_subdomains` opts into subdomains) and visits URLs
181
+ under the start path first. Batch treats each URL like a single `extract()`
182
+ call. Results are kept for 3 days after the job finishes (`expires_at`), up to
183
+ 2 MB per page and 250 MB per job, and a job runs for at most 6 hours.
184
+
185
+ **`wait: true`.** `crawl()` and `batch()` poll the status endpoint every
186
+ `pollInterval` ms (default 2,000) until the job is completed, failed or
187
+ cancelled, then page through every cursor and return
188
+ `{ data: { ...summary, results: [all], next_cursor: null } }`. If the job is
189
+ still running after `waitTimeout` ms (default 10 minutes) they throw
190
+ `ScavioJobTimeoutError`; the job keeps running and the error carries
191
+ `jobId`, `jobType` and `statusUrl` so you can keep polling or cancel it. For
192
+ big crawls, prefer a webhook or your own polling loop.
193
+
194
+ **Retries and idempotency.** The SDK retries 429/5xx and network errors. Pass
195
+ `idempotencyKey` when starting a job: repeating a POST with the same key within
196
+ 24 hours returns the original job, so a retried request never starts a second
197
+ one.
198
+
199
+ #### Completion webhooks
200
+
201
+ Pass `webhook_url` (https) to `crawl()` or `batch()` and Scavio POSTs the job
202
+ summary there when it completes, fails or is cancelled (retried at 0 s, 30 s
203
+ and 5 min). The signing secret comes back once, as `job.data.webhook_secret`.
204
+ Each delivery is signed with
205
+ `Scavio-Signature: t=<unix>,v1=<hex HMAC-SHA256 of "<t>.<raw body>">`; verify
206
+ it against the **raw** body with `verifyWebhook` (async, Web Crypto, so it runs
207
+ on Node 18+, Deno, Bun and edge runtimes):
208
+
209
+ ```typescript
210
+ import { verifyWebhook } from "scavio";
211
+
212
+ // e.g. a Next.js / Hono / Workers route handler
213
+ export async function POST(req: Request) {
214
+ const raw = await req.text();
215
+ const ok = await verifyWebhook(
216
+ process.env.SCAVIO_WEBHOOK_SECRET!, // the job's webhook_secret
217
+ req.headers.get("Scavio-Signature"),
218
+ raw,
219
+ 300, // max signature age in seconds (default)
220
+ );
221
+ if (!ok) return new Response("bad signature", { status: 401 });
222
+ const event = JSON.parse(raw);
223
+ // event.id, event.status, ...
224
+ return new Response("ok");
225
+ }
226
+ ```
227
+
228
+ `verifyWebhook` returns `false` (never throws) for a missing, malformed,
229
+ tampered or stale signature.
230
+
106
231
  ### Google
107
232
 
108
233
  Each method hits its own `/api/v2/google*` endpoint and returns Google's full
@@ -534,6 +659,10 @@ await client.tiktok.userPosts({ sec_user_id: "abc123", count: 30 });
534
659
  // Video details
535
660
  await client.tiktok.video({ video_id: "vid123" });
536
661
 
662
+ // Video transcript: timestamped segments and full text (video_id or url;
663
+ // optional 2-letter language). Caption-less videos return available: false.
664
+ await client.tiktok.videoTranscript({ video_id: "7609177768793787679" });
665
+
537
666
  // Video comments
538
667
  await client.tiktok.videoComments({ video_id: "vid123", count: 20 });
539
668
 
@@ -1225,7 +1354,8 @@ All error classes:
1225
1354
  | `InsufficientCreditsError` | 402 | No credits remaining |
1226
1355
  | `NotFoundError` | 404 | No data for that id (see TikTok Shop above) |
1227
1356
  | `RateLimitError` | 429 | Rate limit exceeded |
1228
- | `ScavioAPIError` | other | Catch-all (has `.statusCode`) |
1357
+ | `ScavioAPIError` | other | Catch-all (has `.statusCode`; a 409 from cancelling a finished job lands here) |
1358
+ | `ScavioJobTimeoutError` | — | `crawl()`/`batch()` with `wait: true` outlived `waitTimeout`; carries `jobId` |
1229
1359
 
1230
1360
  Threads and Kuaishou answer a missing or conflicting identifier with **422**, not
1231
1361
  400 — those routes have no 400 at all. Both map to `BadRequestError`, so one
package/dist/index.cjs CHANGED
@@ -1,7 +1,9 @@
1
1
  "use strict";
2
+ var __create = Object.create;
2
3
  var __defProp = Object.defineProperty;
3
4
  var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
5
  var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
5
7
  var __hasOwnProp = Object.prototype.hasOwnProperty;
6
8
  var __export = (target, all) => {
7
9
  for (var name in all)
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
15
17
  }
16
18
  return to;
17
19
  };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
18
28
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
29
 
20
30
  // src/index.ts
@@ -30,6 +40,7 @@ __export(index_exports, {
30
40
  CostcoNamespace: () => CostcoNamespace,
31
41
  DEFAULT_MAX_REQUESTS_PER_SECOND: () => DEFAULT_MAX_REQUESTS_PER_SECOND,
32
42
  EbayNamespace: () => EbayNamespace,
43
+ ExtractNamespace: () => ExtractNamespace,
33
44
  G2Namespace: () => G2Namespace,
34
45
  GlassdoorNamespace: () => GlassdoorNamespace,
35
46
  GoogleAdsNamespace: () => GoogleAdsNamespace,
@@ -57,6 +68,7 @@ __export(index_exports, {
57
68
  ScavioAPIError: () => ScavioAPIError,
58
69
  ScavioConnectionError: () => ScavioConnectionError,
59
70
  ScavioError: () => ScavioError,
71
+ ScavioJobTimeoutError: () => ScavioJobTimeoutError,
60
72
  ScavioTimeoutError: () => ScavioTimeoutError,
61
73
  TargetNamespace: () => TargetNamespace,
62
74
  ThreadsNamespace: () => ThreadsNamespace,
@@ -68,7 +80,8 @@ __export(index_exports, {
68
80
  XNamespace: () => XNamespace,
69
81
  YelpNamespace: () => YelpNamespace,
70
82
  YouTubeNamespace: () => YouTubeNamespace,
71
- ZillowNamespace: () => ZillowNamespace
83
+ ZillowNamespace: () => ZillowNamespace,
84
+ verifyWebhook: () => verifyWebhook
72
85
  });
73
86
  module.exports = __toCommonJS(index_exports);
74
87
 
@@ -155,6 +168,22 @@ var ScavioAPIError = class extends ScavioError {
155
168
  this.responseBody = responseBody;
156
169
  }
157
170
  };
171
+ var ScavioJobTimeoutError = class extends ScavioError {
172
+ constructor(jobType, jobId, statusUrl, lastStatus) {
173
+ super(
174
+ `${jobType} ${jobId} was still ${lastStatus} when the wait timed out. It keeps running: poll extract.${jobType}Status("${jobId}") or cancel it.`
175
+ );
176
+ this.jobType = jobType;
177
+ this.jobId = jobId;
178
+ this.statusUrl = statusUrl;
179
+ this.lastStatus = lastStatus;
180
+ this.name = "ScavioJobTimeoutError";
181
+ }
182
+ jobType;
183
+ jobId;
184
+ statusUrl;
185
+ lastStatus;
186
+ };
158
187
 
159
188
  // src/retry.ts
160
189
  var DEFAULT_RETRY_STATUSES = /* @__PURE__ */ new Set([
@@ -203,7 +232,7 @@ function parseRetryAfter(header) {
203
232
  var BASE_URL = "https://api.scavio.dev";
204
233
  var DEFAULT_TIMEOUT = 3e4;
205
234
  var DEFAULT_MAX_RETRIES = 2;
206
- var SDK_VERSION = true ? "0.22.0" : "0.0.0";
235
+ var SDK_VERSION = true ? "0.23.0" : "0.0.0";
207
236
  var USER_AGENT = `scavio-js/${SDK_VERSION}`;
208
237
  function isServerRuntime() {
209
238
  const g = globalThis;
@@ -261,9 +290,18 @@ function getHeader(response, name) {
261
290
  }
262
291
  return null;
263
292
  }
293
+ function buildQuery(query) {
294
+ if (!query) return "";
295
+ const params = new URLSearchParams();
296
+ for (const [key, value] of Object.entries(query)) {
297
+ if (value !== void 0) params.append(key, String(value));
298
+ }
299
+ const qs = params.toString();
300
+ return qs ? `?${qs}` : "";
301
+ }
264
302
  async function request(options) {
265
- const url = `${options.baseUrl}${options.path}`;
266
- const headers = buildHeaders(options.apiKey);
303
+ const url = `${options.baseUrl}${options.path}${buildQuery(options.query)}`;
304
+ const headers = { ...buildHeaders(options.apiKey), ...options.headers };
267
305
  const retry = makeRetryConfig(
268
306
  options.maxRetries ?? DEFAULT_MAX_RETRIES
269
307
  );
@@ -525,6 +563,14 @@ var TikTokNamespace = class {
525
563
  async video(options) {
526
564
  return this.client._post("/api/v1/tiktok/video", options);
527
565
  }
566
+ /**
567
+ * Spoken-word transcript of a TikTok video, as timestamped segments and
568
+ * full text. Pass `video_id` or `url`. Videos with no spoken captions
569
+ * (music, dance, muted clips) return `available: false`. 1 credit.
570
+ */
571
+ async videoTranscript(options) {
572
+ return this.client._post("/api/v1/tiktok/video/transcript", options);
573
+ }
528
574
  async videoComments(options) {
529
575
  return this.client._post("/api/v1/tiktok/video/comments", options);
530
576
  }
@@ -2945,10 +2991,128 @@ var MercadoLibreNamespace = class {
2945
2991
  }
2946
2992
  };
2947
2993
 
2994
+ // src/namespaces/extract.ts
2995
+ var TERMINAL = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
2996
+ var DEFAULT_POLL_INTERVAL = 2e3;
2997
+ var DEFAULT_WAIT_TIMEOUT = 6e5;
2998
+ var COLLECT_PAGE_SIZE = 100;
2999
+ function sleep2(ms) {
3000
+ return new Promise((resolve) => setTimeout(resolve, ms));
3001
+ }
3002
+ var ExtractNamespace = class {
3003
+ constructor(client) {
3004
+ this.client = client;
3005
+ }
3006
+ client;
3007
+ /**
3008
+ * List a site's page URLs, gathered from its sitemaps and its start page,
3009
+ * deduplicated, filtered by path and limited. Pages disallowed by the
3010
+ * site's robots.txt are left out. 1 credit on the normal tier.
3011
+ */
3012
+ async map(options) {
3013
+ return await this.client._post("/api/v1/extract/map", options);
3014
+ }
3015
+ async crawl(options, request2 = {}) {
3016
+ return this.start("crawl", options, request2);
3017
+ }
3018
+ /** A crawl's progress and its page results, paged by cursor. Free. */
3019
+ async crawlStatus(id, query) {
3020
+ return this.status("crawl", id, query);
3021
+ }
3022
+ /**
3023
+ * Stop a queued or running crawl. No further pages are fetched or billed;
3024
+ * results so far stay available. A finished crawl answers 409. Free.
3025
+ */
3026
+ async cancelCrawl(id) {
3027
+ return this.cancel("crawl", id);
3028
+ }
3029
+ async batch(options, request2 = {}) {
3030
+ return this.start("batch", options, request2);
3031
+ }
3032
+ /** A batch's progress and its page results in input order, paged by cursor. Free. */
3033
+ async batchStatus(id, query) {
3034
+ return this.status("batch", id, query);
3035
+ }
3036
+ /** Stop a queued or running batch. A finished batch answers 409. Free. */
3037
+ async cancelBatch(id) {
3038
+ return this.cancel("batch", id);
3039
+ }
3040
+ async start(type, body, request2) {
3041
+ const headers = request2.idempotencyKey ? { "Idempotency-Key": request2.idempotencyKey } : void 0;
3042
+ const created = await this.client._post(
3043
+ `/api/v1/extract/${type}`,
3044
+ body,
3045
+ headers
3046
+ );
3047
+ if (!request2.wait) return created;
3048
+ return this.waitFor(type, created.data.id, created.data.status_url, request2);
3049
+ }
3050
+ async status(type, id, query = {}) {
3051
+ return await this.client._get(`/api/v1/extract/${type}/${encodeURIComponent(id)}`, {
3052
+ cursor: query.cursor,
3053
+ limit: query.limit,
3054
+ status: query.status,
3055
+ include_content: query.include_content
3056
+ });
3057
+ }
3058
+ async cancel(type, id) {
3059
+ return await this.client._delete(
3060
+ `/api/v1/extract/${type}/${encodeURIComponent(id)}`
3061
+ );
3062
+ }
3063
+ async waitFor(type, id, statusUrl, request2) {
3064
+ const interval = request2.pollInterval ?? DEFAULT_POLL_INTERVAL;
3065
+ const deadline = Date.now() + (request2.waitTimeout ?? DEFAULT_WAIT_TIMEOUT);
3066
+ for (; ; ) {
3067
+ const probe = await this.status(type, id, { limit: 1, include_content: false });
3068
+ if (TERMINAL.has(probe.data.status)) break;
3069
+ if (Date.now() + interval > deadline) {
3070
+ throw new ScavioJobTimeoutError(type, id, statusUrl, probe.data.status);
3071
+ }
3072
+ await sleep2(interval);
3073
+ }
3074
+ const results = [];
3075
+ let cursor;
3076
+ let last;
3077
+ do {
3078
+ last = await this.status(type, id, { cursor, limit: COLLECT_PAGE_SIZE });
3079
+ results.push(...last.data.results);
3080
+ cursor = last.data.next_cursor ?? void 0;
3081
+ } while (cursor);
3082
+ return { data: { ...last.data, results, next_cursor: null } };
3083
+ }
3084
+ };
3085
+
2948
3086
  // src/client.ts
2949
3087
  var DEFAULT_MAX_REQUESTS_PER_SECOND = 1;
2950
3088
  var MAX_REQUESTS_PER_SECOND = 50;
2951
- var Scavio = class {
3089
+ var ScavioCore = class {
3090
+ /**
3091
+ * Read ANY web page and get it back as readability Markdown (the default),
3092
+ * plain text, or raw HTML. Returns `{ url, format, mode, content,
3093
+ * content_length }`.
3094
+ *
3095
+ * Extract is a core endpoint, not a platform: `scavio.extract({ url })`
3096
+ * reads one page, and `scavio.extract.map()` / `.crawl()` / `.batch()` work
3097
+ * on whole sites and URL lists.
3098
+ *
3099
+ * Credits are a function of `mode`, not a flat per-call constant:
3100
+ * "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
3101
+ * only on a successful extraction - a dead link, bot wall or timeout costs
3102
+ * nothing.
3103
+ *
3104
+ * Start on "normal". Move to "advanced" when the page builds its content in
3105
+ * the browser, and to "ultra" only when a bot wall blocks the other two.
3106
+ *
3107
+ * @example
3108
+ * const page = await scavio.extract({ url: "https://example.com/pricing" });
3109
+ * console.log(page.content);
3110
+ */
3111
+ async extract(options) {
3112
+ return this._post("/api/v1/extract", options);
3113
+ }
3114
+ };
3115
+ var Scavio = class extends ScavioCore {
2952
3116
  google;
2953
3117
  amazon;
2954
3118
  walmart;
@@ -2991,6 +3155,7 @@ var Scavio = class {
2991
3155
  maxRetries;
2992
3156
  rateLimiter;
2993
3157
  constructor(config) {
3158
+ super();
2994
3159
  this.apiKey = config?.apiKey ?? process.env.SCAVIO_API_KEY ?? "";
2995
3160
  if (!this.apiKey) {
2996
3161
  throw new MissingAPIKeyError();
@@ -3041,9 +3206,19 @@ var Scavio = class {
3041
3206
  this.realtor = new RealtorNamespace(this);
3042
3207
  this.rightmove = new RightmoveNamespace(this);
3043
3208
  this.mercadolibre = new MercadoLibreNamespace(this);
3209
+ const extractNs = new ExtractNamespace(this);
3210
+ this.extract = Object.assign(ScavioCore.prototype.extract.bind(this), {
3211
+ map: extractNs.map.bind(extractNs),
3212
+ crawl: extractNs.crawl.bind(extractNs),
3213
+ crawlStatus: extractNs.crawlStatus.bind(extractNs),
3214
+ cancelCrawl: extractNs.cancelCrawl.bind(extractNs),
3215
+ batch: extractNs.batch.bind(extractNs),
3216
+ batchStatus: extractNs.batchStatus.bind(extractNs),
3217
+ cancelBatch: extractNs.cancelBatch.bind(extractNs)
3218
+ });
3044
3219
  }
3045
3220
  /** @internal */
3046
- async _post(path, body) {
3221
+ async _post(path, body, headers) {
3047
3222
  return request({
3048
3223
  method: "POST",
3049
3224
  path,
@@ -3052,11 +3227,12 @@ var Scavio = class {
3052
3227
  timeout: this.timeout,
3053
3228
  maxRetries: this.maxRetries,
3054
3229
  rateLimiter: this.rateLimiter,
3055
- body
3230
+ body,
3231
+ headers
3056
3232
  });
3057
3233
  }
3058
3234
  /** @internal */
3059
- async _get(path) {
3235
+ async _get(path, query) {
3060
3236
  return request({
3061
3237
  method: "GET",
3062
3238
  path,
@@ -3064,39 +3240,73 @@ var Scavio = class {
3064
3240
  baseUrl: this.baseUrl,
3065
3241
  timeout: this.timeout,
3066
3242
  maxRetries: this.maxRetries,
3243
+ rateLimiter: this.rateLimiter,
3244
+ query
3245
+ });
3246
+ }
3247
+ /** @internal */
3248
+ async _delete(path) {
3249
+ return request({
3250
+ method: "DELETE",
3251
+ path,
3252
+ apiKey: this.apiKey,
3253
+ baseUrl: this.baseUrl,
3254
+ timeout: this.timeout,
3255
+ maxRetries: this.maxRetries,
3067
3256
  rateLimiter: this.rateLimiter
3068
3257
  });
3069
3258
  }
3070
3259
  async search(options) {
3071
3260
  return this.google.search(options);
3072
3261
  }
3073
- /**
3074
- * Read ANY web page and get it back as readability Markdown (the default),
3075
- * plain text, or raw HTML. Returns `{ url, format, mode, content,
3076
- * content_length }`.
3077
- *
3078
- * This is a core endpoint, not a platform, so it lives on the client itself:
3079
- * `scavio.extract({ url })`, never `scavio.extract.extract()`.
3080
- *
3081
- * Credits are a function of `mode`, not a flat per-call constant:
3082
- * "normal" costs 1, "advanced" costs 1, "ultra" costs 2. Billing happens
3083
- * only on a successful extraction - a dead link, bot wall or timeout costs
3084
- * nothing.
3085
- *
3086
- * Start on "normal". Move to "advanced" when the page builds its content in
3087
- * the browser, and to "ultra" only when a bot wall blocks the other two.
3088
- *
3089
- * @example
3090
- * const page = await scavio.extract({ url: "https://example.com/pricing" });
3091
- * console.log(page.content);
3092
- */
3093
- async extract(options) {
3094
- return this._post("/api/v1/extract", options);
3095
- }
3096
3262
  async getUsage() {
3097
3263
  return this._get("/api/v1/usage");
3098
3264
  }
3099
3265
  };
3266
+
3267
+ // src/webhook.ts
3268
+ async function verifyWebhook(secret, header, body, toleranceSec = 300) {
3269
+ if (!secret || !header) return false;
3270
+ const parts = {};
3271
+ for (const piece of header.split(",")) {
3272
+ const i = piece.indexOf("=");
3273
+ if (i > 0) parts[piece.slice(0, i).trim()] = piece.slice(i + 1).trim();
3274
+ }
3275
+ const t = Number(parts.t);
3276
+ const v1 = parts.v1;
3277
+ if (!parts.t || !Number.isFinite(t) || !v1 || !/^[0-9a-f]+$/i.test(v1)) return false;
3278
+ if (Math.abs(Math.floor(Date.now() / 1e3) - t) > toleranceSec) return false;
3279
+ const subtle = await getSubtle();
3280
+ const enc = new TextEncoder();
3281
+ const key = await subtle.importKey(
3282
+ "raw",
3283
+ enc.encode(secret),
3284
+ { name: "HMAC", hash: "SHA-256" },
3285
+ false,
3286
+ ["sign"]
3287
+ );
3288
+ const bodyBytes = typeof body === "string" ? enc.encode(body) : body;
3289
+ const prefix = enc.encode(`${t}.`);
3290
+ const message = new Uint8Array(prefix.length + bodyBytes.length);
3291
+ message.set(prefix, 0);
3292
+ message.set(bodyBytes, prefix.length);
3293
+ const mac = new Uint8Array(await subtle.sign("HMAC", key, message));
3294
+ let expected = "";
3295
+ for (const b of mac) expected += b.toString(16).padStart(2, "0");
3296
+ return timingSafeEqualHex(expected, v1.toLowerCase());
3297
+ }
3298
+ function timingSafeEqualHex(a, b) {
3299
+ if (a.length !== b.length) return false;
3300
+ let diff = 0;
3301
+ for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
3302
+ return diff === 0;
3303
+ }
3304
+ async function getSubtle() {
3305
+ const g = globalThis;
3306
+ if (g.crypto?.subtle) return g.crypto.subtle;
3307
+ const nodeCrypto = await import("crypto");
3308
+ return nodeCrypto.webcrypto.subtle;
3309
+ }
3100
3310
  // Annotate the CommonJS export names for ESM import in node:
3101
3311
  0 && (module.exports = {
3102
3312
  AirbnbNamespace,
@@ -3109,6 +3319,7 @@ var Scavio = class {
3109
3319
  CostcoNamespace,
3110
3320
  DEFAULT_MAX_REQUESTS_PER_SECOND,
3111
3321
  EbayNamespace,
3322
+ ExtractNamespace,
3112
3323
  G2Namespace,
3113
3324
  GlassdoorNamespace,
3114
3325
  GoogleAdsNamespace,
@@ -3136,6 +3347,7 @@ var Scavio = class {
3136
3347
  ScavioAPIError,
3137
3348
  ScavioConnectionError,
3138
3349
  ScavioError,
3350
+ ScavioJobTimeoutError,
3139
3351
  ScavioTimeoutError,
3140
3352
  TargetNamespace,
3141
3353
  ThreadsNamespace,
@@ -3147,6 +3359,7 @@ var Scavio = class {
3147
3359
  XNamespace,
3148
3360
  YelpNamespace,
3149
3361
  YouTubeNamespace,
3150
- ZillowNamespace
3362
+ ZillowNamespace,
3363
+ verifyWebhook
3151
3364
  });
3152
3365
  //# sourceMappingURL=index.cjs.map