serpkite 0.3.0 → 0.4.0

package/README.md

@@ -94,6 +94,13 @@ vertical-specific extras and `meta` (`request_id`, `credits_used`, `cached`, `la
 | `autocomplete(params)` | `POST /v1/autocomplete` | `SearchParams` | `AutocompleteResponse` (`results[].value`) |
 | `webpage(params)` | `POST /v1/webpage` | `WebpageParams` (`url`, `include_html`) | `WebpageResponse` (`markdown`, `text`, `metadata`) |
 | `rank(params)` | `POST /v1/rank` | `RankParams` (`q`, `domain`, `num`: 10\|20\|30\|50\|100) | `RankResponse` (`position` or `null`, `matches`, `checked`) |
+| `extract(params)` | `POST /v1/extract` | `ExtractParams` (`urls` ≤ 20, `format`, `query`, `highlights`, …) | `ExtractResponse` (`results`, `failed`) |
+| `map(params)` | `POST /v1/map` | `MapParams` (`url`, `search`, `limit`, `sitemap`, path filters) | `MapResponse` (`results[].url`) |
+| `crawl(params)` | `POST /v1/crawl` | `CrawlParams` (`url`, `limit` ≤ 1000, `max_depth` ≤ 10, …) | `TaskCreated` (`202`) |
+| `getCrawl(id)` / `cancelCrawl(id)` | `GET` / `DELETE /v1/crawl/{id}` | task id | `CrawlTask` / `TaskCancelResponse` |
+| `waitForCrawl(idOrTask, options?)` | polls `GET /v1/crawl/{id}` | task id or task | `CrawlTask` (`completed`, `failed` or `canceled`) |
+| `monitors.create/list/get/update/delete/run` | `/v1/monitors…` | `MonitorCreateParams`, `MonitorUpdateParams` | `Monitor`, `MonitorList` |
+| `monitors.runs(id, params?)` / `monitors.iterRuns(id)` | `GET /v1/monitors/{id}/runs` | `limit`, `before` | `MonitorRunList` / `AsyncIterable<MonitorRun>` |
 | `account()` | `GET /v1/account` | none | `Account` (`balance`, `plan`, `rate_limit_rps`, `month`, …) |
 | `batches.create(params)` | `POST /v1/batches` | `BatchCreateParams` | `BatchCreateResponse` |
 | `batches.get(id)` | `GET /v1/batches/{id}` | batch id | `Batch` |
@@ -146,6 +153,24 @@ await sk.search({ q: "espresso", max_age: 3600 });
 "compact"` resolves to a `CompactResponse`. The return type follows the literal you pass. With
 `fields` the response contains only the requested keys, so treat the typed fields as optional.
 
+## Search controls
+
+Domain filters and date ranges work on search, news, images and videos; `boost_domains` on search
+and news; `highlights` on search with `include_content`. None of them costs extra credits.
+
+```ts
+const res = await sk.search({
+  q: "connection pooling",
+  include_domains: ["postgresql.org", "github.com/pgbouncer", ".edu"], // host, path prefix or TLD (≤ 20)
+  exclude_domains: ["pinterest.com"],
+  boost_domains: ["postgresql.org"], // to the top, keeping the rest
+  start_date: "2026-01-01", // YYYY-MM-DD, end_date too
+  include_content: 3,
+  highlights: true, // 3 query-ranked passages per page instead of the whole page
+});
+for (const r of res.results) console.log(r.position, r.published_at, r.highlights?.[0]?.text);
+```
+
 ## Search engines & fallback
 
 By default every request is answered by Google only (`engine: "google"`); SerpKite already
@@ -173,6 +198,104 @@ await sk.news({ q: "espresso", engine: ["google", "brave"] });
   and its `outcome` (absent on cache hits). `request.engine` echoes what you asked for.
 - Credits (`meta.credits_used`, `X-Credits-Used`) follow the answering provider's price.
 
+## Map and extract
+
+```ts
+// The URLs of a site (1 credit): sitemaps + start page, canonicalised and deduplicated
+const site = await sk.map({ url: "https://docs.example.com/", search: "install", include_paths: ["^/guides/"] });
+for (const u of site.results) console.log(u.url);
+
+// Up to 20 URLs (HTML or PDF) as Markdown in one call: 1 credit per URL that came back
+const pages = await sk.extract({ urls: site.results.slice(0, 5).map((u) => u.url), query: "install", highlights: 3 });
+for (const f of pages.failed) console.log("failed", f.url, f.error.code); // not charged
+```
+
+## Crawl
+
+```ts
+const task = await sk.crawl({
+  url: "https://docs.example.com/",
+  limit: 200, // up to 1000 pages; max_depth up to 10
+  include_paths: ["^/guides/"],
+  sitemap: "include", // include (default) | only | skip
+  query: "authentication", // read the most relevant pages first
+});
+const done = await sk.waitForCrawl(task); // polls until completed/failed/canceled
+for (const p of done.result?.pages ?? []) console.log(p.url, p.markdown?.length);
+console.log(done.result?.stats.stopped); // done | limit | time_limit | size_limit | too_many_failures | canceled
+```
+
+- `crawl` reserves `limit` credits and charges 1 per page read (0.5 from cache); the rest is
+  refunded. A crawl that reads nothing fails (`no_pages`) and costs nothing.
+- It returns `202` with a task; `waitForCrawl` takes `{ timeoutMs, pollIntervalMs, maxPollIntervalMs,
+  signal }` like `batches.wait` (default timeout 35 minutes: a crawl runs for up to 30) and
+  returns failed or canceled tasks instead of throwing.
+- `cancelCrawl(id)` refunds a queued crawl; a running one stops at its next checkpoint
+  (`status: "canceling"`) and is charged for the pages read.
+- Robots.txt (per host) and Crawl-delay are honoured; `result.stats` reports `discovered`, `queued`,
+  `duplicates`, `sitemap_urls`, `robots`, `robots_blocked` and why it `stopped`.
+- Pass `webhook_url` to get a signed `crawl.completed` delivery instead of polling. A result over
+  4 MB arrives as `result: null` with `result_omitted: true`; fetch it from `poll_url`.
+
+## Monitors
+
+```ts
+const mon = await sk.monitors.create({
+  q: "ai agents",
+  endpoint: "news",
+  interval: "hourly", // hourly | daily | weekly, or interval_seconds (3600-2592000)
+  webhook_url: "https://example.com/hooks/serpkite",
+});
+await sk.monitors.run(mon.id); // due within ~30 s
+await sk.monitors.update(mon.id, { active: false });
+await sk.monitors.update(mon.id, { q: "ai agents frameworks", num: 20 }); // change the saved search
+const { results } = await sk.monitors.list();
+
+// Run history, newest first (new results kept 24 h): one page, or every run
+const page = await sk.monitors.runs(mon.id, { limit: 20 }); // page.next_before → { before }
+for await (const run of sk.monitors.iterRuns(mon.id)) console.log(run.status, run.new_results);
+await sk.monitors.delete(mon.id);
+```
+
+A `webpage` monitor watches one page instead of a search: it checks `url` each interval (1 credit
+per check) and reports it as a `MonitorPageChange` (`url`, `title`, `change: "new" | "changed"`,
+`content_hash`, `markdown`) when its content changes. `metadata` (your own JSON object, ≤ 2 KB) is
+stored on any monitor and echoed in its `monitor.results` webhooks; on `update`, an object replaces it
+and `null` clears it.
+
+```ts
+const watch = await sk.monitors.create({
+  endpoint: "webpage",
+  url: "https://example.com/pricing",
+  interval: "daily",
+  metadata: { customer: "acme" },
+});
+await sk.monitors.update(watch.id, { metadata: null });
+```
+
+Each search run costs what its search costs (1 credit per 10 results, `num` 100 is 7; empty and failed runs are free) and POSTs only results it
+hasn't seen before (among the top `num`) as a `monitor.results` webhook. `webhook_url` is optional:
+without one, read new results from `monitors.runs`. `active: false` creates a monitor paused; after 10
+failed runs in a row it pauses itself (`last_status: "paused"`) and `update(id, { active: true })`
+resumes it. `webhook_url: ""` removes the webhook; a changed search reports every result as new once.
+
+## Webhooks
+
+Every delivery (`batch.completed`, `crawl.completed`, `monitor.results`) is signed with
+`X-SerpKite-Signature`. `parseWebhook` verifies it and returns the event typed by
+`X-SerpKite-Event`, with `deliveryId` from `X-SerpKite-Delivery` (for `monitor.results`, the
+`run_id`) to deduplicate retries:
+
+```ts
+import { parseWebhook } from "serpkite";
+
+const ev = await parseWebhook(process.env.SERPKITE_WEBHOOK_SECRET!, rawBody, req.headers); // throws invalid_signature
+if (ev.type === "monitor.results") console.log(ev.data.metadata, ev.data.url, ev.data.new_results.length);
+if (ev.type === "crawl.completed") console.log(ev.data.status, ev.data.poll_url);
+```
+
+`verifyWebhook(secret, rawBody, headers)` only checks the signature and returns a boolean.
+
 ## Batches
 
 Batch jobs cost half price. Submit 1-100 requests for one endpoint; each request becomes a job.
@@ -246,8 +369,8 @@ Failed, empty and blocked searches are refunded, so they never cost credits.
 Requests are retried up to `maxRetries` times (default 2) on `429`, `5xx`, network errors and
 timeouts, with exponential backoff (`retryDelayMs · 2^attempt`, capped at 8 s) and jitter. A
 `Retry-After` header (seconds or HTTP date, capped at 60 s) takes precedence. Other `4xx` errors
-are never retried. `batches.create` retries only on `429`, so a lost response can never queue
-(and bill) the same requests twice.
+are never retried. `batches.create`, `crawl`, `monitors.create` and `monitors.run` retry only on
+`429`, so a lost response can never queue (and bill) the same work twice.
 
 ## Types
 
@@ -257,7 +380,9 @@ Types are generated from the OpenAPI contract (`backend/api/serp-api.yaml`) with
 `ReviewsResponse`, `ShoppingResponse`, `ScholarResponse`, `PatentsResponse`,
 `AutocompleteResponse`, `WebpageResponse`, `RankResponse`,
 `Account`, `Batch`, `BatchCreateParams`, `Meta`, `Engine`, `Provider`, `RouteStep`,
-`CompactResponse` and more. The raw
+`CompactResponse`, `MapResponse`, `ExtractResponse`, `CrawlParams`, `CrawlTask`, `CrawlResult`,
+`TaskCreated`, `Monitor`, `MonitorRun`, `MonitorPageChange`, `TaskCompletedEvent`,
+`MonitorResultsEvent`, `WebhookEvent` and more. The raw
 `paths`, `components` and `operations` types are exported as well.
 
 ## Development

package/dist/index.cjs

@@ -50,8 +50,9 @@ function errorFromResponse(status, headers, body) {
 }
 
 // src/client.ts
-var VERSION = "0.3.0";
+var VERSION = "0.4.0";
 var DEFAULT_BASE_URL = "https://api.serpkite.com";
+var TASK_DONE = /* @__PURE__ */ new Set(["completed", "failed", "canceled"]);
 var SerpKite = class {
   baseUrl;
   timeoutMs;
@@ -117,10 +118,150 @@ var SerpKite = class {
   rank(params, options) {
     return this.request({ method: "POST", path: "/v1/rank", body: params, options });
   }
+  /**
+   * Read up to 20 URLs (HTML or PDF) as Markdown, text or HTML in one call, with optional
+   * query-ranked `highlights` (BM25), `include_links` and `include_images`. 1 credit per URL
+   * that comes back (0.5 from cache); URLs that fail are listed in `failed` and cost nothing.
+   */
+  extract(params, options) {
+    const wait = ((params.timeout ?? 50) + 15) * 1e3;
+    return this.request({
+      method: "POST",
+      path: "/v1/extract",
+      body: params,
+      options: {
+        ...options,
+        timeoutMs: options?.timeoutMs ?? (this.timeoutMs > 0 ? Math.max(this.timeoutMs, wait) : 0)
+      },
+      retry: "rate_limit"
+    });
+  }
+  /**
+   * The URLs of a site from its sitemaps (XML, RSS/Atom, gzip, plain text) and start page,
+   * canonicalised and deduplicated, optionally ranked by `search`. `include_paths` /
+   * `exclude_paths` (regular expressions on the URL path) and `ignore_query_parameters`
+   * narrow the list. 1 credit; free when nothing is found.
+   */
+  map(params, options) {
+    return this.request({ method: "POST", path: "/v1/map", body: params, options });
+  }
   /** Balance, limits and this month's usage for the key's account. */
   account(options) {
     return this.request({ method: "GET", path: "/v1/account", options });
   }
+  // ── Async tasks: crawl ────────────────────────────────────────────────
+  /**
+   * Starts an async crawl of one site and returns the queued task (`202`). 1 credit per page
+   * read (0.5 from cache); `limit` credits (up to 1000 pages, `max_depth` up to 10) are
+   * reserved up front and the rest refunded. `sitemap` (`include` default, `only`, `skip`)
+   * seeds the crawl with sitemap URLs; `query` reads the most relevant pages first. robots.txt
+   * and Crawl-delay are honoured; `result.stats.stopped` says why the crawl ended (`done`,
+   * `limit`, `time_limit`, `size_limit`, `too_many_failures`, `canceled`).
+   * Only retried on 429, so a lost response never starts (and reserves) a second crawl.
+   */
+  crawl(params, options) {
+    return this.request({
+      method: "POST",
+      path: "/v1/crawl",
+      body: params,
+      options,
+      retry: "rate_limit"
+    });
+  }
+  /** Polls a crawl (results are kept for 24 hours). */
+  getCrawl(id, options) {
+    return this.request({ method: "GET", path: `/v1/crawl/${encodeURIComponent(id)}`, options });
+  }
+  /** Cancels a crawl: refunded if still queued, else stops at the next checkpoint. */
+  cancelCrawl(id, options) {
+    return this.request({
+      method: "DELETE",
+      path: `/v1/crawl/${encodeURIComponent(id)}`,
+      options
+    });
+  }
+  /**
+   * Polls until the crawl is `completed`, `failed` or `canceled` and returns it (a failed
+   * task is returned, not thrown; check `status` and `error`). Accepts an id or the task
+   * returned by `crawl`.
+   */
+  async waitForCrawl(idOrTask, options = {}) {
+    return this.#waitTask(taskId(idOrTask), (id) => this.getCrawl(id, { signal: options.signal }), {
+      ...options,
+      timeoutMs: options.timeoutMs ?? 21e5
+    });
+  }
+  // ── Monitors ──────────────────────────────────────────────────────────
+  monitors = {
+    /**
+     * Saves a `search` or `news` request (`q`) that runs on a schedule (hourly to monthly) and
+     * POSTs results it hasn't seen before to `webhook_url` (`monitor.results`). With
+     * `endpoint: "webpage"` and a `url` it checks that page each interval (1 credit per check)
+     * and reports it (`MonitorPageChange`) when its content changes. `metadata` (your JSON
+     * object, at most 2 KB) is echoed on the monitor and in its webhooks. Without a
+     * `webhook_url`, read new results from `monitors.runs`. `active: false`
+     * creates it paused. A monitor pauses itself after 10 failed runs in a row
+     * (`last_status: "paused"`); `update(id, { active: true })` resumes it. Each run costs
+     * what its search costs. Only retried on 429.
+     */
+    create: (params, options) => this.request({
+      method: "POST",
+      path: "/v1/monitors",
+      body: params,
+      options,
+      retry: "rate_limit"
+    }),
+    /** The account's monitors. */
+    list: (options) => this.request({ method: "GET", path: "/v1/monitors", options }),
+    get: (id, options) => this.request({ method: "GET", path: `/v1/monitors/${encodeURIComponent(id)}`, options }),
+    /**
+     * Changes `name`, `interval` / `interval_seconds`, `webhook_url` (`""` removes it),
+     * `active` (`true` resumes a paused monitor), `metadata` (an object replaces it, `null`
+     * clears it) or the saved request (`endpoint`, `q`, `url`, `country`, `num`, domains,
+     * `engine`…). Switching `endpoint` between a search and `webpage` drops the other kind's
+     * fields. A changed search reports every result as new on the next run.
+     */
+    update: (id, params, options) => this.request({
+      method: "PATCH",
+      path: `/v1/monitors/${encodeURIComponent(id)}`,
+      body: params,
+      options
+    }),
+    delete: async (id, options) => {
+      await this.request({
+        method: "DELETE",
+        path: `/v1/monitors/${encodeURIComponent(id)}`,
+        options
+      });
+    },
+    /** Runs an active monitor at the next scheduler poll (within about 30 seconds).
+     * Paused monitors return 409; update(id, { active: true }) resumes and schedules them. */
+    run: (id, options) => this.request({
+      method: "POST",
+      path: `/v1/monitors/${encodeURIComponent(id)}/run`,
+      options,
+      retry: "rate_limit"
+    }),
+    /**
+     * One page of the monitor's run history, newest first (`limit` 1-100, default 20). Each
+     * run carries its new results for 24 hours. Pass `next_before` as `before` for the next
+     * page; it is null on the last one.
+     */
+    runs: (id, params = {}, options) => this.request({
+      method: "GET",
+      path: `/v1/monitors/${encodeURIComponent(id)}/runs${query(params)}`,
+      options
+    }),
+    /**
+     * Iterates over every run of the monitor, newest first, fetching pages of `limit` as
+     * needed (starting at `before` when given).
+     *
+     * ```ts
+     * for await (const run of sk.monitors.iterRuns("m1")) console.log(run.new_results);
+     * ```
+     */
+    iterRuns: (id, params = {}, options) => this.#iterRuns(id, params, options)
+  };
   // ── Batches ───────────────────────────────────────────────────────────
   batches = {
     /**
@@ -150,6 +291,35 @@ var SerpKite = class {
     wait: (idOrEntry, options = {}) => this.#wait(idOrEntry, options)
   };
   // ── Plumbing ──────────────────────────────────────────────────────────
+  async *#iterRuns(id, params, options) {
+    let before = params.before;
+    for (; ; ) {
+      const page = await this.monitors.runs(id, { ...params, before }, options);
+      yield* page.results;
+      if (!page.next_before || page.results.length === 0) return;
+      before = page.next_before;
+    }
+  }
+  async #waitTask(id, get, o) {
+    const timeoutMs = o.timeoutMs ?? 6e5;
+    const maxInterval = o.maxPollIntervalMs ?? 1e4;
+    let interval = o.pollIntervalMs ?? 1e3;
+    const deadline = Date.now() + timeoutMs;
+    for (; ; ) {
+      const task = await get(id);
+      if (TASK_DONE.has(task.status)) return task;
+      const left = deadline - Date.now();
+      if (left <= 0) {
+        throw new SerpKiteError(
+          0,
+          "timeout",
+          `task ${id} still ${task.status} after ${timeoutMs} ms`
+        );
+      }
+      await sleep(Math.min(interval, left), o.signal);
+      interval = Math.min(interval * 1.5, maxInterval);
+    }
+  }
   #vertical(path) {
     return ((params, options) => this.request({ method: "POST", path, body: params, options }));
   }
@@ -284,6 +454,19 @@ var SerpKite = class {
     return after === void 0 ? this.#backoff(attempt) : Math.min(after, 6e4);
   }
 };
+function query(params) {
+  const q = new URLSearchParams();
+  for (const [k, v] of Object.entries(params)) {
+    if (v !== void 0 && v !== null) q.set(k, String(v));
+  }
+  const s = q.toString();
+  return s ? `?${s}` : "";
+}
+function taskId(idOrTask) {
+  const id = typeof idOrTask === "string" ? idOrTask : idOrTask?.id;
+  if (!id) throw new SerpKiteError(0, "invalid_request", "a task id is required");
+  return id;
+}
 function parseRetryAfter(v) {
   if (!v) return void 0;
   const secs = Number(v);
@@ -404,6 +587,29 @@ async function verifyWebhook(secret, payload, headers, options = {}) {
   for (let i = 0; i < want.length; i++) diff |= want.charCodeAt(i) ^ got.charCodeAt(i);
   return diff === 0;
 }
+async function parseWebhook(secret, payload, headers, options = {}) {
+  if (!await verifyWebhook(secret, payload, headers, options)) {
+    throw new SerpKiteError(400, "invalid_signature", "webhook signature or timestamp is invalid");
+  }
+  const text = typeof payload === "string" ? payload : new TextDecoder().decode(payload);
+  let data;
+  try {
+    data = JSON.parse(text);
+  } catch (err) {
+    throw new SerpKiteError(
+      400,
+      "invalid_payload",
+      "webhook body is not JSON",
+      void 0,
+      void 0,
+      {
+        cause: err
+      }
+    );
+  }
+  const type = header(headers, "x-serpkite-event") ?? (typeof data?.event === "string" ? data.event : "batch.completed");
+  return { type, deliveryId: header(headers, "x-serpkite-delivery"), data };
+}
 
 exports.DEFAULT_BASE_URL = DEFAULT_BASE_URL;
 exports.SerpKite = SerpKite;
@@ -411,6 +617,7 @@ exports.SerpKiteError = SerpKiteError;
 exports.VERSION = VERSION;
 exports.isBatchError = isBatchError;
 exports.parseRetryAfter = parseRetryAfter;
+exports.parseWebhook = parseWebhook;
 exports.verifyWebhook = verifyWebhook;
 //# sourceMappingURL=index.cjs.map
 //# sourceMappingURL=index.cjs.map