webforai 3.0.0 → 4.0.0

package/dist/types/platform/error.d.ts

@@ -3,9 +3,11 @@
  * `{ error: { code, message } }` envelope.
  *
  * `code` values worth branching on: `invalid_request` (400), `invalid_api_key` (401),
- * `payment_required` (402), `job_not_found` (404), `rate_limited` (429, `retryAfter` set on
- * the demo endpoint), `engine_unavailable` (503). The client never retries on its own; a
- * caller that wants backoff should honour `retryAfter` when present.
+ * `payment_required` / `spend_cap_reached` (402), `robots_disallowed` (403), `job_not_found`
+ * (404), `rate_limited` / `too_many_jobs` (429, `retryAfter` set when the server sends it),
+ * `engine_unavailable` / `result_unavailable` / `scheduling_unknown` (503). 401 and 402
+ * messages end with a hint naming the deployment's dashboard. The client never retries on its
+ * own; a caller that wants backoff should honour `retryAfter` when present.
  *
  * Network-level failures (DNS, refused connections) are NOT wrapped — they propagate as the
  * runtime's own `fetch` errors. `code` is `invalid_response` when the server answered with

package/dist/types/platform/error.js.map

@@ -1 +1 @@
-{"version":3,"file":"error.js","sourceRoot":"","sources":["../../../src/platform/error.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAM1C,YAAY,IAAY,EAAE,OAAe,EAAE,MAAc,EAAE,UAAmB;QAC7E,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC9B,CAAC;CACD"}
+{"version":3,"file":"error.js","sourceRoot":"","sources":["../../../src/platform/error.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAM1C,YAAY,IAAY,EAAE,OAAe,EAAE,MAAc,EAAE,UAAmB;QAC7E,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC9B,CAAC;CACD"}

package/dist/types/platform/fetch.d.ts

@@ -14,6 +14,8 @@ export interface FetchRequestInit {
     method: string;
     headers?: Record<string, string>;
     body?: string;
+    /** Cancels an in-flight status request when polling stops or times out. */
+    signal?: AbortSignal;
 }
 /** The subset of a Response the client reads. */
 export interface FetchResponseLike {

package/dist/types/platform/fetch.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../src/platform/fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAmBH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAkB,EAAa,EAAE;IAC7D,IAAI,MAAM,EAAE,CAAC;QACZ,OAAO,MAAM,CAAC;IACf,CAAC;IACD,MAAM,WAAW,GAAI,UAAoC,CAAC,KAAK,CAAC;IAChE,IAAI,OAAO,WAAW,KAAK,UAAU,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,kGAAkG,CAAC,CAAC;IACrH,CAAC;IACD,OAAO,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AACrC,CAAC,CAAC"}
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../src/platform/fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAqBH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAkB,EAAa,EAAE;IAC7D,IAAI,MAAM,EAAE,CAAC;QACZ,OAAO,MAAM,CAAC;IACf,CAAC;IACD,MAAM,WAAW,GAAI,UAAoC,CAAC,KAAK,CAAC;IAChE,IAAI,OAAO,WAAW,KAAK,UAAU,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,kGAAkG,CAAC,CAAC;IACrH,CAAC;IACD,OAAO,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AACrC,CAAC,CAAC"}

package/dist/types/platform/index.d.ts

@@ -1,4 +1,4 @@
 export { DEFAULT_BASE_URL, createPlatformClient, type PlatformClient, type PlatformClientOptions, type WaitForJobOptions, } from "./client";
 export { PlatformApiError } from "./error";
 export type { FetchLike, FetchRequestInit, FetchResponseLike } from "./fetch";
-export { ENGINES, REGIONS, isStoredPageStub, type BatchOptions, type ConvertOptions, type CrawlOptions, type DemoResult, type DemoScrapeOptions, type Engine, type ExtractorPreset, type JobRef, type JobResultItem, type JobResultsPage, type JobState, type JobStatus, type JobType, type PageError, type PageFailure, type PageResult, type PageSuccess, type Region, type ScrapeOptions, type ScrapeResult, type StoredPageStub, } from "./types";
+export { CRAWL_SITEMAP_MODES, ENGINES, REGIONS, REQUESTED_ENGINES, isStoredPageStub, type BatchOptions, type ConvertOptions, type CrawlOptions, type CrawlSitemapMode, type DemoResult, type DemoScrapeOptions, type Engine, type ExtractorPreset, type JobRef, type JobResultItem, type JobResultsPage, type JobState, type JobStatus, type JobType, type PageError, type PageFailure, type PageResult, type PageSuccess, type Region, type RequestedEngine, type ScrapeOptions, type ScrapeResult, type StoredPageStub, } from "./types";

package/dist/types/platform/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/platform/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EACN,gBAAgB,EAChB,oBAAoB,GAIpB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,EACN,OAAO,EACP,OAAO,EACP,gBAAgB,GAsBhB,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/platform/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EACN,gBAAgB,EAChB,oBAAoB,GAIpB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,EACN,mBAAmB,EACnB,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,gBAAgB,GAwBhB,MAAM,SAAS,CAAC"}

package/dist/types/platform/types.d.ts

@@ -8,8 +8,16 @@
 /** How the platform acquires the page's HTML. */
 export declare const ENGINES: readonly ["fetch", "browser", "proxy-fetch", "proxy-browser"];
 export type Engine = (typeof ENGINES)[number];
+/**
+ * What a request may ask for: a concrete engine, or `auto` (the server default) — the
+ * cheapest engine for the request, escalated to browser rendering when the fetched HTML is
+ * a client-rendered shell. Responses report the concrete engine that ran.
+ */
+export declare const REQUESTED_ENGINES: readonly ["fetch", "browser", "proxy-fetch", "proxy-browser", "auto"];
+export type RequestedEngine = (typeof REQUESTED_ENGINES)[number];
 /** Coarse egress location, honoured by the proxy engines only. */
-export declare const REGIONS: readonly ["auto", "us", "eu", "uk", "jp", "asia"];
+/** Egress regions the hosted platform can honour (Japan, or no geo-targeting). */
+export declare const REGIONS: readonly ["auto", "jp"];
 export type Region = (typeof REGIONS)[number];
 /** Extraction presets understood by the platform (passed through to the webforai library). */
 export type ExtractorPreset = "auto" | "takumi" | "minimal" | "none";
@@ -21,13 +29,19 @@ export interface ConvertOptions {
     baseUrl?: string;
 }
 interface CommonRequestOptions {
-    /** Defaults to `fetch` server-side. */
-    engine?: Engine;
+    /** Defaults to `auto` server-side. */
+    engine?: RequestedEngine;
     /** Browser engines only (`browser`, `proxy-browser`); +1 credit. */
     screenshot?: boolean;
     /** Re-upload the page's images behind expiring URLs; +1 credit per started 5 images. */
     rehostImages?: boolean;
     region?: Region;
+    /**
+     * Honor the target site's robots.txt rules: each URL is checked before it is fetched, and a
+     * disallowed URL fails with `robots_disallowed` (never billed). Defaults to `false` for
+     * scrape and batch and to `true` for crawl (pass `false` to turn it off).
+     */
+    respectRobotsTxt?: boolean;
     convert?: ConvertOptions;
 }
 export interface ScrapeOptions extends CommonRequestOptions {
@@ -37,6 +51,13 @@ export interface BatchOptions extends CommonRequestOptions {
     /** 1–100 URLs. */
     urls: string[];
 }
+/**
+ * How a crawl uses the site's sitemaps (robots.txt `Sitemap:` lines, else `/sitemap.xml`):
+ * `skip` (default) ignores them; `include` adds their same-origin URLs at depth 1 next to the
+ * links found on pages; `only` crawls the seed plus sitemap URLs and follows no page links.
+ */
+export declare const CRAWL_SITEMAP_MODES: readonly ["skip", "include", "only"];
+export type CrawlSitemapMode = (typeof CRAWL_SITEMAP_MODES)[number];
 export interface CrawlOptions extends CommonRequestOptions {
     url: string;
     /** 0–5, default 2. */
@@ -48,10 +69,13 @@ export interface CrawlOptions extends CommonRequestOptions {
     excludePaths?: string[];
     /** The platform currently only supports same-origin crawls; sending `false` is a 400. */
     sameOrigin?: true;
+    /** Seed the crawl from the site's sitemaps; default `skip`. See {@link CRAWL_SITEMAP_MODES}. */
+    sitemap?: CrawlSitemapMode;
 }
 export interface ScrapeResult {
     /** Final URL after redirects. */
     url: string;
+    /** The engine that produced the result — `auto` requests resolve to a concrete one. */
     engine: Engine;
     markdown: string;
     metadata: Record<string, unknown>;
@@ -63,6 +87,8 @@ export interface ScrapeResult {
         original: string;
         rehosted: string;
     }[];
+    /** Present when the result is probably degraded (e.g. an unrendered client-side shell). */
+    warning?: string;
 }
 export interface JobRef {
     jobId: string;
@@ -92,7 +118,8 @@ export type PageSuccess = ScrapeResult & {
 export interface PageFailure {
     status: "error";
     url: string;
-    engine: Engine;
+    /** As requested — a failed `auto` page never resolved to a concrete engine. */
+    engine: RequestedEngine;
     error: PageError;
 }
 /** One page of an async job, after the client resolves any spilled results. */
@@ -105,7 +132,7 @@ export type PageResult = PageSuccess | PageFailure;
 export interface StoredPageStub {
     status: "ok";
     url: string;
-    engine: Engine;
+    engine: RequestedEngine;
     credits: number;
     resultUrl: string;
 }
@@ -124,7 +151,9 @@ export interface DemoScrapeOptions {
 export interface DemoResult {
     url: string;
     region: Region;
-    /** Truncated to ~8000 characters by the demo endpoint. */
+    /** The engine `auto` resolved to. Absent from deployments older than 2026-09-24. */
+    engine?: Engine;
+    /** Truncated to ~40,000 characters by the demo endpoint (~8000 before 2026-09-24). */
     markdown: string;
     truncated: boolean;
     title?: string;

package/dist/types/platform/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/platform/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,iDAAiD;AACjD,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,eAAe,CAAU,CAAC;AAGrF,kEAAkE;AAClE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAU,CAAC;AAwIzE,+DAA+D;AAC/D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAmB,EAA0B,EAAE,CAC/E,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,WAAW,IAAI,IAAI,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/platform/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,iDAAiD;AACjD,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,eAAe,CAAU,CAAC;AAGrF;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAG,OAAO,EAAE,MAAM,CAAU,CAAC;AAG/D,kEAAkE;AAClE,kFAAkF;AAClF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,IAAI,CAAU,CAAC;AAwC/C;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAU,CAAC;AAiHxE,+DAA+D;AAC/D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAmB,EAA0B,EAAE,CAC/E,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,WAAW,IAAI,IAAI,CAAC"}

package/package.json

@@ -1,23 +1,34 @@
 {
   "name": "webforai",
-  "version": "3.0.0",
-  "description": "A library that provides a web interface for AI",
+  "version": "4.0.0",
+  "description": "Convert web pages and local HTML to clean, LLM-ready Markdown: library, CLI (npx webforai) and hosted-API client.",
   "author": "inaridiy",
   "license": "Apache-2.0",
   "keywords": [
-    "web",
+    "html-to-markdown",
+    "markdown",
+    "llm",
     "ai",
+    "scraper",
+    "crawler",
+    "readability",
+    "web-scraping",
+    "cli",
+    "rag",
     "html",
     "html2md",
-    "markdown",
     "mdast",
     "hast"
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/inaridiy/webforai.git"
+    "url": "https://github.com/inaridiy/webforai.git",
+    "directory": "packages/webforai"
   },
   "homepage": "https://webforai.dev/",
+  "bugs": {
+    "url": "https://github.com/inaridiy/webforai/issues"
+  },
   "files": [
     "dist",
     "!dist/types/**/*.js"