webforai 2.1.1 → 3.0.0

package/dist/types/platform/types.d.ts

@@ -0,0 +1,135 @@
+/**
+ * Wire types for the webforai platform HTTP API (`/v1`).
+ *
+ * These mirror the server contract exactly (`apps/platform/src/routes/*` in the webforai
+ * repository is the source of truth). Request bodies are validated strictly server-side:
+ * unknown keys are a `400`, so the client only ever sends fields the caller provided.
+ */
+/** How the platform acquires the page's HTML. */
+export declare const ENGINES: readonly ["fetch", "browser", "proxy-fetch", "proxy-browser"];
+export type Engine = (typeof ENGINES)[number];
+/** Coarse egress location, honoured by the proxy engines only. */
+export declare const REGIONS: readonly ["auto", "us", "eu", "uk", "jp", "asia"];
+export type Region = (typeof REGIONS)[number];
+/** Extraction presets understood by the platform (passed through to the webforai library). */
+export type ExtractorPreset = "auto" | "takumi" | "minimal" | "none";
+export interface ConvertOptions {
+    extractor?: ExtractorPreset;
+    /** Prepend YAML front matter built from the page's metadata. */
+    frontmatter?: boolean;
+    /** Base URL used to absolutize relative links; defaults to the fetched URL. */
+    baseUrl?: string;
+}
+interface CommonRequestOptions {
+    /** Defaults to `fetch` server-side. */
+    engine?: Engine;
+    /** Browser engines only (`browser`, `proxy-browser`); +1 credit. */
+    screenshot?: boolean;
+    /** Re-upload the page's images behind expiring URLs; +1 credit per started 5 images. */
+    rehostImages?: boolean;
+    region?: Region;
+    convert?: ConvertOptions;
+}
+export interface ScrapeOptions extends CommonRequestOptions {
+    url: string;
+}
+export interface BatchOptions extends CommonRequestOptions {
+    /** 1–100 URLs. */
+    urls: string[];
+}
+export interface CrawlOptions extends CommonRequestOptions {
+    url: string;
+    /** 0–5, default 2. */
+    maxDepth?: number;
+    /** 1–500 pages, default 50. */
+    limit?: number;
+    /** Regexes matched against the pathname. */
+    includePaths?: string[];
+    excludePaths?: string[];
+    /** The platform currently only supports same-origin crawls; sending `false` is a 400. */
+    sameOrigin?: true;
+}
+export interface ScrapeResult {
+    /** Final URL after redirects. */
+    url: string;
+    engine: Engine;
+    markdown: string;
+    metadata: Record<string, unknown>;
+    credits: number;
+    /** Present when a screenshot was requested; the URL expires after ~24h. */
+    screenshotUrl?: string;
+    /** Present when `rehostImages` was requested. */
+    images?: {
+        original: string;
+        rehosted: string;
+    }[];
+}
+export interface JobRef {
+    jobId: string;
+}
+export type JobType = "batch" | "crawl";
+export type JobState = "queued" | "running" | "completed" | "failed";
+export interface JobStatus {
+    jobId: string;
+    type: JobType;
+    status: JobState;
+    total: number;
+    completed: number;
+    failed: number;
+    credits: number;
+    /** When the stored results disappear (ISO timestamp, 7 days after creation). */
+    expiresAt: string;
+    /** Present only when the job failed. */
+    error?: string;
+}
+export interface PageError {
+    code: string;
+    message: string;
+}
+export type PageSuccess = ScrapeResult & {
+    status: "ok";
+};
+export interface PageFailure {
+    status: "error";
+    url: string;
+    engine: Engine;
+    error: PageError;
+}
+/** One page of an async job, after the client resolves any spilled results. */
+export type PageResult = PageSuccess | PageFailure;
+/**
+ * Results larger than ~100KiB are stored out-of-band; the API returns this stub with an
+ * expiring `resultUrl` instead of the inline markdown. `jobResults()` resolves these
+ * automatically; `getJobResults()` returns them as-is.
+ */
+export interface StoredPageStub {
+    status: "ok";
+    url: string;
+    engine: Engine;
+    credits: number;
+    resultUrl: string;
+}
+export type JobResultItem = PageResult | StoredPageStub;
+export interface JobResultsPage {
+    jobId: string;
+    status: JobState;
+    results: JobResultItem[];
+    /** Absent on the last page. */
+    cursor?: string;
+}
+export interface DemoScrapeOptions {
+    url: string;
+    region?: Region;
+}
+export interface DemoResult {
+    url: string;
+    region: Region;
+    /** Truncated to ~8000 characters by the demo endpoint. */
+    markdown: string;
+    truncated: boolean;
+    title?: string;
+    metadata: Record<string, unknown>;
+}
+/** Narrows a job-results item to the out-of-band stub form. */
+export declare const isStoredPageStub: (item: JobResultItem) => item is StoredPageStub;
+export {};

package/dist/types/platform/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/platform/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,iDAAiD;AACjD,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,eAAe,CAAU,CAAC;AAGrF,kEAAkE;AAClE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAU,CAAC;AAwIzE,+DAA+D;AAC/D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAmB,EAA0B,EAAE,CAC/E,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,WAAW,IAAI,IAAI,CAAC"}

package/dist/types/utils/hast-fast.d.ts

@@ -0,0 +1,101 @@
+/**
+ * Performance-critical HAST helpers.
+ *
+ * The v2 extractors leaned on `unist-util-filter` and `hast-util-to-string`, both of which
+ * walk (and in the case of `filter`, deep-clone) the entire tree on every call. A single run
+ * of `takumiExtractor` therefore cloned the document four times and re-walked it for every
+ * length measurement. The helpers here replace both with a single mutation pass and a
+ * memoised post-order metric walk.
+ */
+import type { Element, Nodes as Hast, Parent, RootContent } from "hast";
+/** Text metrics for a subtree, gathered in one post-order pass. */
+export interface TextMetrics {
+    /** Length of the whitespace-normalised text content. */
+    text: number;
+    /** Length of text that sits inside an `<a>` descendant. */
+    link: number;
+    /** Number of sentence-separating commas (ASCII, CJK and Arabic). */
+    commas: number;
+    /** Number of `<p>`, `<pre>` and `<li>` descendants. */
+    paragraphs: number;
+    /** Number of element descendants (self excluded). */
+    elements: number;
+}
+export declare const isElement: (node: unknown) => node is Element;
+export declare const isParent: (node: unknown) => node is Parent;
+/** Normalised class list, tolerating both the array and the raw-string shapes hast can produce. */
+export declare const classList: (element: Element) => string[];
+/** `"tagname#id .class1 .class2"` — the string the heuristic regexps are matched against. */
+export declare const matchString: (element: Element) => string;
+/**
+ * A memoised metric collector bound to one snapshot of a tree.
+ *
+ * The cache is only sound while the tree is unchanged, so callers create a fresh collector
+ * after every mutation pass rather than invalidating entries individually.
+ */
+export declare class MetricsCollector {
+    #private;
+    metrics(node: Hast): TextMetrics;
+    textLength(node: Hast): number;
+    /** Fraction of a subtree's text that is link anchor text. `0` for empty subtrees. */
+    linkDensity(node: Hast): number;
+}
+/**
+ * Removes every node for which `keep` returns `false`, mutating `tree` in place.
+ *
+ * Unlike `unist-util-filter` this performs no cloning, so callers own the responsibility of
+ * copying the tree beforehand when the input must stay untouched.
+ *
+ * `keep` is called top-down; returning `false` for a node skips its subtree entirely, which is
+ * what makes a single pass sufficient.
+ */
+export declare const pruneInPlace: (tree: Hast, keep: (node: Hast) => boolean) => Hast;
+/**
+ * Depth-first pre-order walk. Return `false` from `visit` to skip a subtree.
+ *
+ * `boolean | void` is deliberate: it is the standard visitor signature (unist-util-visit uses
+ * the same shape) and lets callers that only observe nodes omit a return entirely.
+ */
+export declare const walk: (tree: Hast, visit: (node: Hast, parent: Parent | undefined) => boolean | void) => void;
+/**
+ * Collects every element matching `predicate`.
+ *
+ * This exists to replace repeated `hast-util-select` calls: the CSS engine re-parses the
+ * selector and re-walks the tree per call, while callers here usually only need a tag-name or
+ * attribute test that a single traversal can answer.
+ */
+export declare const collectElements: (tree: Hast, predicate: (element: Element) => boolean) => Element[];
+/** First element matching `predicate`, in document order. */
+export declare const findElement: (tree: Hast, predicate: (element: Element) => boolean) => Element | undefined;
+/**
+ * Structural clone of a HAST tree.
+ *
+ * `structuredClone` is available on every runtime the package targets (Node 18+, Deno, Bun,
+ * Workers) and is markedly faster than a JSON round trip, but it rejects the `Object.create(null)`
+ * prototypes that some HAST producers use for `properties`, so we fall back on a manual copy.
+ */
+export declare const cloneHast: <T extends Hast>(tree: T) => T;
+/** Wraps loose nodes into a root so downstream utilities always receive a single tree. */
+export declare const asRoot: (nodes: Hast | RootContent[]) => Hast;
+/**
+ * Reads a raw property, accepting the attribute spelling as written in HTML.
+ *
+ * `hast-util-from-html` stores attributes under their DOM property names, so `aria-hidden`
+ * becomes `ariaHidden` and `data-rwidth` becomes `dataRwidth`. Looking up the hyphenated form
+ * directly always misses — a mistake that silently disabled several v2 filters — so this
+ * helper checks the verbatim key first and then the camel-cased spelling.
+ */
+export declare const property: (element: Element, key: string) => unknown;
+/** Reads a property as a finite number, tolerating the string form attributes arrive in. */
+export declare const numericProperty: (element: Element, key: string) => number | undefined;
+/** Reads a property as a string, joining the array form hast uses for token lists. */
+export declare const stringProperty: (element: Element, key: string) => string | undefined;
+/** True when the attribute is present at all, regardless of its value. */
+export declare const hasProperty: (element: Element, key: string) => boolean;
+/**
+ * True for attributes that follow the ARIA/HTML "true unless explicitly false" convention.
+ *
+ * Boolean HTML attributes (`hidden`) arrive as `true`, while ARIA states arrive as the strings
+ * `"true"`/`"false"`, and an empty string means present-and-true.
+ */
+export declare const isTruthyAttribute: (element: Element, key: string) => boolean;

package/dist/types/utils/hast-fast.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hast-fast.js","sourceRoot":"","sources":["../../../src/utils/hast-fast.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;;;;;;;AAkBH,MAAM,aAAa,GAAgB,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;AAE/F,+EAA+E;AAC/E,MAAM,WAAW,GAAG,SAAS,CAAC;AAE9B,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAE3F;;;GAGG;AACH,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE/G,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAa,EAAmB,EAAE,CAC3D,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAK,IAAa,CAAC,IAAI,KAAK,SAAS,CAAC;AAEhF,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAkB,EAAE,CACzD,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,IAAe,CAAC,QAAQ,CAAC,CAAC;AAEvF,mGAAmG;AACnG,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,OAAgB,EAAY,EAAE;IACvD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC;IAChD,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,OAAO,SAAqB,CAAC;IAC9B,CAAC;IACD,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,EAAE,CAAC;AACX,CAAC,CAAC;AAEF,6FAA6F;AAC7F,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAU,EAAE;IACvD,MAAM,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,OAAO,CAAC,OAAO,CAAC;IACxB,CAAC;IACD,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACtF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,OAAO,gBAAgB;IAA7B;QACU,kCAAS,IAAI,OAAO,EAAuB,EAAC;IA0EtD,CAAC;IAxEA,OAAO,CAAC,IAAU;QACjB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/B,OAAO,aAAa,CAAC;YACtB,CAAC;YACD,OAAO;gBACN,IAAI,EAAE,KAAK,CAAC,MAAM;gBAClB,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM;gBAC/C,UAAU,EAAE,CAAC;gBACb,QAAQ,EAAE,CAAC;aACX,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,OAAO,aAAa,CAAC;QACtB,CAAC;QAED,MAAM,MAAM,GAAG,uBAAA,IAAI,+BAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,MAAM,EAAE,CAAC;YACZ,OAAO,MAAM,CAAC;QACf,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzD,uBAAA,IAAI,+BAAO,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;YACrC,OAAO,aAAa,CAAC;QACtB,CAAC;QAED,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,KAAa,CAAC,CAAC;YACjD,IAAI,IAAI,YAAY,CAAC,IAAI,CAAC;YAC1B,IAAI,IAAI,YAAY,CAAC,IAAI,CAAC;YAC1B,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC;YAC9B,UAAU,IAAI,YAAY,CAAC,UAAU,CAAC;YACtC,QAAQ,IAAI,YAAY,CAAC,QAAQ,CAAC;YAClC,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtB,QAAQ,IAAI,CAAC,CAAC;gBACd,IAAI,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;oBACvC,UAAU,IAAI,CAAC,CAAC;gBACjB,CAAC;YACF,CAAC;QACF,CAAC;QAED,iFAAiF;QACjF,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,GAAG,EAAE,CAAC;YAC7C,IAAI,GAAG,IAAI,CAAC;QACb,CAAC;QAED,MAAM,OAAO,GAAgB,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;QAC1E,uBAAA,IAAI,+BAAO,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/B,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,UAAU,CAAC,IAAU;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;IAChC,CAAC;IAED,qFAAqF;IACrF,WAAW,CAAC,IAAU;QACrB,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YAChB,OAAO,CAAC,CAAC;QACV,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;IACjC,CAAC;CACD;;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAU,EAAE,IAA6B,EAAQ,EAAE;IAC/E,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,KAAK,GAAa,CAAC,IAAI,CAAC,CAAC;IAE/B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,4EAA4E;QAC5E,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,0FAA0F;QAC1F,6DAA6D;QAC7D,uEAAuE;QACvE,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAS,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAClB,SAAS;YACV,CAAC;YACD,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,KAAoB,CAAC;YACzC,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,CAAC;QACF,CAAC;QAED,QAAQ,CAAC,MAAM,GAAG,KAAK,CAAC;IACzB,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF;;;;;GAKG;AACH,0FAA0F;AAC1F,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAU,EAAE,KAAiE,EAAQ,EAAE;IAC3G,MAAM,KAAK,GAAsD,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAErG,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,4EAA4E;QAC5E,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;QACtC,IAAI,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,KAAK,EAAE,CAAC;YACnC,SAAS;QACV,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,0EAA0E;YAC1E,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpD,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9D,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAU,EAAE,SAAwC,EAAa,EAAE;IAClG,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;IACF,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,6DAA6D;AAC7D,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAU,EAAE,SAAwC,EAAuB,EAAE;IACxG,IAAI,KAA0B,CAAC;IAC/B,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,KAAK,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,KAAK,GAAG,IAAI,CAAC;YACb,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAiB,IAAO,EAAK,EAAE;IACvD,IAAI,CAAC;QACJ,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAI,IAAO,EAAK,EAAE;IACrC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,CAAM,CAAC;IACnC,CAAC;IACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,IAAI,GAA4B,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAS,CAAC;AAClB,CAAC,CAAC;AAEF,0FAA0F;AAC1F,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,KAA2B,EAAQ,EAAE;IAC3D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,OAAgB,EAAE,GAAW,EAAW,EAAE;IAClE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IACtC,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IACf,CAAC;IACD,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,OAAO,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,kBAAkB,GAAG,CAAC,GAAW,EAAU,EAAE,CAClD,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,EAAE,SAAiB,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;AAEhF,4FAA4F;AAC5F,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,OAAgB,EAAE,GAAW,EAAsB,EAAE;IACpF,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/C,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACtC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACrD,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF,sFAAsF;AACtF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,OAAgB,EAAE,GAAW,EAAsB,EAAE;IACnF,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC;IACZ,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,CAAC;IACX,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF,0EAA0E;AAC1E,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,GAAW,EAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,SAAS,CAAC;AAE5G;;;;;GAKG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,OAAgB,EAAE,GAAW,EAAW,EAAE;IAC3E,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QACxD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}

package/dist/utils/hast-fast.js

@@ -0,0 +1,246 @@
+// src/utils/hast-fast.ts
+var EMPTY_METRICS = { text: 0, link: 0, commas: 0, paragraphs: 0, elements: 0 };
+var COMMA_REGEX = /[,、，،]/g;
+var PARAGRAPH_LIKE = /* @__PURE__ */ new Set(["p", "pre", "li", "blockquote", "figcaption", "dd", "dt"]);
+var NON_PROSE_TAGS = /* @__PURE__ */ new Set(["script", "style", "noscript", "template", "svg", "math", "iframe", "object"]);
+var isElement = (node) => typeof node === "object" && node !== null && node.type === "element";
+var isParent = (node) => typeof node === "object" && node !== null && Array.isArray(node.children);
+var classList = (element) => {
+  const className = element.properties?.className;
+  if (Array.isArray(className)) {
+    return className;
+  }
+  if (typeof className === "string") {
+    return className.split(/\s+/).filter(Boolean);
+  }
+  return [];
+};
+var matchString = (element) => {
+  const id = element.properties?.id;
+  const classes = classList(element);
+  if (!id && classes.length === 0) {
+    return element.tagName;
+  }
+  return `${element.tagName} ${typeof id === "string" ? id : ""} ${classes.join(" ")}`;
+};
+var MetricsCollector = class {
+  #cache = /* @__PURE__ */ new WeakMap();
+  metrics(node) {
+    if (node.type === "text") {
+      const value = node.value;
+      if (value.trim().length === 0) {
+        return EMPTY_METRICS;
+      }
+      return {
+        text: value.length,
+        link: 0,
+        commas: (value.match(COMMA_REGEX) ?? []).length,
+        paragraphs: 0,
+        elements: 0
+      };
+    }
+    if (!isParent(node)) {
+      return EMPTY_METRICS;
+    }
+    const cached = this.#cache.get(node);
+    if (cached) {
+      return cached;
+    }
+    if (isElement(node) && NON_PROSE_TAGS.has(node.tagName)) {
+      this.#cache.set(node, EMPTY_METRICS);
+      return EMPTY_METRICS;
+    }
+    let text = 0;
+    let link = 0;
+    let commas = 0;
+    let paragraphs = 0;
+    let elements = 0;
+    for (const child of node.children) {
+      const childMetrics = this.metrics(child);
+      text += childMetrics.text;
+      link += childMetrics.link;
+      commas += childMetrics.commas;
+      paragraphs += childMetrics.paragraphs;
+      elements += childMetrics.elements;
+      if (isElement(child)) {
+        elements += 1;
+        if (PARAGRAPH_LIKE.has(child.tagName)) {
+          paragraphs += 1;
+        }
+      }
+    }
+    if (isElement(node) && node.tagName === "a") {
+      link = text;
+    }
+    const metrics = { text, link, commas, paragraphs, elements };
+    this.#cache.set(node, metrics);
+    return metrics;
+  }
+  textLength(node) {
+    return this.metrics(node).text;
+  }
+  /** Fraction of a subtree's text that is link anchor text. `0` for empty subtrees. */
+  linkDensity(node) {
+    const { text, link } = this.metrics(node);
+    if (text === 0) {
+      return 0;
+    }
+    return Math.min(1, link / text);
+  }
+};
+var pruneInPlace = (tree, keep) => {
+  if (!isParent(tree)) {
+    return tree;
+  }
+  const stack = [tree];
+  while (stack.length > 0) {
+    const parent = stack.pop();
+    const children = parent.children;
+    let write = 0;
+    for (let read = 0; read < children.length; read++) {
+      const child = children[read];
+      if (!keep(child)) {
+        continue;
+      }
+      children[write++] = child;
+      if (isParent(child)) {
+        stack.push(child);
+      }
+    }
+    children.length = write;
+  }
+  return tree;
+};
+var walk = (tree, visit) => {
+  const stack = [{ node: tree, parent: void 0 }];
+  while (stack.length > 0) {
+    const { node, parent } = stack.pop();
+    if (visit(node, parent) === false) {
+      continue;
+    }
+    if (isParent(node)) {
+      for (let i = node.children.length - 1; i >= 0; i--) {
+        stack.push({ node: node.children[i], parent: node });
+      }
+    }
+  }
+};
+var collectElements = (tree, predicate) => {
+  const found = [];
+  walk(tree, (node) => {
+    if (isElement(node) && predicate(node)) {
+      found.push(node);
+    }
+  });
+  return found;
+};
+var findElement = (tree, predicate) => {
+  let found;
+  walk(tree, (node) => {
+    if (found) {
+      return false;
+    }
+    if (isElement(node) && predicate(node)) {
+      found = node;
+      return false;
+    }
+  });
+  return found;
+};
+var cloneHast = (tree) => {
+  try {
+    return structuredClone(tree);
+  } catch {
+    return manualClone(tree);
+  }
+};
+var manualClone = (node) => {
+  if (Array.isArray(node)) {
+    return node.map(manualClone);
+  }
+  if (typeof node !== "object" || node === null) {
+    return node;
+  }
+  const copy = {};
+  for (const [key, value] of Object.entries(node)) {
+    copy[key] = manualClone(value);
+  }
+  return copy;
+};
+var asRoot = (nodes) => {
+  if (Array.isArray(nodes)) {
+    return { type: "root", children: nodes };
+  }
+  return nodes;
+};
+var property = (element, key) => {
+  const properties = element.properties;
+  if (!properties) {
+    return void 0;
+  }
+  const direct = properties[key];
+  if (direct !== void 0) {
+    return direct;
+  }
+  if (!(key.includes("-") || key.includes(":"))) {
+    return void 0;
+  }
+  return properties[camelCaseAttribute(key)];
+};
+var camelCaseAttribute = (key) => key.replace(/[-:]([a-z])/g, (_, character) => character.toUpperCase());
+var numericProperty = (element, key) => {
+  const raw = property(element, key);
+  if (typeof raw === "number") {
+    return Number.isFinite(raw) ? raw : void 0;
+  }
+  if (typeof raw === "string" && raw.length > 0) {
+    const parsed = Number.parseFloat(raw);
+    return Number.isFinite(parsed) ? parsed : void 0;
+  }
+  return void 0;
+};
+var stringProperty = (element, key) => {
+  const raw = property(element, key);
+  if (typeof raw === "string") {
+    return raw;
+  }
+  if (Array.isArray(raw)) {
+    return raw.join(" ");
+  }
+  if (raw === true) {
+    return "";
+  }
+  return void 0;
+};
+var hasProperty = (element, key) => property(element, key) !== void 0;
+var isTruthyAttribute = (element, key) => {
+  const raw = property(element, key);
+  if (raw === void 0 || raw === null || raw === false) {
+    return false;
+  }
+  if (raw === true) {
+    return true;
+  }
+  if (typeof raw === "string") {
+    return raw.toLowerCase() !== "false";
+  }
+  return true;
+};
+export {
+  MetricsCollector,
+  asRoot,
+  classList,
+  cloneHast,
+  collectElements,
+  findElement,
+  hasProperty,
+  isElement,
+  isParent,
+  isTruthyAttribute,
+  matchString,
+  numericProperty,
+  property,
+  pruneInPlace,
+  stringProperty,
+  walk
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webforai",
-  "version": "2.1.1",
+  "version": "3.0.0",
   "description": "A library that provides a web interface for AI",
   "author": "inaridiy",
   "license": "Apache-2.0",
@@ -38,6 +38,11 @@
       "import": "./dist/index.js",
       "require": "./dist/cjs/index.js"
     },
+    "./platform": {
+      "types": "./dist/types/platform/index.d.ts",
+      "import": "./dist/platform/index.js",
+      "require": "./dist/cjs/platform/index.js"
+    },
     "./loaders/playwright": {
       "types": "./dist/types/loaders/playwright.d.ts",
       "import": "./dist/loaders/playwright.js",
@@ -75,6 +80,9 @@
       ],
       "loaders/puppeteer": [
         "./dist/types/loaders/puppeteer.d.ts"
+      ],
+      "platform": [
+        "./dist/types/platform/index.d.ts"
       ]
     }
   },
@@ -110,8 +118,7 @@
     "mdast-util-to-markdown": "^2.1.0",
     "picocolors": "^1.0.1",
     "trim-trailing-lines": "^2.1.0",
-    "unist-util-filter": "^5.0.1",
-    "zx": "^8.1.5"
+    "unist-util-filter": "^5.0.1"
   },
   "devDependencies": {
     "@cloudflare/puppeteer": "^0.0.6",

package/dist/types/cli/commands/webforai/index.d.ts

@@ -1,6 +0,0 @@
-export declare const webforaiCommand: (initialPath: string, options: {
-    output?: string;
-    mode?: string;
-    loader?: string;
-    debug?: boolean;
-}) => Promise<void>;

package/dist/types/cli/commands/webforai/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/cli/commands/webforai/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5B,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,MAAM,aAAa,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAC/E,MAAM,sBAAsB,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAE3F,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EACnC,WAAmB,EACnB,OAA6E,EAC5E,EAAE;IACH,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,wBAAwB,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAExE,MAAM,UAAU,GAAG,WAAW,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAEvD,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACtF,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;IAE/C,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;IACzE,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAEvD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC;IACzD,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAE3C,IAAI,IAAY,CAAC;IACjB,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;IACpB,IAAI,CAAC;QACJ,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAC9B,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACpE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,CAAC;QAC1C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAE3C,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE;QACrC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QACnD,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,sBAAsB,CAAC;KAC3D,CAAC,CAAC;IACH,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;IAEnD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3C,MAAM,iBAAiB,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IAEtF,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACxB,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,CAAC;IACD,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEzC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC,CAAC"}