webforai 3.0.0 → 4.1.0

package/dist/types/detect-client-shell.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Heuristic detection of HTML documents that carry no readable content because the page is
+ * built client-side (SPA shells), gated behind an anti-bot interstitial, or simply empty.
+ *
+ * Converting such a document yields a markdown body that is at best the `<title>` — the
+ * caller almost always wants to re-fetch with a JavaScript-rendering engine instead. The
+ * verdict is advisory: thresholds are tuned so that short-but-real static pages (an
+ * example.com-sized document) are NOT flagged.
+ *
+ * Expects a full document; fragments without `<body>` are judged as a whole and tiny
+ * fragments will read as `empty-body`.
+ */
+export type ClientShellReason = "empty-body" | "spa-shell" | "noscript-only" | "anti-bot-challenge";
+export interface ClientShellVerdict {
+    isShell: boolean;
+    reason?: ClientShellReason;
+    /** Characters of visible body text after stripping markup — what a conversion could use. */
+    visibleTextLength: number;
+}
+/** Body text at or above this length is always treated as real content. */
+export declare const SHELL_SUSPICION_THRESHOLD = 300;
+/** Below this length the document is a shell even without a framework marker. */
+export declare const SHELL_EMPTY_THRESHOLD = 50;
+export declare const detectClientShell: (html: string) => ClientShellVerdict;

package/dist/types/detect-client-shell.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"detect-client-shell.js","sourceRoot":"","sources":["../../src/detect-client-shell.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAWH,2EAA2E;AAC3E,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAC7C,iFAAiF;AACjF,MAAM,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAExC,MAAM,YAAY,GAAG,gCAAgC,CAAC;AACtD,MAAM,YAAY,GAAG,+BAA+B,CAAC;AACrD,MAAM,gBAAgB,GAAG,yCAAyC,CAAC;AACnE,MAAM,yBAAyB,GAAG,6DAA6D,CAAC;AAChG,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAC3C,MAAM,WAAW,GAAG,UAAU,CAAC;AAE/B;;;GAGG;AACH,MAAM,mBAAmB,GACxB,6HAA6H,CAAC;AAE/H,+FAA+F;AAC/F,MAAM,iBAAiB,GACtB,8HAA8H,CAAC;AAEhI,MAAM,QAAQ,GAA2B;IACxC,QAAQ,EAAE,GAAG;IACb,OAAO,EAAE,GAAG;IACZ,MAAM,EAAE,GAAG;IACX,MAAM,EAAE,GAAG;IACX,QAAQ,EAAE,GAAG;IACb,OAAO,EAAE,GAAG;CACZ,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,MAAc,EAAU,EAAE,CAChD,MAAM;KACJ,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC;KAC7B,OAAO,CAAC,yBAAyB,EAAE,GAAG,CAAC;KACvC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;KACzB,OAAO,CAAC,gCAAgC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC;KAC9E,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;KACpB,IAAI,EAAE,CAAC;AAEV,MAAM,0BAA0B,GAAG,CAAC,IAAY,EAAW,EAAE;IAC5D,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACrD,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YACxC,OAAO,IAAI,CAAC;QACb,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAsB,EAAE;IACrE,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IAC7E,MAAM,iBAAiB,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAErD,IAAI,iBAAiB,IAAI,yBAAyB,EAAE,CAAC;QACpD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC9C,CAAC;IACD,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,CAAC;IAC3E,CAAC;IACD,IAAI,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,CAAC;IAClE,CAAC;IACD,IAAI,0BAA0B,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,iBAAiB,EAAE,CAAC;IACtE,CAAC;IACD,IAAI,iBAAiB,GAAG,qBAAqB,EAAE,CAAC;QAC/C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,iBAAiB,EAAE,CAAC;IACnE,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC;AAC9C,CAAC,CAAC"}

package/dist/types/extract-mdast.js.map

@@ -1 +1 @@
-{"version":3,"file":"extract-mdast.js","sourceRoot":"","sources":["../../src/extract-mdast.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,eAAe,GAAG,CAAC,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AAEvE,MAAM,sBAAsB,GAAG,CAAC,IAAW,EAAE,EAAE;IAC9C,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAK,IAAe,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAW,EAAE,EAAE;IAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QACvC,IAAI,CAAC,sBAAsB,CAAC,IAAa,CAAC,EAAE,CAAC;YAC5C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IACH,OAAO,SAAkB,CAAC;AAC3B,CAAC,CAAC"}
+{"version":3,"file":"extract-mdast.js","sourceRoot":"","sources":["../../src/extract-mdast.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,eAAe,GAAG,CAAC,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AAEvE,MAAM,sBAAsB,GAAG,CAAC,IAAW,EAAE,EAAE;IAC9C,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAK,IAAe,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,QAA2B,EAAE,IAAqB,EAAqB,EAAE;IAC1F,MAAM,KAAK,GAAG,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,IAAI,EAAE,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3B,OAAO,QAAQ,CAAC;IACjB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;IAC/E,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;IAC3B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACP,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG,CAAC,SAAoB,EAA6B,EAAE;IAC3E,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAE,KAAe,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,CAAC;QAC3E,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,IAAI,GAAG,GAAsB,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,GAAG,EAAE;QAClB,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;QACxD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/C,CAAC;QACD,GAAG,GAAG,EAAE,CAAC;IACV,CAAC,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;QACxC,IAAK,KAAe,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACtC,KAAK,EAAE,CAAC;YACR,GAAG,CAAC,IAAI,CAAC,KAA+B,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACP,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjB,CAAC;IACF,CAAC;IACD,KAAK,EAAE,CAAC;IACR,OAAO,GAAG,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAW,EAAQ,EAAE;IAC/C,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,EAAE,CAAC;QAC3B,OAAO;IACR,CAAC;IACD,MAAM,MAAM,GAAG,IAAc,CAAC;IAC9B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAC7D,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAU,CAAC;QAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/E,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,EAAE,GAAI,MAA6B,CAAC,CAAC;YACpE,KAAK,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YAC3B,SAAS;QACV,CAAC;QACD,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAW,EAAE,EAAE;IAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QACvC,IAAI,CAAC,sBAAsB,CAAC,IAAa,CAAC,EAAE,CAAC;YAC5C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IACH,IAAI,SAAS,EAAE,CAAC;QACf,iBAAiB,CAAC,SAAkB,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,SAAkB,CAAC;AAC3B,CAAC,CAAC"}

package/dist/types/extract-meta-refresh.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Meta-refresh redirect detection.
+ *
+ * Some sites answer HTTP 200 with a stub page whose only job is to send the browser
+ * elsewhere — `<meta http-equiv="refresh" content="0; url=/ja/">`, typically paired with a
+ * JavaScript fallback (GitHub Pages language redirects are the canonical case). HTTP-level
+ * redirect following never triggers on these, so a fetch-based pipeline converts the stub
+ * and loses the page. This parser lets loaders treat such a page as the redirect it is.
+ *
+ * A refresh only counts as a redirect when it names a URL, the URL resolves to http(s),
+ * the target is not the page itself, and the delay is small — long delays are auto-reload
+ * timers (live blogs, dashboards), not redirects.
+ */
+export interface MetaRefreshTarget {
+    /** The redirect target, resolved against `baseUrl` when one was given. */
+    url: string;
+    delaySeconds: number;
+}
+/** Above this delay a refresh is an auto-reload timer, not a redirect. */
+export declare const MAX_META_REFRESH_DELAY_SECONDS = 10;
+/**
+ * Extracts the redirect a page's meta-refresh declares, if any.
+ *
+ * @param html - The fetched document.
+ * @param baseUrl - The document's own URL; used to resolve relative targets and to reject
+ *   self-refreshes. Without it, only absolute targets are recognised.
+ * @returns The target and delay, or `undefined` when the page declares no usable redirect.
+ */
+export declare const extractMetaRefresh: (html: string, baseUrl?: string) => MetaRefreshTarget | undefined;

package/dist/types/extract-meta-refresh.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-meta-refresh.js","sourceRoot":"","sources":["../../src/extract-meta-refresh.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAQH,0EAA0E;AAC1E,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,CAAC;AAEjD,MAAM,gBAAgB,GAAG,iBAAiB,CAAC;AAC3C,MAAM,0BAA0B,GAAG,iEAAiE,CAAC;AACrG,MAAM,oBAAoB,GAAG,oDAAoD,CAAC;AAElF,4FAA4F;AAC5F,MAAM,qBAAqB,GAAG,2EAA2E,CAAC;AAE1G,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,OAA2B,EAAmB,EAAE;IACnF,IAAI,CAAC;QACJ,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAE,OAAgB,EAAiC,EAAE;IACnG,KAAK,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,SAAS;QACV,CAAC;QACD,MAAM,YAAY,GAAG,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5E,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAG,qBAAqB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,SAAS;QACV,CAAC;QACD,6DAA6D;QAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,IAAI,CAAC,GAAG,IAAI,YAAY,GAAG,8BAA8B,EAAE,CAAC;YAC3D,SAAS;QACV,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC,EAAE,CAAC;YAC9E,SAAS;QACV,CAAC;QACD,IAAI,OAAO,KAAK,SAAS,IAAI,aAAa,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,IAAI,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC;YACtF,kFAAkF;YAClF,SAAS;QACV,CAAC;QAED,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC;IAC3C,CAAC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}

package/dist/types/extractors/index.d.ts

@@ -1,5 +1,6 @@
 export { pipeExtractors, type ExtractorSelectors, type ExtractorSelector, } from "./pipeExtractors";
 export { takumiExtractor } from "./presets/takumi";
+export { createKiwameExtractor, kiwameExtractor, type KiwameExtractorOptions } from "./presets/kiwame";
 export { type ExtractParams, type Extractor } from "./types";
 export { minimalFilter } from "./presets/minimal-filter";
 export { autoExtractor, createAutoExtractor, type AutoExtractorOptions } from "./presets/auto";

package/dist/types/extractors/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/extractors/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EACN,cAAc,GAGd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAA6B,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/extractors/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EACN,cAAc,GAGd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAA+B,MAAM,kBAAkB,CAAC;AAEvG,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAA6B,MAAM,gBAAgB,CAAC"}

package/dist/types/extractors/lib/block-features.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Per-block features for the learned main-content model.
+ *
+ * Everything here is cheap and local: text statistics of the block and its neighbours (the
+ * shallow features of Kohlschütter et al., WSDM 2010), the structure around it, and the
+ * hand-written heuristics of the rest of the extractor turned into evidence rather than
+ * decisions — class-name patterns, the furniture tiers and candidate scores. Computing them does
+ * not modify the tree.
+ */
+import type { Element, Nodes as Hast } from "hast";
+import { type MetricsCollector } from "../../utils/hast-fast";
+import { type TextBlock } from "./blocks";
+/** Feature names, in vector order. The generated model refers to features by position. */
+export declare const FEATURE_NAMES: readonly ["bias", "logLen", "linkDensity", "logLinks", "hasImage", "textless", "punctDensity", "digitRatio", "sentenceEnds", "tag_p", "tag_li", "tag_cell", "tag_h1", "tag_h2_6", "tag_pre", "tag_div", "tag_dtdd", "tag_blockquote", "tag_figcaption", "in_nav", "in_header", "in_footer", "in_aside", "in_article", "in_main", "in_form", "in_table", "in_list", "role_chrome", "role_content", "cls_unlikely", "cls_positive", "cls_negative", "cls_strong", "cls_special", "cls_byline", "cls_comment", "landmark", "doom_strong", "doom_weak", "depth", "relPos", "relCharPos", "prev_logLen", "prev_linkDensity", "next_logLen", "next_linkDensity", "win_linkDensity", "win_logLen", "parent_linkDensity", "parent_logText", "gparent_linkDensity", "gparent_logText", "inTopCandidate", "candidateScore", "inTopSibling", "takumi_kept"];
+export declare const FEATURE_COUNT: 57;
+export interface BlockFeatureInput {
+    /** The tree the blocks came from (after invisible content was removed). */
+    root: Hast;
+    blocks: TextBlock[];
+    collector: MetricsCollector;
+    lang?: string;
+    /** Elements the heuristic extractor keeps (see `takumiSelection`). */
+    takumiKept?: Set<Element>;
+}
+/**
+ * Builds one feature vector per block, row-major in a single `Float32Array`.
+ *
+ * Cost: one pass over the blocks, ancestor features memoised per element, plus one furniture
+ * scan and one candidate-scoring pass over the tree — both of which the heuristic extractor
+ * performs anyway.
+ */
+export declare const blockFeatures: ({ root, blocks, collector, lang, takumiKept }: BlockFeatureInput) => Float32Array;
+/** Second-stage features: the first stage's probabilities, aggregated over context. */
+export declare const STACK_FEATURE_NAMES: readonly ["s1_logit", "s1_win1", "s1_win3", "s1_parent", "s1_gparent", "s1_ggparent", "s1_parentShare", "s1_distHigh", "s1_docShare"];
+export declare const STACKED_FEATURE_COUNT: number;
+/**
+ * Appends context aggregates of first-stage probabilities to every feature row.
+ *
+ * A block surrounded by main content, or inside a container whose text is mostly main content,
+ * is main content far more often than its own features say — the sequential structure that
+ * Web2Text and BoilerNet model with an HMM or an LSTM, captured here with a few averages.
+ */
+export declare const stackedFeatures: (blocks: TextBlock[], base: Float32Array, probabilities: Float32Array) => Float32Array;

package/dist/types/extractors/lib/block-features.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"block-features.js","sourceRoot":"","sources":["../../../../src/extractors/lib/block-features.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;;;;;;;AAIH,OAAO,EAAyB,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACtG,OAAO,EAAmC,UAAU,EAAE,MAAM,UAAU,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/D,0FAA0F;AAC1F,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B,MAAM;IACN,QAAQ;IACR,aAAa;IACb,UAAU;IACV,UAAU;IACV,UAAU;IACV,cAAc;IACd,YAAY;IACZ,cAAc;IACd,OAAO;IACP,QAAQ;IACR,UAAU;IACV,QAAQ;IACR,UAAU;IACV,SAAS;IACT,SAAS;IACT,UAAU;IACV,gBAAgB;IAChB,gBAAgB;IAChB,QAAQ;IACR,WAAW;IACX,WAAW;IACX,UAAU;IACV,YAAY;IACZ,SAAS;IACT,SAAS;IACT,UAAU;IACV,SAAS;IACT,aAAa;IACb,cAAc;IACd,cAAc;IACd,cAAc;IACd,cAAc;IACd,YAAY;IACZ,aAAa;IACb,YAAY;IACZ,aAAa;IACb,UAAU;IACV,aAAa;IACb,WAAW;IACX,OAAO;IACP,QAAQ;IACR,YAAY;IACZ,aAAa;IACb,kBAAkB;IAClB,aAAa;IACb,kBAAkB;IAClB,iBAAiB;IACjB,YAAY;IACZ,oBAAoB;IACpB,gBAAgB;IAChB,qBAAqB;IACrB,iBAAiB;IACjB,gBAAgB;IAChB,gBAAgB;IAChB,cAAc;IACd,aAAa;CACJ,CAAC;AAEX,MAAM,CAAC,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC;AAElD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC;AAC7C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;AAE5C,qFAAqF;AACrF,MAAM,SAAS,GAAG,CAAC,IAAY,EAAwD,EAAE;IACxF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,KAAK,IAAI,CAAC,CAAC;YACX,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YACxC,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC;gBACnG,SAAS,IAAI,CAAC,CAAC;YAChB,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AACrC,CAAC,CAAC;AACF,MAAM,OAAO,GAAG,+BAA+B,CAAC;AAEhD,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;AAcrD,MAAM,SAAS,GAAG,CAAC,OAAgB,EAAE,KAAuC,EAAkB,EAAE;IAC/F,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACjC,IAAI,OAAO,EAAE,CAAC;QACb,OAAO,OAAO,CAAC;IAChB,CAAC;IACD,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;IACnD,MAAM,KAAK,GAAG,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3E,OAAO,GAAG;QACT,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7F,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;QACtC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;QACtC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjE,OAAO,EAAE,OAAO,CAAC,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC;QACtD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QAClC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;QAC5B,UAAU,EAAE,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC;QACpC,WAAW,EAAE,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC;KACpC,CAAC;IACF,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5B,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,6FAA6F;AAC7F,MAAM,aAAa,GAAG,CAAC,CAAC;AAExB,MAAM,YAAY,GAAG,CAAC,CAAC;AAiCvB,6CAA6C;AAC7C,MAAM,SAAS;IAEd,YAAqB,GAAiB;QAAjB,QAAG,GAAH,GAAG,CAAc;QADtC,WAAM,GAAG,CAAC,CAAC;IAC8B,CAAC;IAC1C,IAAI,CAAC,KAAa;QACjB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC;IACjC,CAAC;IACD,IAAI,CAAC,SAAuC;QAC3C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;CACD;AAED,MAAM,WAAW,GAAU;IAC1B,KAAK,EAAE,CAAC;IACR,QAAQ,EAAE,CAAC;IACX,QAAQ,EAAE,CAAC;IACX,OAAO,EAAE,CAAC;IACV,SAAS,EAAE,CAAC;IACZ,MAAM,EAAE,CAAC;IACT,MAAM,EAAE,CAAC;IACT,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,CAAC;IACT,QAAQ,EAAE,CAAC;IACX,UAAU,EAAE,CAAC;IACb,QAAQ,EAAE,CAAC;IACX,KAAK,EAAE,CAAC;IACR,YAAY,EAAE,CAAC;IACf,YAAY,EAAE,CAAC;IACf,QAAQ,EAAE,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC;CAChE,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,OAAuB,EAAa,EAAE,CAAC;IAC3D,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,OAAO;CACf,CAAC;AAEF,MAAM,GAAG,GAAG,CAAC,MAAc,EAAE,SAAkB,EAAU,EAAE,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE1F;;;GAGG;AACH,MAAM,aAAa;IAIlB,YACU,SAAuB,EACvB,cAAwC,EACxC,OAA8B,EAC9B,GAAwB;;QAHxB,cAAS,GAAT,SAAS,CAAc;QACvB,mBAAc,GAAd,cAAc,CAA0B;QACxC,YAAO,GAAP,OAAO,CAAuB;QAC9B,QAAG,GAAH,GAAG,CAAqB;QAPzB,gCAAU,IAAI,GAAG,EAAkB,EAAC;QACpC,iCAAW,IAAI,OAAO,EAA2B,EAAC;IAOxD,CAAC;IAEJ,wEAAwE;IACxE,OAAO,CAAC,KAA6B;QACpC,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,KAAK,GAAG,WAAW,CAAC;QACxB,OAAO,OAAO,EAAE,CAAC;YAChB,MAAM,IAAI,GAAG,uBAAA,IAAI,6BAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,IAAI,EAAE,CAAC;gBACV,KAAK,GAAG,IAAI,CAAC;gBACb,MAAM;YACP,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;QAC1B,CAAC;QACD,KAAK,IAAI,KAAK,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YAC1D,KAAK,GAAG,uBAAA,IAAI,uDAAQ,MAAZ,IAAI,EAAS,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;YACpD,uBAAA,IAAI,6BAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,KAAK,CAAC;IACd,CAAC;CA8BD;gLA5BQ,MAAa,EAAE,OAAgB;IACtC,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC;IAC5B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,uBAAA,IAAI,8BAAS,CAAC,CAAC,CAAC;IAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;IACrB,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1D,OAAO;QACN,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,KAAK,KAAK,CAAC;QACvC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,KAAK,QAAQ,CAAC;QAChD,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,KAAK,QAAQ,CAAC;QAChD,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,KAAK,OAAO,CAAC;QAC7C,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,KAAK,SAAS,CAAC;QACnD,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,KAAK,MAAM,CAAC;QAC1C,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,KAAK,MAAM,CAAC;QAC1C,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,KAAK,OAAO,CAAC;QAC7C,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI,CAAC;QACxD,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC3D,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,KAAK,QAAQ,CAAC;QACrD,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,KAAK,MAAM,CAAC;QAC/C,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,KAAK,GAAG,CAAC;QACzC,YAAY,EAAE,GAAG,CAChB,MAAM,CAAC,YAAY,EACnB,SAAS,KAAK,SAAS,IAAI,OAAO,KAAK,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,SAAS,CACrF;QACD,YAAY,EAAE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,YAAY;QACrE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;KACvF,CAAC;AACH,CAAC;AAGF,MAAM,QAAQ,GAAG,CAAC,QAAgB,EAAU,EAAE,CAAC,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEnG,MAAM,SAAS,GAAG,CAAC,GAAc,EAAE,KAAgB,EAAE,OAAe,EAAQ,EAAE;IAC7E,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;IACjC,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACpC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACZ,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACxB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAClB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3B,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;IACvB,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;IAClD,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IACnD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;AAClC,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,GAAc,EAAE,GAAW,EAAQ,EAAE;IACtD,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;IACtB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC;IACvB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC;IACvC,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC;IACvB,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC;IACxB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,SAAS,CAAC,CAAC;IAC7C,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC;IACvC,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC;IAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,GAAc,EAAE,KAAY,EAAQ,EAAE;IACzD,KAAK,MAAM,KAAK,IAAI;QACnB,KAAK,CAAC,KAAK;QACX,KAAK,CAAC,QAAQ;QACd,KAAK,CAAC,QAAQ;QACd,KAAK,CAAC,OAAO;QACb,KAAK,CAAC,SAAS;QACf,KAAK,CAAC,MAAM;QACZ,KAAK,CAAC,MAAM;QACZ,KAAK,CAAC,OAAO;QACb,KAAK,CAAC,MAAM;KACZ,EAAE,CAAC;QACH,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACjB,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACvC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9B,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC3B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AAC1B,CAAC,CAAC;AAEF,mFAAmF;AACnF,MAAM,WAAW,GAAG,CAAC,OAAiB,EAAE,OAAiB,EAAE,KAAa,EAAE,MAAc,EAAoB,EAAE;IAC7G,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;QAC3G,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;QACtB,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,CAAC;IACX,CAAC;IACD,OAAO,CAAC,IAAI,GAAG,IAAI,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC;AAClC,CAAC,CAAC;AAEF,kFAAkF;AAClF,MAAM,kBAAkB,GAAG,CAAC,GAAc,EAAE,KAAa,EAAE,OAAiB,EAAE,OAAiB,EAAQ,EAAE;IACxG,MAAM,WAAW,GAAG,KAAK,GAAG,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAC3C,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IACxE,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACrB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACrB,CAAC,CAAC;AAEF,gFAAgF;AAChF,MAAM,eAAe,GAAG,CAAC,GAAc,EAAE,KAAgB,EAAE,SAA2B,EAAQ,EAAE;IAC/F,KAAK,MAAM,SAAS,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC;AACF,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAqB,EAAgB,EAAE;IAC/G,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,KAAK,GAAG,aAAa,CAAC,CAAC;IAEpD,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,IAAI,OAAO,EAAmB,CAAC;IACtD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC;IACnE,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAEpH,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3G,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACzE,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5B,MAAM,GAAG,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,GAAG,aAAa,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC;QAC5F,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE1C,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACtC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAEvB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAC/C,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QAC/C,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC;QACnC,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;QAE9B,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACjD,eAAe,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QAEvC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC7B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC7B,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,GAAG,CAAC;AACZ,CAAC,CAAC;AAEF,uFAAuF;AACvF,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAClC,UAAU;IACV,SAAS;IACT,SAAS;IACT,WAAW;IACX,YAAY;IACZ,aAAa;IACb,gBAAgB;IAChB,aAAa;IACb,aAAa;CACJ,CAAC;AAEX,MAAM,CAAC,MAAM,qBAAqB,GAAG,aAAa,GAAG,mBAAmB,CAAC,MAAM,CAAC;AAEhF,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAEjF,0FAA0F;AAC1F,MAAM,YAAY,GAAG,CAAC,MAAmB,EAAE,aAA2B,EAAE,EAAE;IACzE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAqC,CAAC;IAC1D,MAAM,GAAG,GAAG,CAAC,OAA4B,EAAE,CAAS,EAAE,MAAc,EAAE,EAAE;QACvE,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,OAAO;QACR,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAChC,IAAI,KAAK,EAAE,CAAC;YACX,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;YACtB,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC;QACnB,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACjD,CAAC;IACF,CAAC,CAAC;IACF,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrC,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACrC,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACrC,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC;QACnB,IAAI,IAAI,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,iBAAiB,GAAG,CAAC,OAA4B,EAAU,EAAE;QAClE,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtD,OAAO,KAAK,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,EAAE,iBAAiB,EAAE,CAAC;AAC5E,CAAC,CAAC;AAEF,oFAAoF;AACpF,MAAM,cAAc,GAAG,CAAC,aAA2B,EAAgB,EAAE;IACpE,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrD,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC;IACd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,GAAG,KAAK,CAAC;QACd,CAAC;QACD,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;YACf,QAAQ,CAAC,KAAK,CAAC,GAAG,KAAK,GAAG,IAAI,CAAC;QAChC,CAAC;IACF,CAAC;IACD,IAAI,GAAG,CAAC,CAAC,CAAC;IACV,KAAK,IAAI,KAAK,GAAG,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QACjD,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,GAAG,KAAK,CAAC;QACd,CAAC;QACD,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;YACf,QAAQ,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC,CAAC;QAC3D,CAAC;IACF,CAAC;IACD,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAmB,EAAE,IAAkB,EAAE,aAA2B,EAAgB,EAAE;IACrH,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,MAAM,MAAM,GAAG,qBAAqB,CAAC;IACrC,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;IAE7C,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,YAAY,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAClF,MAAM,MAAM,GAAG,CAAC,OAA4B,EAAE,QAAgB,EAAE,EAAE;QACjE,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtD,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7C,CAAC,CAAC;IACF,MAAM,QAAQ,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC;IAE/C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC;QAC3B,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,aAAa,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,EAAE,GAAG,CAAC,CAAC;QAChF,MAAM,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,CAAC,MAAc,EAAE,EAAE;YACjC,IAAI,GAAG,GAAG,CAAC,CAAC;YACZ,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,KAAK,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;gBAClG,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;oBACpB,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC;oBAC3B,CAAC,IAAI,CAAC,CAAC;gBACR,CAAC;YACF,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;QAC9B,CAAC,CAAC;QACF,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5B,IAAI,CAAC,GAAG,GAAG,GAAG,aAAa,CAAC;QAC5B,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACpB,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACrB,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACrB,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3C,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3C,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3C,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QACnD,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QACvC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC;IACrB,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC,CAAC"}

package/dist/types/extractors/lib/block-model.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Runtime evaluation of the block classifier.
+ *
+ * Two model shapes are supported, both plain generated arrays: a linear model
+ * (logistic regression) and an ensemble of complete binary trees of fixed depth, stored flat —
+ * tree `t`'s internal node `i` is at `t * internalCount + i`, its children at `2i+1` and `2i+2`.
+ */
+export interface LinearBlockModel {
+    kind: "linear";
+    /** Weights over raw features in FEATURE_NAMES order; index 0 is the bias. */
+    weights: readonly number[];
+}
+export interface TreeBlockModel {
+    kind: "trees";
+    depth: number;
+    base: number;
+    /** Feature index per internal node; -1 always goes left. */
+    features: readonly number[];
+    /** Go left when `feature <= threshold`. */
+    thresholds: readonly number[];
+    /** Leaf values, `2^depth` per tree. */
+    leaves: readonly number[];
+}
+export type BlockModel = LinearBlockModel | TreeBlockModel;
+/** True when a model matches a feature layout of `stride` features. */
+export declare const isUsableModel: (model: BlockModel, stride?: number) => boolean;
+/** Probability of each block being main content. */
+export declare const scoreBlocks: (features: Float32Array, count: number, model: BlockModel, stride?: number) => Float32Array;

package/dist/types/extractors/lib/block-model.generated.d.ts

@@ -0,0 +1,5 @@
+import type { BlockModel } from "./block-model";
+/** Stage 1: per-block features (FEATURE_NAMES). */
+export declare const BLOCK_MODEL: BlockModel;
+/** Stage 2: stage-1 features plus context aggregates (STACK_FEATURE_NAMES), or `undefined`. */
+export declare const BLOCK_STACK_MODEL: BlockModel | undefined;