webforai 2.1.1 → 4.0.0

package/dist/types/constants.d.ts

@@ -1 +1,7 @@
-export declare const DEFAULT_EXTRACTORS: ((params: import("./extractors").ExtractParams) => import("hast").Nodes)[];
+/**
+ * Extractors applied when the caller does not specify any.
+ *
+ * {@link autoExtractor} dispatches to a site adapter when one recognises the page and falls back
+ * to generic scoring otherwise, so a single entry covers both paths.
+ */
+export declare const DEFAULT_EXTRACTORS: import("./extractors").Extractor[];

package/dist/types/constants.js.map

@@ -1 +1 @@
-{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAE9D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,eAAe,CAAC,CAAC"}
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,aAAa,CAAC,CAAC"}

package/dist/types/detect-client-shell.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Heuristic detection of HTML documents that carry no readable content because the page is
+ * built client-side (SPA shells), gated behind an anti-bot interstitial, or simply empty.
+ *
+ * Converting such a document yields a markdown body that is at best the `<title>` — the
+ * caller almost always wants to re-fetch with a JavaScript-rendering engine instead. The
+ * verdict is advisory: thresholds are tuned so that short-but-real static pages (an
+ * example.com-sized document) are NOT flagged.
+ *
+ * Expects a full document; fragments without `<body>` are judged as a whole and tiny
+ * fragments will read as `empty-body`.
+ */
+export type ClientShellReason = "empty-body" | "spa-shell" | "noscript-only" | "anti-bot-challenge";
+export interface ClientShellVerdict {
+    isShell: boolean;
+    reason?: ClientShellReason;
+    /** Characters of visible body text after stripping markup — what a conversion could use. */
+    visibleTextLength: number;
+}
+/** Body text at or above this length is always treated as real content. */
+export declare const SHELL_SUSPICION_THRESHOLD = 300;
+/** Below this length the document is a shell even without a framework marker. */
+export declare const SHELL_EMPTY_THRESHOLD = 50;
+export declare const detectClientShell: (html: string) => ClientShellVerdict;

package/dist/types/detect-client-shell.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"detect-client-shell.js","sourceRoot":"","sources":["../../src/detect-client-shell.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAWH,2EAA2E;AAC3E,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAC7C,iFAAiF;AACjF,MAAM,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAExC,MAAM,YAAY,GAAG,gCAAgC,CAAC;AACtD,MAAM,YAAY,GAAG,+BAA+B,CAAC;AACrD,MAAM,gBAAgB,GAAG,yCAAyC,CAAC;AACnE,MAAM,yBAAyB,GAAG,6DAA6D,CAAC;AAChG,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAC3C,MAAM,WAAW,GAAG,UAAU,CAAC;AAE/B;;;GAGG;AACH,MAAM,mBAAmB,GACxB,6HAA6H,CAAC;AAE/H,+FAA+F;AAC/F,MAAM,iBAAiB,GACtB,8HAA8H,CAAC;AAEhI,MAAM,QAAQ,GAA2B;IACxC,QAAQ,EAAE,GAAG;IACb,OAAO,EAAE,GAAG;IACZ,MAAM,EAAE,GAAG;IACX,MAAM,EAAE,GAAG;IACX,QAAQ,EAAE,GAAG;IACb,OAAO,EAAE,GAAG;CACZ,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,MAAc,EAAU,EAAE,CAChD,MAAM;KACJ,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC;KAC7B,OAAO,CAAC,yBAAyB,EAAE,GAAG,CAAC;KACvC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;KACzB,OAAO,CAAC,gCAAgC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC;KAC9E,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;KACpB,IAAI,EAAE,CAAC;AAEV,MAAM,0BAA0B,GAAG,CAAC,IAAY,EAAW,EAAE;IAC5D,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACrD,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YACxC,OAAO,IAAI,CAAC;QACb,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAsB,EAAE;IACrE,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IAC7E,MAAM,iBAAiB,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAErD,IAAI,iBAAiB,IAAI,yBAAyB,EAAE,CAAC;QACpD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC9C,CAAC;IACD,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,CAAC;IAC3E,CAAC;IACD,IAAI,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,CAAC;IAClE,CAAC;IACD,IAAI,0BAA0B,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,iBAAiB,EAAE,CAAC;IACtE,CAAC;IACD,IAAI,iBAAiB,GAAG,qBAAqB,EAAE,CAAC;QAC/C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,iBAAiB,EAAE,CAAC;IACnE,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC;AAC9C,CAAC,CAAC"}

package/dist/types/extract-meta-refresh.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Meta-refresh redirect detection.
+ *
+ * Some sites answer HTTP 200 with a stub page whose only job is to send the browser
+ * elsewhere — `<meta http-equiv="refresh" content="0; url=/ja/">`, typically paired with a
+ * JavaScript fallback (GitHub Pages language redirects are the canonical case). HTTP-level
+ * redirect following never triggers on these, so a fetch-based pipeline converts the stub
+ * and loses the page. This parser lets loaders treat such a page as the redirect it is.
+ *
+ * A refresh only counts as a redirect when it names a URL, the URL resolves to http(s),
+ * the target is not the page itself, and the delay is small — long delays are auto-reload
+ * timers (live blogs, dashboards), not redirects.
+ */
+export interface MetaRefreshTarget {
+    /** The redirect target, resolved against `baseUrl` when one was given. */
+    url: string;
+    delaySeconds: number;
+}
+/** Above this delay a refresh is an auto-reload timer, not a redirect. */
+export declare const MAX_META_REFRESH_DELAY_SECONDS = 10;
+/**
+ * Extracts the redirect a page's meta-refresh declares, if any.
+ *
+ * @param html - The fetched document.
+ * @param baseUrl - The document's own URL; used to resolve relative targets and to reject
+ *   self-refreshes. Without it, only absolute targets are recognised.
+ * @returns The target and delay, or `undefined` when the page declares no usable redirect.
+ */
+export declare const extractMetaRefresh: (html: string, baseUrl?: string) => MetaRefreshTarget | undefined;

package/dist/types/extract-meta-refresh.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-meta-refresh.js","sourceRoot":"","sources":["../../src/extract-meta-refresh.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAQH,0EAA0E;AAC1E,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,CAAC;AAEjD,MAAM,gBAAgB,GAAG,iBAAiB,CAAC;AAC3C,MAAM,0BAA0B,GAAG,iEAAiE,CAAC;AACrG,MAAM,oBAAoB,GAAG,oDAAoD,CAAC;AAElF,4FAA4F;AAC5F,MAAM,qBAAqB,GAAG,2EAA2E,CAAC;AAE1G,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,OAA2B,EAAmB,EAAE;IACnF,IAAI,CAAC;QACJ,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAE,OAAgB,EAAiC,EAAE;IACnG,KAAK,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,SAAS;QACV,CAAC;QACD,MAAM,YAAY,GAAG,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5E,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAG,qBAAqB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,SAAS;QACV,CAAC;QACD,6DAA6D;QAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,IAAI,CAAC,GAAG,IAAI,YAAY,GAAG,8BAA8B,EAAE,CAAC;YAC3D,SAAS;QACV,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC,EAAE,CAAC;YAC9E,SAAS;QACV,CAAC;QACD,IAAI,OAAO,KAAK,SAAS,IAAI,aAAa,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,IAAI,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC;YACtF,kFAAkF;YAClF,SAAS;QACV,CAAC;QAED,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC;IAC3C,CAAC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}

package/dist/types/extractors/index.d.ts

@@ -2,3 +2,4 @@ export { pipeExtractors, type ExtractorSelectors, type ExtractorSelector, } from
 export { takumiExtractor } from "./presets/takumi";
 export { type ExtractParams, type Extractor } from "./types";
 export { minimalFilter } from "./presets/minimal-filter";
+export { autoExtractor, createAutoExtractor, type AutoExtractorOptions } from "./presets/auto";

package/dist/types/extractors/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/extractors/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EACN,cAAc,GAGd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/extractors/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EACN,cAAc,GAGd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAA6B,MAAM,gBAAgB,CAAC"}

package/dist/types/extractors/lib/constants.d.ts

@@ -0,0 +1,81 @@
+/**
+ * Shared vocabulary for the content extractors.
+ *
+ * The regular expressions descend from @mozilla/readability (Apache-2.0, original copyright
+ * (c) 2010 Arc90 Inc — see https://github.com/mozilla/readability). They have been extended for
+ * modern component-framework class names, Japanese publishers, cookie/consent banners and the
+ * newsletter/paywall furniture that post-dates the original heuristics.
+ */
+export declare const REGEXPS: {
+    /**
+     * Containers that almost never hold the article body.
+     *
+     * Note the words that are deliberately *absent*. `header` and `footer` match the heading
+     * anchors and article furniture that documentation generators emit (`<a class="header">`,
+     * `class="post-header"`), so matching them here deletes the page's headings. The landmark
+     * versions of those elements are handled structurally instead.
+     */
+    readonly unlikelyCandidates: RegExp;
+    /** Rescues containers whose class merely happens to contain an unlikely substring. */
+    readonly okMaybeItsaCandidate: RegExp;
+    /** Very high-confidence boilerplate, not rescued by the clause above. */
+    readonly stronglyUnlikely: RegExp;
+    /** Class/id fragments that mark real article content. */
+    readonly positive: RegExp;
+    /** Class/id fragments that mark furniture. */
+    readonly negative: RegExp;
+    /** Placeholder imagery that should never reach the output. */
+    readonly hidden: RegExp;
+    readonly byline: RegExp;
+    /** Wiki/CMS chrome that generic scoring alone tends to keep. */
+    readonly specialUnlikelyCandidates: RegExp;
+};
+/** Never carry any semantic content into Markdown. */
+export declare const NON_CONTENT_TAGS: Set<string>;
+/**
+ * Structural chrome removed before scoring.
+ *
+ * `<header>` and `<footer>` are deliberately absent. They are landmarks only when they sit at
+ * the top level of the document; nested inside content they are section furniture, and every
+ * major documentation generator wraps section headings in one. Removing them wholesale deletes
+ * the article's headings. The top-level case is caught by the `banner`/`contentinfo` roles in
+ * {@link UNLIKELY_ROLES} and by {@link isTopLevelLandmark}.
+ */
+export declare const CHROME_TAGS: Set<string>;
+/** Landmark tags that are chrome at the top level of a document but content when nested. */
+export declare const CONTEXTUAL_LANDMARK_TAGS: Set<string>;
+/**
+ * Class names that, on their own, mean an element is not displayed.
+ *
+ * Matched as whole class tokens, never as substrings. Tailwind alone ships `overflow-hidden`,
+ * `group-data-[collapsible=icon]:hidden` and a dozen similar utilities; a substring match on
+ * "hidden" deletes most of a modern component-framework page.
+ */
+export declare const HIDDEN_CLASS_NAMES: Set<string>;
+/**
+ * Responsive utilities that re-display an element at a wider breakpoint.
+ *
+ * `class="hidden md:block"` is hidden on a phone and visible on a desktop. Since extraction
+ * models a desktop reader, the presence of any such override cancels the hidden classification.
+ */
+export declare const RESPONSIVE_DISPLAY_OVERRIDE: RegExp;
+/** ARIA roles whose subtrees are chrome rather than content. */
+export declare const UNLIKELY_ROLES: Set<string>;
+/** Roles that positively mark the content region. */
+export declare const CONTENT_ROLES: Set<string>;
+/** Elements scored directly during candidate discovery. */
+export declare const SCOREABLE_TAGS: Set<string>;
+/** Block containers that can be promoted to a candidate. */
+export declare const CANDIDATE_TAGS: Set<string>;
+/** Tags that count as real content even when they hold little text. */
+export declare const EMBEDDED_CONTENT_TAGS: Set<string>;
+/** Tags whose emptiness means they should be dropped. */
+export declare const DROP_WHEN_EMPTY_TAGS: Set<string>;
+/**
+ * Minimum characters a candidate must hold before it can beat the whole-document fallback.
+ *
+ * Japanese and Chinese pack far more meaning per character than European languages, so a flat
+ * threshold either rejects real CJK articles or lets English navigation blocks through.
+ */
+export declare const MIN_CONTENT_LENGTH: Record<string, number>;
+export declare const minContentLength: (lang: string | undefined) => number;

package/dist/types/extractors/lib/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../../src/extractors/lib/constants.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG;IACtB;;;;;;;OAOG;IACH,kBAAkB,EACjB,seAAse;IACve,sFAAsF;IACtF,oBAAoB,EAAE,uFAAuF;IAC7G,yEAAyE;IACzE,gBAAgB,EACf,0RAA0R;IAC3R,yDAAyD;IACzD,QAAQ,EACP,uIAAuI;IACxI,8CAA8C;IAC9C,QAAQ,EACP,mTAAmT;IACpT,8DAA8D;IAC9D,MAAM,EAAE,iBAAiB;IACzB,MAAM,EAAE,0EAA0E;IAClF,gEAAgE;IAChE,yBAAyB,EAAE,0FAA0F;CAC5G,CAAC;AAEX,sDAAsD;AACtD,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IACvC,QAAQ;IACR,OAAO;IACP,MAAM;IACN,MAAM;IACN,UAAU;IACV,UAAU;IACV,MAAM;IACN,OAAO;IACP,QAAQ;IACR,OAAO;IACP,OAAO;CACP,CAAC,CAAC;AAEH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AAEvE,4FAA4F;AAC5F,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEtE;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC;IACzC,QAAQ;IACR,WAAW;IACX,WAAW;IACX,QAAQ;IACR,cAAc;IACd,MAAM;IACN,cAAc;CACd,CAAC,CAAC;AAEH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,2BAA2B,GACvC,sGAAsG,CAAC;AAExG,gEAAgE;AAChE,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IACrC,MAAM;IACN,SAAS;IACT,UAAU;IACV,eAAe;IACf,YAAY;IACZ,OAAO;IACP,aAAa;IACb,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,QAAQ;IACR,WAAW;IACX,SAAS;IACT,SAAS;IACT,SAAS;IACT,2FAA2F;IAC3F,wFAAwF;IACxF,+BAA+B;CAC/B,CAAC,CAAC;AAEH,qDAAqD;AACrD,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;AAEtE,2DAA2D;AAC3D,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;AAEhH,4DAA4D;AAC5D,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IACrC,KAAK;IACL,SAAS;IACT,SAAS;IACT,MAAM;IACN,IAAI;IACJ,KAAK;IACL,YAAY;IACZ,MAAM;IACN,IAAI;IACJ,IAAI;IACJ,IAAI;CACJ,CAAC,CAAC;AAEH,uEAAuE;AACvE,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC;IAC5C,KAAK;IACL,SAAS;IACT,OAAO;IACP,OAAO;IACP,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,KAAK;IACL,4FAA4F;IAC5F,MAAM;IACN,eAAe;IACf,QAAQ;CACR,CAAC,CAAC;AAEH,yDAAyD;AACzD,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IAC3C,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,YAAY;IACZ,YAAY;IACZ,IAAI;IACJ,IAAI;IACJ,GAAG;IACH,MAAM;IACN,KAAK;IACL,SAAS;IACT,SAAS;CACT,CAAC,CAAC;AAEH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAA2B;IACzD,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,OAAO,EAAE,GAAG;CACZ,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAwB,EAAU,EAAE;IACpE,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,OAAO,kBAAkB,CAAC,OAAO,CAAC;IACnC,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,OAAO,kBAAkB,CAAC,OAAO,CAAC,IAAI,kBAAkB,CAAC,OAAO,CAAC;AAClE,CAAC,CAAC"}

package/dist/types/extractors/lib/sanitize.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Removal passes used before and after candidate selection.
+ *
+ * Everything here mutates the tree in place via `pruneInPlace`, so a single traversal removes a
+ * whole class of nodes. Callers are responsible for cloning first when the input must survive.
+ */
+import type { Element, Nodes as Hast } from "hast";
+import { type MetricsCollector } from "../../utils/hast-fast";
+/**
+ * Rescues content that only exists inside `<noscript>`.
+ *
+ * The standard lazy-loading fallback is a placeholder `<img>` followed by a `<noscript>` holding
+ * the real one. Deleting `<noscript>` as non-content — which is otherwise correct — therefore
+ * discards the real image and keeps the placeholder. Readability carries a dedicated pass for
+ * exactly this case, and so does this one.
+ *
+ * The whole noscript body is hoisted, not just its images: fallbacks routinely include a caption
+ * or a paragraph alongside the image, and keeping only the image silently loses that text.
+ */
+export declare const hoistNoscriptImages: (tree: Hast) => void;
+/** Removes comments, metadata elements and anything the browser did not display. */
+export declare const stripNonContent: (tree: Hast) => Hast;
+/**
+ * Finds `<header>`/`<footer>` elements that act as page landmarks rather than section furniture.
+ *
+ * A landmark sits near the root and outside any `<article>`/`<main>`. The same tags nested
+ * inside content hold the section's heading and byline, so they must survive — deleting them is
+ * how a documentation page loses every heading it has.
+ */
+export declare const findLandmarkChrome: (tree: Hast) => Set<Element>;
+/**
+ * Collects the elements {@link stripUnlikely} would remove, without touching the tree.
+ *
+ * Callers use this to price the pass before paying for it: measuring the text about to be lost
+ * is far cheaper than cloning the document, pruning the copy and re-measuring the result.
+ */
+export declare const findUnlikelyElements: (tree: Hast) => Element[];
+/**
+ * Removes page furniture ahead of scoring.
+ *
+ * This pass is aggressive by design, so callers compare the surviving text against the input and
+ * fall back to the unpruned tree when it cut too deep.
+ */
+export declare const stripUnlikely: (tree: Hast) => Hast;
+/**
+ * Drops elements that would render as nothing.
+ *
+ * Empty wrappers are extremely common in component markup and each one becomes a stray blank
+ * line in the Markdown output. Preformatted regions are exempt, because there an "empty" element
+ * is usually meaningful whitespace.
+ */
+export declare const dropEmptyElements: (tree: Hast, collector: MetricsCollector) => Hast;
+/**
+ * Finds the widgets {@link cleanContent} would remove, without touching the tree.
+ *
+ * Separated from the removal itself so callers can price the pass first. Cloning the selected
+ * content just to keep a fallback copy costs more than the entire rest of extraction.
+ */
+export declare const findCleanupTargets: (tree: Hast, collector: MetricsCollector) => Element[];
+/**
+ * Final clean-up inside the selected article container.
+ *
+ * Removes share bars, related-post rails and comment forms, then drops the empty wrappers that
+ * component markup leaves behind.
+ */
+export declare const cleanContent: (tree: Hast, collector: MetricsCollector) => Hast;

package/dist/types/extractors/lib/sanitize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sanitize.js","sourceRoot":"","sources":["../../../../src/extractors/lib/sanitize.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAEN,gBAAgB,IAAI,oBAAoB,EACxC,SAAS,EACT,SAAS,EACT,iBAAiB,EACjB,WAAW,EACX,eAAe,EACf,YAAY,EACZ,cAAc,EACd,IAAI,GACJ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACN,WAAW,EACX,wBAAwB,EACxB,oBAAoB,EACpB,qBAAqB,EACrB,kBAAkB,EAClB,gBAAgB,EAChB,OAAO,EACP,2BAA2B,EAC3B,cAAc,GACd,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,iEAAiE;AACjE,MAAM,eAAe,GAAG,+DAA+D,CAAC;AAExF;;;;;GAKG;AACH,MAAM,cAAc,GAAG,CAAC,OAAgB,EAAW,EAAE;IACpD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC3D,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,YAAY,GAAG,CAAC,IAAU,EAAW,EAAE;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACxD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC9D,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,QAAQ,GAAG,CAAC,OAAgB,EAAW,EAAE;IAC9C,wFAAwF;IACxF,4FAA4F;IAC5F,6FAA6F;IAC7F,wDAAwD;IACxD,IAAI,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAC;IACd,CAAC;IAED,IAAI,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,KAAK,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IAED,6FAA6F;IAC7F,oEAAoE;IACpE,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IACxD,IAAI,KAAK,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,oFAAoF;AACpF,MAAM,qBAAqB,GAAG,6DAA6D,CAAC;AAE5F;;;;;;GAMG;AACH,MAAM,kBAAkB,GAAG,CAAC,OAAgB,EAAW,EAAE;IACxD,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC3C,OAAO,GAAG,KAAK,SAAS,IAAI,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wEAAwE;IACxE,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YACjD,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,IAAU,EAAQ,EAAE;IACvD,MAAM,KAAK,GAAG,CAAC,MAAc,EAAQ,EAAE;QACtC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,SAAS;YACV,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;gBAClC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACb,SAAS;YACV,CAAC;YAED,uFAAuF;YACvF,yDAAyD;YACzD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CACnC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CACnF,CAAC;YACF,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,SAAS;YACV,CAAC;YAED,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC;YAC/B,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC;YAE7C,2EAA2E;YAC3E,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC5C,IAAI,KAAK,GAAG,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACtE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;gBACrC,KAAK,IAAI,CAAC,CAAC;YACZ,CAAC;YAED,KAAK,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7B,CAAC;IACF,CAAC,CAAC;IAEF,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACxB,KAAK,CAAC,IAAc,CAAC,CAAC;IACvB,CAAC;AACF,CAAC,CAAC;AAEF,qDAAqD;AACrD,MAAM,YAAY,GAAG,CAAC,OAAgB,EAAW,EAAE;IAClD,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC;IACb,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC;QACb,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,oFAAoF;AACpF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAU,EAAQ,EAAE;IACnD,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAE1B,OAAO,YAAY,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QAClC,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAU,EAAgB,EAAE;IAC9D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAW,CAAC;IAElC,MAAM,KAAK,GAAG,CAAC,IAAU,EAAE,KAAa,EAAE,aAAsB,EAAQ,EAAE;QACzE,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,EAAE,CAAC;YAC3B,OAAO;QACR,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,SAAS;YACV,CAAC;YAED,MAAM,gBAAgB,GACrB,aAAa;gBACb,KAAK,CAAC,OAAO,KAAK,SAAS;gBAC3B,KAAK,CAAC,OAAO,KAAK,MAAM;gBACxB,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC;YAE1C,IAAI,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACjF,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAClB,SAAS;YACV,CAAC;YAED,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAC3C,CAAC;IACF,CAAC,CAAC;IAEF,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IACtB,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,mBAAmB,GAAG,CAAC,OAAgB,EAAW,EAAE;IACzD,2EAA2E;IAC3E,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAC/F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC7C,IAAI,IAAI,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACjF,IAAI,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,OAAO,IAAI,CAAC;QACb,CAAC;IACF,CAAC;IAED,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzF,OAAO,IAAI,CAAC;IACb,CAAC;IAED,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,IAAU,EAAa,EAAE;IAC7D,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAc,EAAE,CAAC;IAE7B,MAAM,KAAK,GAAG,CAAC,IAAU,EAAQ,EAAE;QAClC,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,EAAE,CAAC;YAC3B,OAAO;QACR,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,SAAS;YACV,CAAC;YACD,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,SAAS,CAAC,2BAA2B;YACtC,CAAC;YACD,KAAK,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACF,CAAC,CAAC;IAEF,KAAK,CAAC,IAAI,CAAC,CAAC;IACZ,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAU,EAAQ,EAAE;IACjD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC;IACnD,OAAO,YAAY,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC,CAAC;AAEF,yEAAyE;AACzE,MAAM,iBAAiB,GAAG,CAAC,OAAgB,EAAW,EAAE;IACvD,IAAI,qBAAqB,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,OAAO,CAAC,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC;AAC5F,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,2BAA2B,GAAG,CAAC,IAAU,EAAgB,EAAE;IAChE,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAW,CAAC;IAE7C,MAAM,WAAW,GAAG,CAAC,IAAU,EAAQ,EAAE;QACxC,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,EAAE,CAAC;YAC3B,OAAO;QACR,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtB,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7B,WAAW,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACF,CAAC;IACF,CAAC,CAAC;IAEF,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,CAAC,EAAE,CAAC;YAC5E,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC5B,WAAW,CAAC,IAAI,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC,CAAC,CAAC;IAEH,OAAO,iBAAiB,CAAC;AAC1B,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,IAAU,EAAE,SAA2B,EAAQ,EAAE;IAClF,MAAM,YAAY,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAC;IAEvD,MAAM,KAAK,GAAG,CAAC,MAAc,EAAQ,EAAE;QACtC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,SAAS;YACV,CAAC;YACD,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzE,KAAK,CAAC,KAAK,CAAC,CAAC;gBACb,SAAS;YACV,CAAC;YACD,IAAI,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvE,KAAK,CAAC,KAAK,CAAC,CAAC;gBACb,SAAS;YACV,CAAC;YAED,gFAAgF;YAChF,+EAA+E;YAC/E,gFAAgF;YAChF,IAAI,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;gBACtD,SAAS;YACV,CAAC;YAED,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACjC,KAAK,IAAI,CAAC,CAAC;QACZ,CAAC;IACF,CAAC,CAAC;IAEF,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACxB,KAAK,CAAC,IAAc,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,+EAA+E;AAC/E,MAAM,eAAe,GAAG,CAAC,OAAgB,EAAW,EAAE;IACrD,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACrD,OAAO,IAAI,CAAC;QACb,CAAC;QACD,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;YAChD,OAAO,IAAI,CAAC;QACb,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,OAAgB,EAAW,EAAE;IAC3D,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACvB,SAAS;QACV,CAAC;QACD,IAAI,iBAAiB,CAAC,KAAK,CAAC,IAAI,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9D,OAAO,IAAI,CAAC;QACb,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,iBAAiB,GACtB,qIAAqI,CAAC;AAEvI,gFAAgF;AAChF,MAAM,2BAA2B,GAAG,EAAE,CAAC;AAEvC,MAAM,YAAY,GAAG,CAAC,OAAgB,EAAW,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AAErF,8EAA8E;AAC9E,MAAM,eAAe,GAAG,CAAC,QAA4B,EAAE,KAAa,EAAuB,EAAE;IAC5F,KAAK,IAAI,MAAM,GAAG,KAAK,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;QACpD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC;QAChB,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChE,OAAO,SAAS,CAAC;QAClB,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,gBAAgB,GAAG,CACxB,KAAc,EACd,QAA4B,EAC5B,KAAa,EACb,SAA2B,EAC3B,MAAiB,EACV,EAAE;IACT,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,OAAO,KAAK,SAAS,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;IAEjE,IAAI,SAAS,IAAI,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,OAAO;IACR,CAAC;IACD,IAAI,iBAAiB,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,CAAC;QACnD,OAAO;IACR,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAEnB,IAAI,SAAS,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,2BAA2B,EAAE,CAAC;QACpF,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;AACF,CAAC,CAAC;AAEF,+FAA+F;AAC/F,MAAM,iBAAiB,GAAG,CAAC,QAA4B,EAAE,KAAa,EAAE,SAA2B,EAAW,EAAE;IAC/G,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAChD,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACrD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AACpD,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAU,EAAE,SAA2B,EAAa,EAAE;IACxF,MAAM,MAAM,GAAc,EAAE,CAAC;IAE7B,MAAM,KAAK,GAAG,CAAC,IAAU,EAAQ,EAAE;QAClC,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,EAAE,CAAC;YAC3B,OAAO;QACR,CAAC;QACD,MAAM,QAAQ,GAAI,IAAe,CAAC,QAAQ,CAAC;QAE3C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,SAAS;YACV,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;gBAC7D,KAAK,CAAC,KAAK,CAAC,CAAC;gBACb,SAAS;YACV,CAAC;YAED,IAAI,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,CAAC;gBACvC,gBAAgB,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;gBAC5D,SAAS;YACV,CAAC;YAED,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,SAAS,CAAC,2BAA2B;YACtC,CAAC;YACD,KAAK,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACF,CAAC,CAAC;IAEF,KAAK,CAAC,IAAI,CAAC,CAAC;IACZ,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF,+FAA+F;AAC/F,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;AAEjE,mFAAmF;AACnF,MAAM,sBAAsB,GAAG,CAAC,CAAC;AAEjC;;;;;;;;GAQG;AACH,MAAM,eAAe,GAAG,CAAC,OAAgB,EAAE,SAA2B,EAAW,EAAE;IAClF,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,YAAY,CAAC,OAAO,CAAC,IAAI,sBAAsB,CAAC;AACxD,CAAC,CAAC;AAEF,kEAAkE;AAClE,MAAM,YAAY,GAAG,CAAC,OAAgB,EAAU,EAAE;IACjD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,KAAK,GAAW,CAAC,OAAO,CAAC,CAAC;IAChC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,GAAG,sBAAsB,EAAE,CAAC;QAC3D,4EAA4E;QAC5E,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;QAC1B,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,GAAG,EAAE,CAAC;YAC7C,KAAK,IAAI,CAAC,CAAC;QACZ,CAAC;QACD,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnC,KAAK,CAAC,IAAI,CAAC,KAAa,CAAC,CAAC;YAC3B,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;;GAKG;AACH;;;GAGG;AACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAEzE,MAAM,kBAAkB,GAAG,CAAC,OAAgB,EAAW,EAAE;IACxD,IAAI,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5C,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,IAAI,OAAO,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QAClE,OAAO,IAAI,CAAC;IACb,CAAC;IAED,4FAA4F;IAC5F,gEAAgE;IAChE,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,2FAA2F;IAC3F,4FAA4F;IAC5F,OAAO,cAAc,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAU,EAAE,SAA2B,EAAQ,EAAE;IAC7E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IAE5D,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACrB,YAAY,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,iBAAiB,CAAC,IAAI,EAAE,IAAI,oBAAoB,EAAE,CAAC,CAAC;AAC5D,CAAC,CAAC;AAEF,qEAAqE;AACrE,MAAM,WAAW,GAAG,CAAC,IAAU,EAAU,EAAE;IAC1C,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC,KAAK,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,CAAC;IACX,CAAC;IACD,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnC,IAAI,IAAI,WAAW,CAAC,KAAa,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}

package/dist/types/extractors/lib/score.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Candidate scoring for the main-content extractor.
+ *
+ * The core of the algorithm follows @mozilla/readability (Apache-2.0, original copyright
+ * (c) 2010 Arc90 Inc): score every paragraph-like node by how much prose it holds, propagate
+ * that score to its ancestors with decreasing weight, then pick the best-scoring container.
+ *
+ * Three signals are layered on top of the classic algorithm:
+ *
+ * - **Semantic markup.** `<article>`, `<main>`, `role="main"` and schema.org `articleBody` are
+ *   near-conclusive on modern sites, and pre-date nothing in the 2010 heuristics.
+ * - **Rendered geometry.** When a loader annotated the DOM with `data-rwidth`/`data-rheight`,
+ *   the real layout tells us what the reader actually saw. Zero-height containers are collapsed
+ *   menus regardless of how much text they hold.
+ * - **CJK-aware length.** Character counts mean different things per script, so length-derived
+ *   bonuses are normalised by the language's information density.
+ */
+import type { Element, Nodes as Hast, Parent } from "hast";
+import { type MetricsCollector } from "../../utils/hast-fast";
+export interface ScoredCandidate {
+    element: Element;
+    score: number;
+}
+export interface ScoringContext {
+    collector: MetricsCollector;
+    parents: WeakMap<Hast, Parent>;
+    lang?: string;
+    /** True when the document carries rendered-geometry annotations. */
+    hasGeometry: boolean;
+    /** Paragraph-like elements, collected during the same traversal that built `parents`. */
+    scoreable: Element[];
+}
+/**
+ * Builds parent links, which HAST does not carry, and collects the scoreable nodes.
+ *
+ * Parent discovery, geometry detection and candidate collection all need the same full
+ * traversal, so they share one — extraction is a fifth of total conversion time and each
+ * additional walk over a megabyte-scale document is measurable.
+ */
+export declare const buildScoringContext: (root: Hast, collector: MetricsCollector, lang?: string) => ScoringContext;
+/**
+ * Class/id based weight.
+ *
+ * Deliberately capped: on component-framework sites the class attribute is often a hashed
+ * string that matches a regex by accident, so this must nudge rather than decide.
+ */
+export declare const classWeight: (element: Element) => number;
+/** Strong, near-conclusive markers that an element *is* the article container. */
+export declare const semanticBonus: (element: Element) => number;
+/**
+ * Multiplier derived from the element's rendered box.
+ *
+ * Only applied when a loader supplied geometry. A container the browser laid out at zero height
+ * was never visible, and one that is very narrow is a rail rather than the article column.
+ */
+export declare const geometryMultiplier: (element: Element, context: ScoringContext) => number;
+/**
+ * Scores every candidate container in the document.
+ *
+ * Returns candidates sorted best-first. An empty result means no container looked like prose,
+ * and callers should fall back to the whole document.
+ */
+export declare const scoreCandidates: (_root: Hast, context: ScoringContext) => ScoredCandidate[];
+/**
+ * Expands the winning candidate to include sibling blocks that belong to the same article.
+ *
+ * Publishers routinely split a post across sibling `<div>`s (lead paragraph, body, pull quote),
+ * so returning only the top-scoring node truncates the article. A sibling joins when it scores
+ * comparably, or when it is a substantial low-link paragraph.
+ */
+export declare const collectArticleNodes: (top: ScoredCandidate, candidates: ScoredCandidate[], context: ScoringContext) => Element[];
+/**
+ * Promotes the winning candidate to an ancestor when the ancestor is the real article boundary.
+ *
+ * Score propagation concentrates in the densest section of a long document, so on reference
+ * pages the winner is routinely one *section* — PostgreSQL's SELECT page scored its "Parameters"
+ * section above the `refentry` that also holds the title, synopsis and description, and the
+ * output opened mid-document. Readability climbs from its top candidate for the same reason.
+ *
+ * A hop is taken only when the parent adds a substantial amount of low-link text and itself
+ * scored within a factor of the winner; the first parent that fails either test ends the climb.
+ * Wrapper parents that add nothing fail the gain test, which is what keeps the climb from
+ * drifting up into page chrome one harmless-looking level at a time.
+ */
+export declare const climbToArticleBoundary: (top: ScoredCandidate, candidates: ScoredCandidate[], context: ScoringContext) => Element;

package/dist/types/extractors/lib/score.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.js","sourceRoot":"","sources":["../../../../src/extractors/lib/score.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,EAEN,SAAS,EACT,SAAS,EACT,WAAW,EACX,eAAe,EACf,cAAc,EACd,IAAI,GACJ,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEvG,6EAA6E;AAC7E,MAAM,cAAc,GAA2B;IAC9C,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,EAAE,EAAE,CAAC;IACL,UAAU,EAAE,CAAC;IACb,OAAO,EAAE,EAAE;IACX,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC;IACV,OAAO,EAAE,CAAC,CAAC;IACX,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,CAAC,CAAC;IACN,IAAI,EAAE,CAAC,CAAC;IACR,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,CAAC,CAAC;CACN,CAAC;AAEF,qDAAqD;AACrD,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B,gEAAgE;AAChE,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAiB9B;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,IAAU,EAAE,SAA2B,EAAE,IAAa,EAAkB,EAAE;IAC7G,MAAM,OAAO,GAAG,IAAI,OAAO,EAAgB,CAAC;IAC5C,MAAM,SAAS,GAAc,EAAE,CAAC;IAChC,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;QAC3B,IAAI,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO;QACR,CAAC;QACD,IAAI,CAAC,WAAW,IAAI,eAAe,CAAC,IAAI,EAAE,cAAc,CAAC,KAAK,SAAS,EAAE,CAAC;YACzE,WAAW,GAAG,IAAI,CAAC;QACpB,CAAC;QACD,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;IACF,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC;AAC7D,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAU,EAAE;IACvD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,EAAE,GAAG,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE7C,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,EAAE,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,EAAE,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,EAAE,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,EAAE,CAAC;IACd,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAEF,kFAAkF;AAClF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,OAAgB,EAAU,EAAE;IACzD,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACnC,KAAK,IAAI,EAAE,CAAC;IACb,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAChC,KAAK,IAAI,EAAE,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC7C,IAAI,IAAI,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,KAAK,IAAI,EAAE,CAAC;IACb,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACrD,IAAI,QAAQ,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QACvC,KAAK,IAAI,EAAE,CAAC;IACb,CAAC;IAED,8EAA8E;IAC9E,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACnC,IACC,8GAA8G,CAAC,IAAI,CAClH,KAAK,CACL,EACA,CAAC;QACF,KAAK,IAAI,EAAE,CAAC;IACb,CAAC;IAED,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,OAAgB,EAAE,OAAuB,EAAU,EAAE;IACvF,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC1B,OAAO,CAAC,CAAC;IACV,CAAC;IAED,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IACxD,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAEtD,IAAI,MAAM,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACjD,OAAO,CAAC,CAAC;IACV,CAAC;IACD,4DAA4D;IAC5D,IAAI,MAAM,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,CAAC;IACV,CAAC;IACD,wEAAwE;IACxE,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IACb,CAAC;IAED,OAAO,CAAC,CAAC;AACV,CAAC,CAAC;AAEF,oDAAoD;AACpD,MAAM,cAAc,GAAG,CAAC,UAAkB,EAAE,MAAc,EAAE,cAAsB,EAAU,EAAE,CAC7F,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;AAEnE,MAAM,cAAc,GAAG,CAAC,KAAa,EAAU,EAAE;IAChD,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,CAAC,CAAC;IACV,CAAC;IACD,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,CAAC,CAAC;IACV,CAAC;IACD,OAAO,KAAK,GAAG,CAAC,CAAC;AAClB,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,eAAe,GAAG,CAAC,OAAgB,EAAW,EAAE;IACrD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAChE,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;AACjG,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,KAAW,EAAE,OAAuB,EAAqB,EAAE;IAC1F,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC1C,6EAA6E;IAC7E,MAAM,cAAc,GAAG,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE1D,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,IAAI,GAAG,kBAAkB,EAAE,CAAC;YACvC,SAAS;QACV,CAAC;QAED,MAAM,YAAY,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QAElF,IAAI,OAAO,GAAqB,IAAI,CAAC;QACrC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,kBAAkB,IAAI,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC;YACpE,MAAM,MAAM,GAAuB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACxD,IAAI,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBACpC,MAAM;YACP,CAAC;YACD,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;gBACrG,MAAM,IAAI,GACT,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;gBAC3G,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,YAAY,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,GAAG,MAAM,CAAC;QAClB,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC;QAC1C,qEAAqE;QACrE,MAAM,OAAO,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9E,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QACrC,CAAC;IACF,CAAC;IAED,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC7C,OAAO,UAAU,CAAC;AACnB,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAClC,GAAoB,EACpB,UAA6B,EAC7B,OAAuB,EACX,EAAE;IACd,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAExC,IAAI,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7F,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACvB,SAAS;QACV,CAAC;QACD,IAAI,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,SAAS;QACV,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,YAAY,IAAI,SAAS,EAAE,CAAC;YAC/B,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,SAAS;QACV,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1D,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAC7C,IAAI,OAAO,CAAC,IAAI,GAAG,EAAE,IAAI,OAAO,GAAG,IAAI,EAAE,CAAC;gBACzC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AACvD,CAAC,CAAC;AAEF,gEAAgE;AAChE,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,uFAAuF;AACvF,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAErC,qFAAqF;AACrF,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAEpC,kFAAkF;AAClF,MAAM,2BAA2B,GAAG,CAAC,GAAG,CAAC,CAAC;AAE1C,sFAAsF;AACtF,MAAM,iBAAiB,GAAG,CAAC,GAAG,CAAC,CAAC;AAEhC;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACrC,GAAoB,EACpB,UAA6B,EAC7B,OAAuB,EACb,EAAE;IACZ,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7F,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,GAAG,iBAAiB,CAAC;IAEjD,IAAI,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IAE1B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,EAAE,GAAG,EAAE,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YACpC,MAAM;QACP,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,UAAU,EAAE,CAAC;YAC3D,MAAM;QACP,CAAC;QAED,MAAM,cAAc,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;QAE5D,IAAI,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,cAAc,CAAC,IAAI,GAAG,uBAAuB,CAAC,EAAE,CAAC;YACnG,MAAM;QACP,CAAC;QAED,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC;QAC5D,IAAI,UAAU,GAAG,UAAU,GAAG,2BAA2B,EAAE,CAAC;YAC3D,MAAM;QACP,CAAC;QAED,OAAO,GAAG,MAAM,CAAC;IAClB,CAAC;IAED,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC"}

package/dist/types/extractors/lib/terminators.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Trailing-boilerplate truncation.
+ *
+ * Articles end, pages do not: after the last paragraph come the feedback widget, the
+ * last-modified line, the contribution plea and the related-article rail. Each block is
+ * individually hard to condemn — "Thanks for the feedback…" is a well-formed prose paragraph
+ * with commas and links — but the first of them reliably marks the end of the content, and
+ * everything after it belongs to the page. boilerpipe called these terminating blocks.
+ *
+ * The rule is position-guarded: a marker only terminates when it appears in the tail of the
+ * document, so an article *about* feedback widgets is never cut at its own subject matter.
+ */
+import type { Nodes as Hast, Parent } from "hast";
+/**
+ * Removes the first tail-positioned terminating block and everything after it.
+ *
+ * Offsets are computed in one pass; the cut then removes every node whose text lies entirely at
+ * or beyond the terminator, so containers that straddle the boundary keep their leading content
+ * and lose the rest.
+ *
+ * @returns Whether anything was removed.
+ */
+export declare const truncateTrailingBoilerplate: (tree: Hast) => boolean;
+/** Exported for tests. */
+export declare const TERMINATOR_TAIL_FRACTION = 0.8;
+export type { Parent };

package/dist/types/extractors/lib/terminators.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"terminators.js","sourceRoot":"","sources":["../../../../src/extractors/lib/terminators.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAE1E;;;;;GAKG;AACH,MAAM,mBAAmB,GAAa;IACrC,4CAA4C;IAC5C,2BAA2B;IAC3B,yDAAyD;IACzD,2BAA2B;IAC3B,wBAAwB;IACxB,uFAAuF;IACvF,YAAY;IACZ,6GAA6G;IAC7G,2EAA2E;IAC3E,8BAA8B;IAC9B,mCAAmC;IACnC,yBAAyB;IACzB,eAAe;CACf,CAAC;AAEF,qFAAqF;AACrF,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC,kFAAkF;AAClF,MAAM,2BAA2B,GAAG,GAAG,CAAC;AAExC,4DAA4D;AAC5D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAEvH,4EAA4E;AAC5E,MAAM,iBAAiB,GAAG,CAAC,OAAgB,EAAE,KAAa,EAAE,SAAiB,EAAW,EAAE;IACzF,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;QAC3D,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,2BAA2B,EAAE,CAAC;QACpE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAClE,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,IAAU,EAAU,EAAE;IACrC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC,KAAK,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC;IACX,CAAC;IACD,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnC,IAAI,IAAI,MAAM,CAAC,KAAa,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,IAAU,EAAW,EAAE;IAClE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAgB,CAAC;IACvC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,IAAU,EAAQ,EAAE;QACpC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACxB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1B,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;YAClC,OAAO;QACR,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnC,OAAO,CAAC,KAAa,CAAC,CAAC;YACxB,CAAC;QACF,CAAC;IACF,CAAC,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,CAAC;IAEd,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,GAAG,mBAAmB,CAAC;IAC9C,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC;IAEhB,MAAM,IAAI,GAAG,CAAC,IAAU,EAAQ,EAAE;QACjC,IAAI,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACtC,OAAO;QACR,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;gBACnB,OAAO;YACR,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,SAAS;YACV,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,iBAAiB,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,CAAC;gBAChD,MAAM,GAAG,KAAK,CAAC;gBACf,OAAO;YACR,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,CAAC;QACb,CAAC;IACF,CAAC,CAAC;IACF,IAAI,CAAC,IAAI,CAAC,CAAC;IAEX,IAAI,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;QACnB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,oFAAoF;IACpF,YAAY,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,0FAA0F;QAC1F,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,0BAA0B;AAC1B,MAAM,CAAC,MAAM,wBAAwB,GAAG,mBAAmB,CAAC"}

package/dist/types/extractors/lib/ui-chrome.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Boilerplate detection by visible text.
+ *
+ * Class and id heuristics assume readable class names. Component frameworks that hash their
+ * classes defeat them entirely, which is why blinded evaluation kept flagging leftovers like
+ * "Copy page", "Was this page helpful? Yes No" and "Edit this page" on documentation sites whose
+ * markup carries no usable names at all. Matching what the element actually *says* works
+ * regardless of how it was built — the technique defuddle relies on.
+ *
+ * The rule is deliberately narrow. An element is chrome only when its **entire** text is one of
+ * these phrases, so prose that merely mentions "share" or "subscribe" is untouched.
+ */
+/**
+ * True when an element's entire visible text is a user-interface label.
+ *
+ * @param text - The element's complete text content.
+ */
+export declare const isUiChromeText: (text: string) => boolean;

package/dist/types/extractors/lib/ui-chrome.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ui-chrome.js","sourceRoot":"","sources":["../../../../src/extractors/lib/ui-chrome.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH;;;;GAIG;AACH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IAC1B,qCAAqC;IACrC,MAAM;IACN,WAAW;IACX,WAAW;IACX,WAAW;IACX,QAAQ;IACR,gBAAgB;IAChB,gBAAgB;IAChB,cAAc;IACd,iBAAiB;IACjB,mBAAmB;IACnB,UAAU;IACV,OAAO;IACP,iBAAiB;IACjB,aAAa;IACb,eAAe;IACf,iBAAiB;IACjB,sBAAsB;IACtB,WAAW;IACX,QAAQ;IACR,mBAAmB;IACnB,uBAAuB;IACvB,kBAAkB;IAClB,0BAA0B;IAC1B,QAAQ;IACR,UAAU;IACV,eAAe;IACf,iBAAiB;IACjB,gBAAgB;IAChB,2BAA2B;IAC3B,OAAO;IACP,YAAY;IACZ,oBAAoB;IACpB,kBAAkB;IAClB,mBAAmB;IACnB,WAAW;IACX,SAAS;IACT,SAAS;IACT,QAAQ;IACR,OAAO;IACP,YAAY;IACZ,4BAA4B;IAC5B,eAAe;IACf,gBAAgB;IAChB,WAAW;IACX,mBAAmB;IACnB,UAAU;IACV,yBAAyB;IACzB,WAAW;IACX,WAAW;IACX,WAAW;IACX,UAAU;IACV,WAAW;IACX,UAAU;IACV,MAAM;IACN,eAAe;IACf,WAAW;IACX,MAAM;IACN,OAAO;IACP,QAAQ;IACR,uBAAuB;IACvB,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,UAAU;IACV,OAAO;IACP,OAAO;IACP,MAAM;IACN,UAAU;IACV,SAAS;IACT,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,MAAM;IACN,MAAM;CACN,CAAC,CAAC;AAEH;;;;;GAKG;AACH,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAE7B;;;;;GAKG;AACH,MAAM,SAAS,GAAG,CAAC,IAAY,EAAU,EAAE,CAC1C,IAAI;KACF,WAAW,EAAE;KACb,OAAO,CAAC,oCAAoC,EAAE,GAAG,CAAC;KAClD,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;KACpB,IAAI,EAAE,CAAC;AAEV;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,IAAY,EAAW,EAAE;IACvD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QAC1D,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AACnC,CAAC,CAAC"}

package/dist/types/extractors/pipeExtractors.d.ts

@@ -2,4 +2,9 @@ import type { Nodes as Hast } from "hast";
 import type { ExtractParams, Extractor } from "./types";
 export type ExtractorSelector = Extractor | false;
 export type ExtractorSelectors = ExtractorSelector | ExtractorSelector[];
+/**
+ * Runs extractors in sequence, feeding each one the previous one's output.
+ *
+ * `false` entries are skipped, so callers can toggle a stage without rebuilding the array.
+ */
 export declare const pipeExtractors: (params: ExtractParams, extractors?: ExtractorSelectors) => Hast;

package/dist/types/extractors/pipeExtractors.js.map

@@ -1 +1 @@
-{"version":3,"file":"pipeExtractors.js","sourceRoot":"","sources":["../../../src/extractors/pipeExtractors.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAMlD,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAqB,EAAE,aAAiC,kBAAkB,EAAQ,EAAE;IAClH,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IAC9B,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE1E,MAAM,SAAS,GACd,WAAW,CAAC,MAAM,CAAO,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;QAC3C,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;YACzB,OAAO,GAAG,CAAC;QACZ,CAAC;QACD,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;YACrC,OAAO,SAAS,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAC;IACpD,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAElB,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}
+{"version":3,"file":"pipeExtractors.js","sourceRoot":"","sources":["../../../src/extractors/pipeExtractors.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAMlD;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAqB,EAAE,aAAiC,kBAAkB,EAAQ,EAAE;IAClH,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAEnE,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,0FAA0F;IAC1F,wFAAwF;IACxF,IAAI,YAAY,GAAG,KAAK,IAAI,KAAK,CAAC;IAElC,KAAK,MAAM,SAAS,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;YACzB,SAAS;QACV,CAAC;QACD,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,GAAG,SAAS,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;QACvE,YAAY,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC"}

package/dist/types/extractors/presets/auto.d.ts

@@ -0,0 +1,30 @@
+/**
+ * The default extractor: a site adapter when one applies, generic scoring otherwise.
+ *
+ * Keeping the fallback inside a single extractor (rather than chaining two pipeline stages)
+ * matters, because generic extraction must run on the *original* document. An adapter's output
+ * is already just the content, and re-scoring it would let the heuristics prune text that the
+ * site-specific rules deliberately kept.
+ */
+import type { SiteAdapter } from "../../adapters/types";
+import type { Extractor } from "../types";
+export interface AutoExtractorOptions {
+    /**
+     * Adapters to consider. Pass `false` to disable site-specific handling entirely, or an array
+     * to restrict or extend the built-in set.
+     */
+    adapters?: SiteAdapter[] | false;
+}
+/**
+ * Creates the default extractor.
+ *
+ * @param options - {@link AutoExtractorOptions}
+ */
+export declare const createAutoExtractor: (options?: AutoExtractorOptions) => Extractor;
+/**
+ * Extracts page content, preferring a site-specific adapter when one recognises the page.
+ *
+ * @param params - {@link ExtractParams}
+ * @returns The HAST tree containing the page's content.
+ */
+export declare const autoExtractor: Extractor;

package/dist/types/extractors/presets/auto.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auto.js","sourceRoot":"","sources":["../../../../src/extractors/presets/auto.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAG7C,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAU3C;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,UAAgC,EAAE,EAAa,EAAE;IACpF,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAE7B,OAAO,CAAC,MAAqB,EAAQ,EAAE;QACtC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,SAAS,GAAG,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,QAAQ,IAAI,SAAS,CAAC,CAAC;YAChH,IAAI,SAAS,EAAE,CAAC;gBACf,OAAO,SAAS,CAAC;YAClB,CAAC;QACF,CAAC;QAED,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC,CAAC;AACH,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAc,mBAAmB,EAAE,CAAC"}

package/dist/types/extractors/presets/minimal-filter.d.ts

@@ -1,9 +1,17 @@
+/**
+ * A light-touch extractor.
+ *
+ * Where {@link takumiExtractor} tries to isolate the article, this only removes what is
+ * unambiguously not content: metadata elements, invisible nodes and obvious page chrome. Use it
+ * when you would rather keep some boilerplate than risk losing part of the page — full-page
+ * archiving, diffing, or feeding a model that does its own filtering.
+ */
 import type { Nodes as Hast } from "hast";
 import type { ExtractParams } from "../types";
 /**
- * Simple filter to remove unwanted elements from the HAST tree.
+ * Removes unwanted elements from a HAST tree without attempting to locate the article.
  *
  * @param params - {@link ExtractParams}
- * @returns The HAST tree.
+ * @returns The filtered HAST tree.
  */
 export declare const minimalFilter: (params: ExtractParams) => Hast;

package/dist/types/extractors/presets/minimal-filter.js.map

@@ -1 +1 @@
-{"version":3,"file":"minimal-filter.js","sourceRoot":"","sources":["../../../../src/extractors/presets/minimal-filter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEhE,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G;;;;;;GAMG;AACH,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,yBAAyB,EAAE,8BAA8B;IACzD,kBAAkB,EACjB,iRAAiR;IAClR,oBAAoB,EAAE,mDAAmD;CACzE,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,IAAU,EAAE,EAAE;IACrC,OAAO,CAAC,CACP,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAC1C,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CACnH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAc,CAAC,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,MAAqB,EAAQ,EAAE;IAC5D,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IACxB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAE1C,MAAM,oBAAoB,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAY,CAAC,CAAC,CAAC;IAClF,MAAM,wBAAwB,GAAG,oBAAoB,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC5F,IAAI,CAAC,CAAC,oBAAoB,IAAI,wBAAwB,CAAC,EAAE,CAAC;QACzD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,oBAAoB,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAY,CAAC,CAAC,CAAC;IACjG,MAAM,eAAe,GAAG,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAErE,MAAM,eAAe,GAAG,eAAe,CAAC,MAAM,GAAG,wBAAwB,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,GAAG,IAAI,CAAC;IACtH,MAAM,QAAQ,GAAG,eAAe,IAAI,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAC;IAErF,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC"}
+{"version":3,"file":"minimal-filter.js","sourceRoot":"","sources":["../../../../src/extractors/presets/minimal-filter.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,EACN,gBAAgB,EAChB,SAAS,EACT,WAAW,EACX,SAAS,EACT,iBAAiB,EACjB,WAAW,EACX,YAAY,EACZ,cAAc,GACd,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGlD,8EAA8E;AAC9E,MAAM,qBAAqB,GAAG,CAAC,GAAG,CAAC,CAAC;AAEpC,uFAAuF;AACvF,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAElC,MAAM,QAAQ,GAAG,CAAC,OAAgB,EAAW,EAAE;IAC9C,IAAI,OAAO,CAAC,OAAO,KAAK,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC9F,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC7C,IAAI,IAAI,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,IAAI,KAAK,QAAQ,IAAI,iBAAiB,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC;QACnE,OAAO,IAAI,CAAC;IACb,CAAC;IAED,iDAAiD;IACjD,IAAI,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,MAAqB,EAAQ,EAAE;IAC5D,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IAExB,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;IAC3F,eAAe,CAAC,IAAI,CAAC,CAAC;IAEtB,MAAM,SAAS,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACjC,YAAY,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEvE,MAAM,YAAY,GAAG,IAAI,gBAAgB,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACjE,MAAM,eAAe,GAAG,YAAY,GAAG,QAAQ,GAAG,qBAAqB,IAAI,YAAY,GAAG,oBAAoB,CAAC;IAE/G,OAAO,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC1C,CAAC,CAAC"}

package/dist/types/extractors/presets/takumi.d.ts

@@ -1,10 +1,26 @@
+/**
+ * The main-content extractor.
+ *
+ * "takumi" is written 匠 in Japanese and refers to a highly skilled artisan.
+ *
+ * The pipeline is:
+ *
+ *   1. scope to `<body>` and clone, so the caller's tree is never mutated
+ *   2. strip nodes that can never be content (metadata, comments, invisible elements)
+ *   3. take the semantic shortcut when the page states where its content is
+ *   4. strip likely furniture, reverting if that cut too deep
+ *   5. score candidate containers and pick the best, plus its qualifying siblings
+ *   6. clean widgets out of the winner and drop empty wrappers
+ *
+ * Every narrowing step is guarded: if a step leaves less text than its input can justify, the
+ * extractor keeps the wider tree. Losing the article is far worse than keeping some boilerplate.
+ */
 import type { Nodes as Hast } from "hast";
 import type { ExtractParams } from "../types";
 /**
- * Currently the best Extractor.
- * The word "takumi" is written as 匠 in Japanese, and it refers to a highly skilled artisan or craftsman.
+ * Extracts the main content of a document.
  *
  * @param params - {@link ExtractParams}
- * @returns The HAST tree.
+ * @returns The HAST tree containing only the article content.
  */
 export declare const takumiExtractor: (params: ExtractParams) => Hast;