webforai 1.6.3 → 2.0.1

package/dist/cjs/constants.js

@@ -0,0 +1,34 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var constants_exports = {};
+__export(constants_exports, {
+  DEFAULT_EXTRACTORS: () => DEFAULT_EXTRACTORS,
+  PRESET_EXTRACTORS: () => PRESET_EXTRACTORS
+});
+module.exports = __toCommonJS(constants_exports);
+var import_takumi = require("./extractors/presets/takumi");
+const PRESET_EXTRACTORS = {
+  takumi: import_takumi.takumiExtractor
+};
+const DEFAULT_EXTRACTORS = ["takumi"];
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DEFAULT_EXTRACTORS,
+  PRESET_EXTRACTORS
+});

package/dist/cjs/extractors/index.js

@@ -0,0 +1,31 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var extractors_exports = {};
+__export(extractors_exports, {
+  pipeExtractors: () => import_pipeExtractors.pipeExtractors,
+  takumiExtractor: () => import_takumi.takumiExtractor
+});
+module.exports = __toCommonJS(extractors_exports);
+var import_pipeExtractors = require("./pipeExtractors");
+var import_takumi = require("./presets/takumi");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  pipeExtractors,
+  takumiExtractor
+});

package/dist/cjs/{extract-hast/index.js → extractors/pipeExtractors.js}

@@ -16,29 +16,21 @@ var __copyProps = (to, from, except, desc) => {
   return to;
 };
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var extract_hast_exports = {};
-__export(extract_hast_exports, {
-  DEFAULT_EXTRACT_HAST: () => DEFAULT_EXTRACT_HAST,
-  PRESET_EXTRACT_HAST: () => PRESET_EXTRACT_HAST,
-  extractHast: () => extractHast
+var pipeExtractors_exports = {};
+__export(pipeExtractors_exports, {
+  pipeExtractors: () => pipeExtractors
 });
-module.exports = __toCommonJS(extract_hast_exports);
-var import_readability = require("./readability");
-var import_strongly = require("./strongly");
-const PRESET_EXTRACT_HAST = {
-  strongly: import_strongly.stronglyExtractHast,
-  readability: import_readability.readabilityExtractHast
-};
-const DEFAULT_EXTRACT_HAST = ["readability"];
-const extractHast = (params, extractors = DEFAULT_EXTRACT_HAST) => {
+module.exports = __toCommonJS(pipeExtractors_exports);
+var import_constants = require("../constants");
+const pipeExtractors = (params, extractors = import_constants.DEFAULT_EXTRACTORS) => {
   const { hast, lang } = params;
   const _extractors = Array.isArray(extractors) ? extractors : [extractors];
   const extracted = _extractors.reduce((acc, extractor) => {
     if (extractor === false) {
       return acc;
     }
-    if (typeof extractor === "string" && extractor in PRESET_EXTRACT_HAST) {
-      return PRESET_EXTRACT_HAST[extractor]({ hast: acc, lang });
+    if (typeof extractor === "string" && extractor in import_constants.PRESET_EXTRACTORS) {
+      return import_constants.PRESET_EXTRACTORS[extractor]({ hast: acc, lang });
     }
     if (typeof extractor === "function") {
       return extractor({ hast: acc, lang });
@@ -49,7 +41,5 @@ const extractHast = (params, extractors = DEFAULT_EXTRACT_HAST) => {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  DEFAULT_EXTRACT_HAST,
-  PRESET_EXTRACT_HAST,
-  extractHast
+  pipeExtractors
 });

package/dist/cjs/{extract-hast/readability.js → extractors/presets/takumi.js}

@@ -16,11 +16,11 @@ var __copyProps = (to, from, except, desc) => {
   return to;
 };
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var readability_exports = {};
-__export(readability_exports, {
-  readabilityExtractHast: () => readabilityExtractHast
+var takumi_exports = {};
+__export(takumi_exports, {
+  takumiExtractor: () => takumiExtractor
 });
-module.exports = __toCommonJS(readability_exports);
+module.exports = __toCommonJS(takumi_exports);
 var import_hast_util_select = require("hast-util-select");
 var import_hast_util_to_string = require("hast-util-to-string");
 var import_unist_util_filter = require("unist-util-filter");
@@ -103,7 +103,7 @@ const removeEmptyFilter = (node, _lang) => {
   }
   return true;
 };
-const readabilityExtractHast = (params) => {
+const takumiExtractor = (params) => {
   const { hast, lang = "en" } = params;
   const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
   const metadataFilteredHast = (0, import_unist_util_filter.filter)(body, (node) => metadataFilter(node));
@@ -152,5 +152,5 @@ const readabilityExtractHast = (params) => {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  readabilityExtractHast
+  takumiExtractor
 });

package/dist/cjs/extractors/types.js

@@ -0,0 +1,16 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var types_exports = {};
+module.exports = __toCommonJS(types_exports);

package/dist/cjs/html-to-mdast.js

@@ -23,8 +23,8 @@ __export(html_to_mdast_exports, {
 module.exports = __toCommonJS(html_to_mdast_exports);
 var import_hast_util_from_html = require("hast-util-from-html");
 var import_hast_util_to_mdast = require("hast-util-to-mdast");
-var import_extract_hast = require("./extract-hast");
 var import_extract_mdast = require("./extract-mdast");
+var import_extractors = require("./extractors");
 var import_custom_a_handler = require("./mdast-handlers/custom-a-handler");
 var import_custom_code_handler = require("./mdast-handlers/custom-code-handler");
 var import_custom_div_handler = require("./mdast-handlers/custom-div-handler");
@@ -36,7 +36,7 @@ const htmlToMdast = (htmlOrHast, options) => {
   const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
   const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || (0, import_hast_utils.getLangFromStr)(htmlOrHast), (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true })] : [defaultLang || (0, import_hast_utils.getLangFromHast)(htmlOrHast), htmlOrHast];
   const url = defaultUrl || (0, import_hast_utils.getUrlFromHast)(hast);
-  const extractedHast = (0, import_extract_hast.extractHast)({ hast, lang, url }, extractors);
+  const extractedHast = (0, import_extractors.pipeExtractors)({ hast, lang, url }, extractors);
   const mdast = (0, import_hast_util_to_mdast.toMdast)(extractedHast, {
     handlers: {
       math: import_math_handler.mathHandler,

package/dist/cjs/index.js

@@ -18,25 +18,25 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var src_exports = {};
 __export(src_exports, {
-  DEFAULT_EXTRACT_HAST: () => import_extract_hast.DEFAULT_EXTRACT_HAST,
-  PRESET_EXTRACT_HAST: () => import_extract_hast.PRESET_EXTRACT_HAST,
   htmlToMarkdown: () => import_html_to_markdown.htmlToMarkdown,
   htmlToMdast: () => import_html_to_mdast.htmlToMdast,
   mdastSplitter: () => import_md_splitter.mdastSplitter,
-  mdastToMarkdown: () => import_mdast_to_markdown.mdastToMarkdown
+  mdastToMarkdown: () => import_mdast_to_markdown.mdastToMarkdown,
+  pipeExtractors: () => import_extractors.pipeExtractors,
+  takumiExtractor: () => import_extractors.takumiExtractor
 });
 module.exports = __toCommonJS(src_exports);
 var import_html_to_markdown = require("./html-to-markdown");
 var import_md_splitter = require("./md-splitter");
 var import_html_to_mdast = require("./html-to-mdast");
 var import_mdast_to_markdown = require("./mdast-to-markdown");
-var import_extract_hast = require("./extract-hast/index");
+var import_extractors = require("./extractors");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  DEFAULT_EXTRACT_HAST,
-  PRESET_EXTRACT_HAST,
   htmlToMarkdown,
   htmlToMdast,
   mdastSplitter,
-  mdastToMarkdown
+  mdastToMarkdown,
+  pipeExtractors,
+  takumiExtractor
 });

package/dist/cjs/loaders/puppeteer.js

@@ -44,7 +44,7 @@ const loadHtml = async (url, ctx) => {
   await page.goto(url);
   const html = await page.content();
   await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
-  await page.close();
+  await browser.close();
   return html;
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/constants.js

@@ -0,0 +1,10 @@
+// src/constants.ts
+import { takumiExtractor } from "./extractors/presets/takumi.js";
+var PRESET_EXTRACTORS = {
+  takumi: takumiExtractor
+};
+var DEFAULT_EXTRACTORS = ["takumi"];
+export {
+  DEFAULT_EXTRACTORS,
+  PRESET_EXTRACTORS
+};

package/dist/extractors/index.js

@@ -0,0 +1,9 @@
+// src/extractors/index.ts
+import {
+  pipeExtractors
+} from "./pipeExtractors.js";
+import { takumiExtractor } from "./presets/takumi.js";
+export {
+  pipeExtractors,
+  takumiExtractor
+};

package/dist/{extract-hast/index.js → extractors/pipeExtractors.js}

@@ -1,20 +1,14 @@
-// src/extract-hast/index.ts
-import { readabilityExtractHast } from "./readability.js";
-import { stronglyExtractHast } from "./strongly.js";
-var PRESET_EXTRACT_HAST = {
-  strongly: stronglyExtractHast,
-  readability: readabilityExtractHast
-};
-var DEFAULT_EXTRACT_HAST = ["readability"];
-var extractHast = (params, extractors = DEFAULT_EXTRACT_HAST) => {
+// src/extractors/pipeExtractors.ts
+import { DEFAULT_EXTRACTORS, PRESET_EXTRACTORS } from "../constants.js";
+var pipeExtractors = (params, extractors = DEFAULT_EXTRACTORS) => {
   const { hast, lang } = params;
   const _extractors = Array.isArray(extractors) ? extractors : [extractors];
   const extracted = _extractors.reduce((acc, extractor) => {
     if (extractor === false) {
       return acc;
     }
-    if (typeof extractor === "string" && extractor in PRESET_EXTRACT_HAST) {
-      return PRESET_EXTRACT_HAST[extractor]({ hast: acc, lang });
+    if (typeof extractor === "string" && extractor in PRESET_EXTRACTORS) {
+      return PRESET_EXTRACTORS[extractor]({ hast: acc, lang });
     }
     if (typeof extractor === "function") {
       return extractor({ hast: acc, lang });
@@ -24,7 +18,5 @@ var extractHast = (params, extractors = DEFAULT_EXTRACT_HAST) => {
   return extracted;
 };
 export {
-  DEFAULT_EXTRACT_HAST,
-  PRESET_EXTRACT_HAST,
-  extractHast
+  pipeExtractors
 };

package/dist/{extract-hast/readability.js → extractors/presets/takumi.js}

@@ -1,4 +1,4 @@
-// src/extract-hast/readability.ts
+// src/extractors/presets/takumi.ts
 import { select, selectAll } from "hast-util-select";
 import { toString as hastToString } from "hast-util-to-string";
 import { filter } from "unist-util-filter";
@@ -81,7 +81,7 @@ var removeEmptyFilter = (node, _lang) => {
   }
   return true;
 };
-var readabilityExtractHast = (params) => {
+var takumiExtractor = (params) => {
   const { hast, lang = "en" } = params;
   const body = select("body", hast) ?? hast;
   const metadataFilteredHast = filter(body, (node) => metadataFilter(node));
@@ -129,5 +129,5 @@ var readabilityExtractHast = (params) => {
   return finalTree;
 };
 export {
-  readabilityExtractHast
+  takumiExtractor
 };

package/dist/{extract-hast → extractors/presets}/utils.js

@@ -1,4 +1,4 @@
-// src/extract-hast/utils.ts
+// src/extractors/presets/utils.ts
 var matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")}`;
 var classnames = (element) => {
   if (Array.isArray(element.properties.className)) {

package/dist/html-to-mdast.js

@@ -1,8 +1,8 @@
 // src/html-to-mdast.ts
 import { fromHtml } from "hast-util-from-html";
 import { toMdast } from "hast-util-to-mdast";
-import { extractHast } from "./extract-hast/index.js";
 import { extractMdast } from "./extract-mdast.js";
+import { pipeExtractors } from "./extractors/index.js";
 import { customAHandler } from "./mdast-handlers/custom-a-handler.js";
 import { customCodeHandler } from "./mdast-handlers/custom-code-handler.js";
 import { customDivHandler } from "./mdast-handlers/custom-div-handler.js";
@@ -14,7 +14,7 @@ var htmlToMdast = (htmlOrHast, options) => {
   const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
   const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || getLangFromStr(htmlOrHast), fromHtml(htmlOrHast, { fragment: true })] : [defaultLang || getLangFromHast(htmlOrHast), htmlOrHast];
   const url = defaultUrl || getUrlFromHast(hast);
-  const extractedHast = extractHast({ hast, lang, url }, extractors);
+  const extractedHast = pipeExtractors({ hast, lang, url }, extractors);
   const mdast = toMdast(extractedHast, {
     handlers: {
       math: mathHandler,

package/dist/index.js

@@ -3,12 +3,15 @@ import { htmlToMarkdown } from "./html-to-markdown.js";
 import { mdastSplitter } from "./md-splitter.js";
 import { htmlToMdast } from "./html-to-mdast.js";
 import { mdastToMarkdown } from "./mdast-to-markdown.js";
-import { PRESET_EXTRACT_HAST, DEFAULT_EXTRACT_HAST } from "./extract-hast/index.js";
+import {
+  pipeExtractors,
+  takumiExtractor
+} from "./extractors/index.js";
 export {
-  DEFAULT_EXTRACT_HAST,
-  PRESET_EXTRACT_HAST,
   htmlToMarkdown,
   htmlToMdast,
   mdastSplitter,
-  mdastToMarkdown
+  mdastToMarkdown,
+  pipeExtractors,
+  takumiExtractor
 };

package/dist/loaders/puppeteer.js

@@ -12,7 +12,7 @@ var loadHtml = async (url, ctx) => {
   await page.goto(url);
   const html = await page.content();
   await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
-  await page.close();
+  await browser.close();
   return html;
 };
 export {

package/dist/types/constants.d.ts

@@ -0,0 +1,4 @@
+export declare const PRESET_EXTRACTORS: {
+    takumi: (params: import("./extractors").ExtractParams) => import("hast").Nodes;
+};
+export declare const DEFAULT_EXTRACTORS: "takumi"[];

package/dist/types/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAE9D,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAChC,MAAM,EAAE,eAAe;CACvB,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,QAAiB,CAAC,CAAC"}

package/dist/types/extractors/index.d.ts

@@ -0,0 +1,3 @@
+export { pipeExtractors, type ExtractorSelectors, type ExtractorSelector, type PresetExtractors, } from "./pipeExtractors";
+export { takumiExtractor } from "./presets/takumi";
+export { type ExtractParams, type Extractor } from "./types";

package/dist/types/extractors/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/extractors/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EACN,cAAc,GAId,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/types/extractors/pipeExtractors.d.ts

@@ -0,0 +1,7 @@
+import type { Nodes as Hast } from "hast";
+import { PRESET_EXTRACTORS } from "../constants";
+import type { ExtractParams, Extractor } from "./types";
+export type PresetExtractors = keyof typeof PRESET_EXTRACTORS;
+export type ExtractorSelector = Extractor | false | PresetExtractors;
+export type ExtractorSelectors = ExtractorSelector | ExtractorSelector[];
+export declare const pipeExtractors: (params: ExtractParams, extractors?: ExtractorSelectors) => Hast;

package/dist/types/extractors/pipeExtractors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeExtractors.js","sourceRoot":"","sources":["../../../src/extractors/pipeExtractors.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAOrE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAqB,EAAE,aAAiC,kBAAkB,EAAQ,EAAE;IAClH,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IAC9B,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE1E,MAAM,SAAS,GACd,WAAW,CAAC,MAAM,CAAO,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;QAC3C,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;YACzB,OAAO,GAAG,CAAC;QACZ,CAAC;QACD,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,IAAI,iBAAiB,EAAE,CAAC;YACrE,OAAO,iBAAiB,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;YACrC,OAAO,SAAS,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAC;IACpD,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAElB,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}

package/dist/types/extractors/presets/takumi.d.ts

@@ -0,0 +1,10 @@
+import type { Nodes as Hast } from "hast";
+import type { ExtractParams } from "../types";
+/**
+ * Currently the best Extractor.
+ * The word "takumi" is written as 匠 in Japanese, and it refers to a highly skilled artisan or craftsman.
+ *
+ * @param params - {@link ExtractParams}
+ * @returns The HAST tree.
+ */
+export declare const takumiExtractor: (params: ExtractParams) => Hast;

package/dist/types/extractors/presets/takumi.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"takumi.js","sourceRoot":"","sources":["../../../../src/extractors/presets/takumi.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEhE,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G;;;;;;GAMG;AACH,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,yBAAyB,EAAE,8BAA8B;IACzD,kBAAkB,EACjB,wRAAwR;IACzR,oBAAoB,EAAE,mDAAmD;CACzE,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AAEvG,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEzF,MAAM,mBAAmB,GAAG;IAC3B,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;CACP,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,IAAU,EAAE,EAAE;IACrC,OAAO,CAAC,CACP,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAC1C,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CACnH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAc,CAAC,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,6BAA6B;IAC7B,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/G,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzF,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAU,EAAE,KAAa,EAAE,EAAE;IACvD,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QAC1D,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAqB,EAAQ,EAAE;IAC9D,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAE1C,MAAM,oBAAoB,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAY,CAAC,CAAC,CAAC;IAClF,MAAM,wBAAwB,GAAG,oBAAoB,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC5F,IAAI,CAAC,CAAC,oBAAoB,IAAI,wBAAwB,CAAC,EAAE,CAAC;QACzD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,oBAAoB,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAY,CAAC,CAAC,CAAC;IACjG,MAAM,eAAe,GAAG,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GACzB,eAAe,CAAC,MAAM,GAAG,wBAAwB,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,GAAG,IAAI;QAC5F,CAAC,CAAE,CAAC,WAAmB,EAAE,eAAe,CAAW;QACnD,CAAC,CAAE,CAAC,oBAA4B,EAAE,wBAAwB,CAAW,CAAC;IAExE,IAAI,aAAa,GAAG,IAAI,IAAI,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAwC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACtH,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QACrC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,aAAa,GAAS,QAAQ,CAAC;IACnC,IAAI,aAAa,GAAG,QAAQ,CAAC;IAE7B,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;QACtF,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QAE1C,IAAI,WAAW,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC7B,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;QACzD,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YACvB,SAAS;QACV,CAAC;QAED,IAAI,WAAW,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;YACxC,aAAa,GAAG,OAAO,CAAC;YACxB,aAAa,GAAG,WAAW,CAAC;YAC5B,MAAM;QACP,CAAC;IACF,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE;QACxD,IAAI,CAAC,iBAAiB,CAAC,IAAY,EAAE,IAAI,CAAC,EAAE,CAAC;YAC5C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,qBAAqB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC,CAAS,CAAC;IAEX,MAAM,SAAS,GACd,YAAY,CAAC,iBAAiB,CAAC,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,CAAC;IACvG,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}

package/dist/types/extractors/presets/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../../src/extractors/presets/utils.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE,CAC/C,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AAEhF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,EAAE;IAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACjD,OAAO,OAAO,CAAC,UAAU,CAAC,SAAqB,CAAC;IACjD,CAAC;IACD,OAAO,EAAE,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,KAAc,EAAE,KAAa,EAAE,EAAE;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC"}

package/dist/types/extractors/types.d.ts

@@ -0,0 +1,7 @@
+import type { Nodes as Hast } from "hast";
+export type ExtractParams = {
+    hast: Hast;
+    lang?: string;
+    url?: string;
+};
+export type Extractor = (param: ExtractParams) => Hast;

package/dist/types/extractors/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/extractors/types.ts"],"names":[],"mappings":""}

package/dist/types/html-to-mdast.d.ts

@@ -1,12 +1,12 @@
 import type { Nodes as Hast } from "hast";
 import type { Nodes as Mdast } from "mdast";
-import { type Extracotrs } from "./extract-hast";
+import { type ExtractorSelectors } from "./extractors";
 export type HtmlToMdastOptions = {
     /**
      * An array of extractors to extract specific elements from the HTML.
      * You can define your own functions in addition to the Extractor provided as a preset.
      */
-    extractors?: Extracotrs;
+    extractors?: ExtractorSelectors;
     /** Whether to convert links to plain text. */
     linkAsText?: boolean;
     /** Whether to convert tables to plain text. */

package/dist/types/html-to-mdast.js.map

@@ -1 +1 @@
-{"version":3,"file":"html-to-mdast.js","sourceRoot":"","sources":["../../src/html-to-mdast.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAG7C,OAAO,EAAmB,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAoBrF;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,UAAyB,EAAE,OAA4B,EAAS,EAAE;IAC7F,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;IAEzE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GACjB,OAAO,UAAU,KAAK,QAAQ;QAC7B,CAAC,CAAC,CAAC,WAAW,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACvF,CAAC,CAAC,CAAC,WAAW,IAAI,eAAe,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,CAAC;IAE7D,MAAM,GAAG,GAAG,UAAU,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;IAE/C,MAAM,aAAa,GAAG,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;IAEnE,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,EAAE;QACpC,QAAQ,EAAE;YACT,IAAI,EAAE,WAAW;YACjB,GAAG,EAAE,gBAAgB;YACrB,GAAG,EAAE,iBAAiB;YACtB,CAAC,EAAE,cAAc,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;YAClD,GAAG,EAAE,gBAAgB,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;YACxD,KAAK,EAAE,kBAAkB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;SAC3D;KACD,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAE3C,OAAO,cAAc,CAAC;AACvB,CAAC,CAAC"}
+{"version":3,"file":"html-to-mdast.js","sourceRoot":"","sources":["../../src/html-to-mdast.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAG7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAA2B,cAAc,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAoBrF;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,UAAyB,EAAE,OAA4B,EAAS,EAAE;IAC7F,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;IAEzE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GACjB,OAAO,UAAU,KAAK,QAAQ;QAC7B,CAAC,CAAC,CAAC,WAAW,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACvF,CAAC,CAAC,CAAC,WAAW,IAAI,eAAe,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,CAAC;IAE7D,MAAM,GAAG,GAAG,UAAU,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;IAE/C,MAAM,aAAa,GAAG,cAAc,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;IAEtE,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,EAAE;QACpC,QAAQ,EAAE;YACT,IAAI,EAAE,WAAW;YACjB,GAAG,EAAE,gBAAgB;YACrB,GAAG,EAAE,iBAAiB;YACtB,CAAC,EAAE,cAAc,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;YAClD,GAAG,EAAE,gBAAgB,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;YACxD,KAAK,EAAE,kBAAkB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;SAC3D;KACD,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAE3C,OAAO,cAAc,CAAC;AACvB,CAAC,CAAC"}

package/dist/types/index.d.ts

@@ -2,4 +2,4 @@ export { htmlToMarkdown, type HtmlToMarkdownOptions } from "./html-to-markdown";
 export { mdastSplitter } from "./md-splitter";
 export { htmlToMdast, type HtmlToMdastOptions } from "./html-to-mdast";
 export { mdastToMarkdown } from "./mdast-to-markdown";
-export { PRESET_EXTRACT_HAST, DEFAULT_EXTRACT_HAST } from "./extract-hast/index";
+export { pipeExtractors, takumiExtractor, type ExtractorSelectors, type ExtractorSelector, type PresetExtractors, type ExtractParams, type Extractor, } from "./extractors";

package/dist/types/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EAAE,cAAc,EAA8B,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,WAAW,EAA2B,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EAAE,cAAc,EAA8B,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,WAAW,EAA2B,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EACN,cAAc,EACd,eAAe,GAMf,MAAM,cAAc,CAAC"}

package/dist/types/loaders/puppeteer.js.map

@@ -1 +1 @@
-{"version":3,"file":"puppeteer.js","sourceRoot":"","sources":["../../../src/loaders/puppeteer.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,WAAW,CAAC;AAGlC,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,GAA4B,EAAE,EAAE;IAC3E,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,MAAM,CACrC,GAAG,IAAI;QACN,QAAQ,EAAE,IAAI;QACd,IAAI,EAAE,CAAC,cAAc,EAAE,0BAA0B,CAAC;KAClD,CACD,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAErB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE9D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IAEnB,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}
+{"version":3,"file":"puppeteer.js","sourceRoot":"","sources":["../../../src/loaders/puppeteer.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,WAAW,CAAC;AAGlC,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,GAA4B,EAAE,EAAE;IAC3E,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,MAAM,CACrC,GAAG,IAAI;QACN,QAAQ,EAAE,IAAI;QACd,IAAI,EAAE,CAAC,cAAc,EAAE,0BAA0B,CAAC;KAClD,CACD,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAErB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAElC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE9D,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAEtB,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "webforai",
-  "version": "1.6.3",
+  "version": "2.0.1",
   "description": "A library that provides a web interface for AI",
   "author": "inaridiy",
-  "license": "MIT",
+  "license": "Apache-2.0",
   "keywords": [
     "web",
     "ai",
@@ -17,7 +17,7 @@
     "type": "git",
     "url": "https://github.com/inaridiy/webforai.git"
   },
-  "homepage": "https://github.com/inaridiy/webforai.git",
+  "homepage": "https://webforai.dev/",
   "files": [
     "dist",
     "!dist/types/**/*.js"
@@ -135,7 +135,7 @@
     "copy:package.cjs.json": "pnpm ncp ./package.cjs.json ./dist/cjs/package.json && pnpm ncp ./package.cjs.json ./dist/types/package.json ",
     "clean": "rimraf dist",
     "build": "pnpm clean && tsx build.ts && pnpm copy:package.cjs.json",
-    "test": "echo \"Error: no test specified\"",
+    "typecheck": "tsc --noEmit",
     "prerelease": "pnpm build",
     "release": "np"
   }

package/dist/cjs/extract-hast/strongly.js

@@ -1,57 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var strongly_exports = {};
-__export(strongly_exports, {
-  stronglyExtractHast: () => stronglyExtractHast
-});
-module.exports = __toCommonJS(strongly_exports);
-var import_hast_util_select = require("hast-util-select");
-var import_unist_util_filter = require("unist-util-filter");
-const REMOVE_REGEX = /fallback|icon|header|nav|speechify-ignore|assist|-ad-|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|assist|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i;
-const stronglyExtractHast = (params) => {
-  const { hast } = params;
-  const selectors = ["article", "#article", "#content", "main", "div.container"];
-  let selectedElement = hast;
-  for (const selector of selectors) {
-    const element = (0, import_hast_util_select.select)(selector, hast);
-    if (element) {
-      selectedElement = element;
-      break;
-    }
-  }
-  const removedElements = (0, import_unist_util_filter.filter)(selectedElement, (node) => {
-    if (node.type === "comment") {
-      return false;
-    }
-    if (node.type !== "element") {
-      return true;
-    }
-    const element = node;
-    const elementName = `${element.tagName} ${element.properties.id} ${element.properties.className}`;
-    if (REMOVE_REGEX.test(elementName)) {
-      return false;
-    }
-    return true;
-  });
-  return removedElements;
-};
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  stronglyExtractHast
-});

package/dist/extract-hast/strongly.js

@@ -1,34 +0,0 @@
-// src/extract-hast/strongly.ts
-import { select } from "hast-util-select";
-import { filter } from "unist-util-filter";
-var REMOVE_REGEX = /fallback|icon|header|nav|speechify-ignore|assist|-ad-|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|assist|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i;
-var stronglyExtractHast = (params) => {
-  const { hast } = params;
-  const selectors = ["article", "#article", "#content", "main", "div.container"];
-  let selectedElement = hast;
-  for (const selector of selectors) {
-    const element = select(selector, hast);
-    if (element) {
-      selectedElement = element;
-      break;
-    }
-  }
-  const removedElements = filter(selectedElement, (node) => {
-    if (node.type === "comment") {
-      return false;
-    }
-    if (node.type !== "element") {
-      return true;
-    }
-    const element = node;
-    const elementName = `${element.tagName} ${element.properties.id} ${element.properties.className}`;
-    if (REMOVE_REGEX.test(elementName)) {
-      return false;
-    }
-    return true;
-  });
-  return removedElements;
-};
-export {
-  stronglyExtractHast
-};

package/dist/types/extract-hast/index.d.ts

@@ -1,15 +0,0 @@
-import type { Nodes as Hast } from "hast";
-export type ExtractPresets = "strongly" | "readability";
-export type ExtractParams = {
-    hast: Hast;
-    lang?: string;
-    url?: string;
-};
-export type Extractor = ((param: ExtractParams) => Hast) | false | ExtractPresets;
-export type Extracotrs = Extractor | Extractor[];
-export declare const PRESET_EXTRACT_HAST: {
-    strongly: (params: ExtractParams) => Hast;
-    readability: (params: ExtractParams) => Hast;
-};
-export declare const DEFAULT_EXTRACT_HAST: Extracotrs;
-export declare const extractHast: (params: ExtractParams, extractors?: Extracotrs) => Hast;