webforai 4.0.0 → 4.1.1

package/dist/cjs/extract-mdast.js

@@ -32,6 +32,61 @@ const emptyDeclarationFilter = (node) => {
   }
   return true;
 };
+const trimEdge = (children, edge) => {
+  const index = edge === "start" ? 0 : children.length - 1;
+  const node = children[index];
+  if (node?.type !== "text") {
+    return children;
+  }
+  const value = edge === "start" ? node.value.trimStart() : node.value.trimEnd();
+  const copy = [...children];
+  if (value.length === 0) {
+    copy.splice(index, 1);
+  } else {
+    copy[index] = { ...node, value };
+  }
+  return copy;
+};
+const liftDisplayMath = (paragraph) => {
+  if (!paragraph.children.some((child) => child.type === "math")) {
+    return void 0;
+  }
+  const out = [];
+  let run = [];
+  const flush = () => {
+    const trimmed = trimEdge(trimEdge(run, "start"), "end");
+    if (trimmed.length > 0) {
+      out.push({ ...paragraph, children: trimmed });
+    }
+    run = [];
+  };
+  for (const child of paragraph.children) {
+    if (child.type === "math") {
+      flush();
+      out.push(child);
+    } else {
+      run.push(child);
+    }
+  }
+  flush();
+  return out;
+};
+const liftDisplayMathIn = (node) => {
+  if (!("children" in node)) {
+    return;
+  }
+  const parent = node;
+  for (let index = 0; index < parent.children.length; index++) {
+    const child = parent.children[index];
+    const lifted = child.type === "paragraph" ? liftDisplayMath(child) : void 0;
+    if (lifted) {
+      parent.children.splice(index, 1, ...lifted);
+      index += lifted.length - 1;
+      continue;
+    }
+    liftDisplayMathIn(child);
+  }
+};
 const extractMdast = (node) => {
   const extracted = (0, import_unist_util_filter.filter)(node, (node2) => {
     if (!emptyDeclarationFilter(node2)) {
@@ -39,6 +94,9 @@ const extractMdast = (node) => {
     }
     return true;
   });
+  if (extracted) {
+    liftDisplayMathIn(extracted);
+  }
   return extracted;
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/cjs/extractors/index.js

@@ -20,6 +20,8 @@ var extractors_exports = {};
 __export(extractors_exports, {
   autoExtractor: () => import_auto.autoExtractor,
   createAutoExtractor: () => import_auto.createAutoExtractor,
+  createKiwameExtractor: () => import_kiwame.createKiwameExtractor,
+  kiwameExtractor: () => import_kiwame.kiwameExtractor,
   minimalFilter: () => import_minimal_filter.minimalFilter,
   pipeExtractors: () => import_pipeExtractors.pipeExtractors,
   takumiExtractor: () => import_takumi.takumiExtractor
@@ -27,12 +29,15 @@ __export(extractors_exports, {
 module.exports = __toCommonJS(extractors_exports);
 var import_pipeExtractors = require("./pipeExtractors");
 var import_takumi = require("./presets/takumi");
+var import_kiwame = require("./presets/kiwame");
 var import_minimal_filter = require("./presets/minimal-filter");
 var import_auto = require("./presets/auto");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   autoExtractor,
   createAutoExtractor,
+  createKiwameExtractor,
+  kiwameExtractor,
   minimalFilter,
   pipeExtractors,
   takumiExtractor

package/dist/cjs/extractors/lib/block-features.js

@@ -0,0 +1,465 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var block_features_exports = {};
+__export(block_features_exports, {
+  FEATURE_COUNT: () => FEATURE_COUNT,
+  FEATURE_NAMES: () => FEATURE_NAMES,
+  STACKED_FEATURE_COUNT: () => STACKED_FEATURE_COUNT,
+  STACK_FEATURE_NAMES: () => STACK_FEATURE_NAMES,
+  blockFeatures: () => blockFeatures,
+  stackedFeatures: () => stackedFeatures
+});
+module.exports = __toCommonJS(block_features_exports);
+var import_hast_fast = require("../../utils/hast-fast");
+var import_blocks = require("./blocks");
+var import_constants = require("./constants");
+var import_sanitize = require("./sanitize");
+var import_score = require("./score");
+const FEATURE_NAMES = [
+  "bias",
+  "logLen",
+  "linkDensity",
+  "logLinks",
+  "hasImage",
+  "textless",
+  "punctDensity",
+  "digitRatio",
+  "sentenceEnds",
+  "tag_p",
+  "tag_li",
+  "tag_cell",
+  "tag_h1",
+  "tag_h2_6",
+  "tag_pre",
+  "tag_div",
+  "tag_dtdd",
+  "tag_blockquote",
+  "tag_figcaption",
+  "in_nav",
+  "in_header",
+  "in_footer",
+  "in_aside",
+  "in_article",
+  "in_main",
+  "in_form",
+  "in_table",
+  "in_list",
+  "role_chrome",
+  "role_content",
+  "cls_unlikely",
+  "cls_positive",
+  "cls_negative",
+  "cls_strong",
+  "cls_special",
+  "cls_byline",
+  "cls_comment",
+  "landmark",
+  "doom_strong",
+  "doom_weak",
+  "depth",
+  "relPos",
+  "relCharPos",
+  "prev_logLen",
+  "prev_linkDensity",
+  "next_logLen",
+  "next_linkDensity",
+  "win_linkDensity",
+  "win_logLen",
+  "parent_linkDensity",
+  "parent_logText",
+  "gparent_linkDensity",
+  "gparent_logText",
+  "inTopCandidate",
+  "candidateScore",
+  "inTopSibling",
+  "takumi_kept"
+];
+const FEATURE_COUNT = FEATURE_NAMES.length;
+const PUNCT = /* @__PURE__ */ new Set([...".,;:!?\u3001\u3002\uFF0C\uFF0E\uFF01\uFF1F\uFF1B\uFF1A"]);
+const SENTENCE_END = /* @__PURE__ */ new Set([...".!?\u3002\uFF01\uFF1F"]);
+const textStats = (text) => {
+  let punct = 0;
+  let digits = 0;
+  let sentences = 0;
+  for (let index = 0; index < text.length; index++) {
+    const char = text[index];
+    const code = text.charCodeAt(index);
+    if (code >= 48 && code <= 57) {
+      digits += 1;
+    } else if (PUNCT.has(char)) {
+      punct += 1;
+      const next = text.charCodeAt(index + 1);
+      if (SENTENCE_END.has(char) && (Number.isNaN(next) || next === 32 || next === 10 || code > 12288)) {
+        sentences += 1;
+      }
+    }
+  }
+  return { punct, digits, sentences };
+};
+const COMMENT = /comment|disqus|reply|respond/i;
+const log1p = (value) => Math.log(1 + value);
+const signalsOf = (element, cache) => {
+  let signals = cache.get(element);
+  if (signals) {
+    return signals;
+  }
+  const match = (0, import_hast_fast.matchString)(element);
+  const role = (0, import_hast_fast.stringProperty)(element, "role") ?? "";
+  const names = [...(0, import_hast_fast.classList)(element), (0, import_hast_fast.stringProperty)(element, "id") ?? ""];
+  signals = {
+    unlikely: import_constants.REGEXPS.unlikelyCandidates.test(match) && !import_constants.REGEXPS.okMaybeItsaCandidate.test(match),
+    positive: import_constants.REGEXPS.positive.test(match),
+    negative: import_constants.REGEXPS.negative.test(match),
+    strong: names.some((name) => import_constants.REGEXPS.stronglyUnlikely.test(name)),
+    special: import_constants.REGEXPS.specialUnlikelyCandidates.test(match),
+    byline: import_constants.REGEXPS.byline.test(match),
+    comment: COMMENT.test(match),
+    roleChrome: import_constants.UNLIKELY_ROLES.has(role),
+    roleContent: import_constants.CONTENT_ROLES.has(role)
+  };
+  cache.set(element, signals);
+  return signals;
+};
+const SIGNAL_LEVELS = 6;
+const SIGNAL_COUNT = 9;
+class RowWriter {
+  constructor(out) {
+    this.out = out;
+    this.offset = 0;
+  }
+  push(value) {
+    this.out[this.offset++] = value;
+  }
+  flag(condition) {
+    this.out[this.offset++] = condition ? 1 : 0;
+  }
+}
+const EMPTY_CHAIN = {
+  inNav: 0,
+  inHeader: 0,
+  inFooter: 0,
+  inAside: 0,
+  inArticle: 0,
+  inMain: 0,
+  inForm: 0,
+  inTable: 0,
+  inList: 0,
+  landmark: 0,
+  doomStrong: 0,
+  doomWeak: 0,
+  inTop: 0,
+  inTopSibling: 0,
+  nearestScore: 0,
+  distance: new Array(SIGNAL_COUNT).fill(Number.POSITIVE_INFINITY)
+};
+const signalFlags = (signals) => [
+  signals.roleChrome,
+  signals.roleContent,
+  signals.unlikely,
+  signals.positive,
+  signals.negative,
+  signals.strong,
+  signals.special,
+  signals.byline,
+  signals.comment
+];
+const any = (parent, condition) => parent || condition ? 1 : 0;
+class ChainResolver {
+  constructor(landmarks, candidateScore, parents, top) {
+    this.landmarks = landmarks;
+    this.candidateScore = candidateScore;
+    this.parents = parents;
+    this.top = top;
+  }
+  #chains = /* @__PURE__ */ new Map();
+  #signals = /* @__PURE__ */ new WeakMap();
+  /** Iterative, so a deeply nested page cannot exhaust the call stack. */
+  resolve(frame) {
+    const pending = [];
+    let current = frame;
+    let chain = EMPTY_CHAIN;
+    while (current) {
+      const memo = this.#chains.get(current.element);
+      if (memo) {
+        chain = memo;
+        break;
+      }
+      pending.push(current);
+      current = current.parent;
+    }
+    for (let index = pending.length - 1; index >= 0; index--) {
+      chain = this.#derive(chain, pending[index].element);
+      this.#chains.set(pending[index].element, chain);
+    }
+    return chain;
+  }
+  #derive(parent, element) {
+    const tag = element.tagName;
+    const tier = (0, import_sanitize.unlikelyTier)(element);
+    const flags = signalFlags(signalsOf(element, this.#signals));
+    const top = this.top;
+    const topParent = top ? this.parents.get(top) : void 0;
+    return {
+      inNav: any(parent.inNav, tag === "nav"),
+      inHeader: any(parent.inHeader, tag === "header"),
+      inFooter: any(parent.inFooter, tag === "footer"),
+      inAside: any(parent.inAside, tag === "aside"),
+      inArticle: any(parent.inArticle, tag === "article"),
+      inMain: any(parent.inMain, tag === "main"),
+      inForm: any(parent.inForm, tag === "form"),
+      inTable: any(parent.inTable, tag === "table"),
+      inList: any(parent.inList, tag === "ul" || tag === "ol"),
+      landmark: any(parent.landmark, this.landmarks.has(element)),
+      doomStrong: any(parent.doomStrong, tier === "strong"),
+      doomWeak: any(parent.doomWeak, tier === "weak"),
+      inTop: any(parent.inTop, element === top),
+      inTopSibling: any(
+        parent.inTopSibling,
+        topParent !== void 0 && element !== top && this.parents.get(element) === topParent
+      ),
+      nearestScore: this.candidateScore.get(element) ?? parent.nearestScore,
+      distance: parent.distance.map((distance, signal) => flags[signal] ? 0 : distance + 1)
+    };
+  }
+}
+const evidence = (distance) => distance < SIGNAL_LEVELS ? 1 / (1 + distance) : 0;
+const writeText = (row, block, density) => {
+  const length = block.text.length;
+  const stats = textStats(block.text);
+  row.push(1);
+  row.push(log1p(length));
+  row.push(density);
+  row.push(log1p(block.links));
+  row.flag(block.images > 0);
+  row.flag(length === 0);
+  row.push(length === 0 ? 0 : stats.punct / length);
+  row.push(length === 0 ? 0 : stats.digits / length);
+  row.push(log1p(stats.sentences));
+};
+const writeTag = (row, tag) => {
+  row.flag(tag === "p");
+  row.flag(tag === "li");
+  row.flag(tag === "td" || tag === "th");
+  row.flag(tag === "h1");
+  row.flag(/^h[2-6]$/.test(tag));
+  row.flag(tag === "pre");
+  row.flag(tag === "div" || tag === "section");
+  row.flag(tag === "dt" || tag === "dd");
+  row.flag(tag === "blockquote");
+  row.flag(tag === "figcaption");
+};
+const writeChain = (row, chain) => {
+  for (const value of [
+    chain.inNav,
+    chain.inHeader,
+    chain.inFooter,
+    chain.inAside,
+    chain.inArticle,
+    chain.inMain,
+    chain.inForm,
+    chain.inTable,
+    chain.inList
+  ]) {
+    row.push(value);
+  }
+  for (const distance of chain.distance) {
+    row.push(evidence(distance));
+  }
+  row.push(chain.landmark);
+  row.push(chain.doomStrong);
+  row.push(chain.doomWeak);
+};
+const windowMeans = (textLen, density, index, radius) => {
+  let link = 0;
+  let len = 0;
+  let size = 0;
+  for (let near = Math.max(0, index - radius); near <= Math.min(textLen.length - 1, index + radius); near++) {
+    link += density[near];
+    len += log1p(textLen[near]);
+    size += 1;
+  }
+  return [link / size, len / size];
+};
+const writeNeighbourhood = (row, index, textLen, density) => {
+  const hasPrevious = index > 0;
+  const hasNext = index < textLen.length - 1;
+  row.push(hasPrevious ? log1p(textLen[index - 1]) : 0);
+  row.push(hasPrevious ? density[index - 1] : 0);
+  row.push(hasNext ? log1p(textLen[index + 1]) : 0);
+  row.push(hasNext ? density[index + 1] : 0);
+  const [windowLink, windowLen] = windowMeans(textLen, density, index, 2);
+  row.push(windowLink);
+  row.push(windowLen);
+};
+const writeContainers = (row, block, collector) => {
+  for (const container of [(0, import_blocks.ancestorAt)(block, 1), (0, import_blocks.ancestorAt)(block, 2)]) {
+    row.push(container ? collector.linkDensity(container) : 0);
+    row.push(container ? log1p(collector.textLength(container)) : 0);
+  }
+};
+const blockFeatures = ({ root, blocks, collector, lang, takumiKept }) => {
+  const count = blocks.length;
+  const out = new Float32Array(count * FEATURE_COUNT);
+  const context = (0, import_score.buildScoringContext)(root, collector, lang);
+  const candidates = (0, import_score.scoreCandidates)(root, context);
+  const topScore = candidates[0]?.score ?? 1;
+  const candidateScore = /* @__PURE__ */ new WeakMap();
+  for (const candidate of candidates) {
+    candidateScore.set(candidate.element, candidate.score / topScore);
+  }
+  const chains = new ChainResolver((0, import_sanitize.findLandmarkChrome)(root), candidateScore, context.parents, candidates[0]?.element);
+  const textLen = blocks.map((block) => block.text.length);
+  const density = blocks.map((block) => block.text.length === 0 ? 0 : block.linkChars / block.text.length);
+  const totalChars = textLen.reduce((sum, length) => sum + length, 0) || 1;
+  let charsBefore = 0;
+  for (let index = 0; index < count; index++) {
+    const block = blocks[index];
+    const row = new RowWriter(out.subarray(index * FEATURE_COUNT, (index + 1) * FEATURE_COUNT));
+    const chain = chains.resolve(block.frame);
+    writeText(row, block, density[index]);
+    writeTag(row, block.owner.tagName);
+    writeChain(row, chain);
+    row.push(Math.min(block.frame.depth, 40) / 40);
+    row.push(count <= 1 ? 0 : index / (count - 1));
+    row.push(charsBefore / totalChars);
+    charsBefore += textLen[index];
+    writeNeighbourhood(row, index, textLen, density);
+    writeContainers(row, block, collector);
+    row.push(chain.inTop);
+    row.push(chain.nearestScore);
+    row.push(chain.inTopSibling);
+    row.flag(takumiKept?.has(block.owner));
+  }
+  return out;
+};
+const STACK_FEATURE_NAMES = [
+  "s1_logit",
+  "s1_win1",
+  "s1_win3",
+  "s1_parent",
+  "s1_gparent",
+  "s1_ggparent",
+  "s1_parentShare",
+  "s1_distHigh",
+  "s1_docShare"
+];
+const STACKED_FEATURE_COUNT = FEATURE_COUNT + STACK_FEATURE_NAMES.length;
+const logit = (p) => Math.log(Math.max(1e-6, p) / Math.max(1e-6, 1 - p));
+const ancestorMass = (blocks, probabilities) => {
+  const mass = /* @__PURE__ */ new Map();
+  const add = (element, p, weight) => {
+    if (!element) {
+      return;
+    }
+    const entry = mass.get(element);
+    if (entry) {
+      entry.p += p * weight;
+      entry.w += weight;
+    } else {
+      mass.set(element, { p: p * weight, w: weight });
+    }
+  };
+  let docP = 0;
+  let docW = 0;
+  for (const block of blocks) {
+    const weight = block.text.length + 10;
+    const p = probabilities[block.index];
+    add((0, import_blocks.ancestorAt)(block, 1), p, weight);
+    add((0, import_blocks.ancestorAt)(block, 2), p, weight);
+    add((0, import_blocks.ancestorAt)(block, 3), p, weight);
+    docP += p * weight;
+    docW += weight;
+  }
+  const parentWeightShare = (element) => {
+    const entry = element ? mass.get(element) : void 0;
+    return entry && docW > 0 ? entry.w / docW : 0;
+  };
+  return { mass, docShare: docW === 0 ? 0 : docP / docW, parentWeightShare };
+};
+const distanceToHigh = (probabilities) => {
+  const count = probabilities.length;
+  const distance = new Float32Array(count).fill(count);
+  let last = -1;
+  for (let index = 0; index < count; index++) {
+    if (probabilities[index] >= 0.5) {
+      last = index;
+    }
+    if (last >= 0) {
+      distance[index] = index - last;
+    }
+  }
+  last = -1;
+  for (let index = count - 1; index >= 0; index--) {
+    if (probabilities[index] >= 0.5) {
+      last = index;
+    }
+    if (last >= 0) {
+      distance[index] = Math.min(distance[index], last - index);
+    }
+  }
+  return distance;
+};
+const stackedFeatures = (blocks, base, probabilities) => {
+  const count = blocks.length;
+  const stride = STACKED_FEATURE_COUNT;
+  const out = new Float32Array(count * stride);
+  const { mass, docShare, parentWeightShare } = ancestorMass(blocks, probabilities);
+  const meanOf = (element, fallback) => {
+    const entry = element ? mass.get(element) : void 0;
+    return entry ? entry.p / entry.w : fallback;
+  };
+  const distance = distanceToHigh(probabilities);
+  for (let index = 0; index < count; index++) {
+    const row = index * stride;
+    out.set(base.subarray(index * FEATURE_COUNT, (index + 1) * FEATURE_COUNT), row);
+    const p = probabilities[index];
+    const window = (radius) => {
+      let sum = 0;
+      let n = 0;
+      for (let near = Math.max(0, index - radius); near <= Math.min(count - 1, index + radius); near++) {
+        if (near !== index) {
+          sum += probabilities[near];
+          n += 1;
+        }
+      }
+      return n === 0 ? p : sum / n;
+    };
+    const block = blocks[index];
+    let f = row + FEATURE_COUNT;
+    out[f++] = logit(p);
+    out[f++] = window(1);
+    out[f++] = window(3);
+    out[f++] = meanOf((0, import_blocks.ancestorAt)(block, 1), p);
+    out[f++] = meanOf((0, import_blocks.ancestorAt)(block, 2), p);
+    out[f++] = meanOf((0, import_blocks.ancestorAt)(block, 3), p);
+    out[f++] = parentWeightShare((0, import_blocks.ancestorAt)(block, 1));
+    out[f++] = Math.log1p(distance[index]);
+    out[f++] = docShare;
+  }
+  return out;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  FEATURE_COUNT,
+  FEATURE_NAMES,
+  STACKED_FEATURE_COUNT,
+  STACK_FEATURE_NAMES,
+  blockFeatures,
+  stackedFeatures
+});