webforai 3.0.0 → 4.1.0

package/dist/extract-mdast.js

@@ -10,6 +10,61 @@ var emptyDeclarationFilter = (node) => {
   }
   return true;
 };
+var trimEdge = (children, edge) => {
+  const index = edge === "start" ? 0 : children.length - 1;
+  const node = children[index];
+  if (node?.type !== "text") {
+    return children;
+  }
+  const value = edge === "start" ? node.value.trimStart() : node.value.trimEnd();
+  const copy = [...children];
+  if (value.length === 0) {
+    copy.splice(index, 1);
+  } else {
+    copy[index] = { ...node, value };
+  }
+  return copy;
+};
+var liftDisplayMath = (paragraph) => {
+  if (!paragraph.children.some((child) => child.type === "math")) {
+    return void 0;
+  }
+  const out = [];
+  let run = [];
+  const flush = () => {
+    const trimmed = trimEdge(trimEdge(run, "start"), "end");
+    if (trimmed.length > 0) {
+      out.push({ ...paragraph, children: trimmed });
+    }
+    run = [];
+  };
+  for (const child of paragraph.children) {
+    if (child.type === "math") {
+      flush();
+      out.push(child);
+    } else {
+      run.push(child);
+    }
+  }
+  flush();
+  return out;
+};
+var liftDisplayMathIn = (node) => {
+  if (!("children" in node)) {
+    return;
+  }
+  const parent = node;
+  for (let index = 0; index < parent.children.length; index++) {
+    const child = parent.children[index];
+    const lifted = child.type === "paragraph" ? liftDisplayMath(child) : void 0;
+    if (lifted) {
+      parent.children.splice(index, 1, ...lifted);
+      index += lifted.length - 1;
+      continue;
+    }
+    liftDisplayMathIn(child);
+  }
+};
 var extractMdast = (node) => {
   const extracted = filter(node, (node2) => {
     if (!emptyDeclarationFilter(node2)) {
@@ -17,6 +72,9 @@ var extractMdast = (node) => {
     }
     return true;
   });
+  if (extracted) {
+    liftDisplayMathIn(extracted);
+  }
   return extracted;
 };
 export {

package/dist/extract-meta-refresh.js

@@ -0,0 +1,47 @@
+// src/extract-meta-refresh.ts
+var MAX_META_REFRESH_DELAY_SECONDS = 10;
+var META_TAG_PATTERN = /<meta\b[^>]*>/gi;
+var HTTP_EQUIV_REFRESH_PATTERN = /\bhttp-equiv\s*=\s*("\s*refresh\s*"|'\s*refresh\s*'|refresh\b)/i;
+var CONTENT_ATTR_PATTERN = /\bcontent\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i;
+var CONTENT_VALUE_PATTERN = /^\s*(\d+(?:\.\d+)?)\s*[;,]\s*url\s*=\s*(?:"([^"]*)"|'([^']*)'|(.+?))\s*$/i;
+var resolveTarget = (raw, baseUrl) => {
+  try {
+    return baseUrl === void 0 ? new URL(raw) : new URL(raw, baseUrl);
+  } catch {
+    return void 0;
+  }
+};
+var extractMetaRefresh = (html, baseUrl) => {
+  for (const [tag] of html.matchAll(META_TAG_PATTERN)) {
+    if (!HTTP_EQUIV_REFRESH_PATTERN.test(tag)) {
+      continue;
+    }
+    const contentMatch = CONTENT_ATTR_PATTERN.exec(tag);
+    const content = contentMatch?.[1] ?? contentMatch?.[2] ?? contentMatch?.[3];
+    if (!content) {
+      continue;
+    }
+    const value = CONTENT_VALUE_PATTERN.exec(content);
+    if (!value) {
+      continue;
+    }
+    const delaySeconds = Number.parseFloat(value[1] ?? "");
+    const raw = (value[2] ?? value[3] ?? value[4] ?? "").trim();
+    if (!raw || delaySeconds > MAX_META_REFRESH_DELAY_SECONDS) {
+      continue;
+    }
+    const target = resolveTarget(raw, baseUrl);
+    if (!target || target.protocol !== "http:" && target.protocol !== "https:") {
+      continue;
+    }
+    if (baseUrl !== void 0 && resolveTarget(baseUrl, void 0)?.href === target.href) {
+      continue;
+    }
+    return { url: target.href, delaySeconds };
+  }
+  return void 0;
+};
+export {
+  MAX_META_REFRESH_DELAY_SECONDS,
+  extractMetaRefresh
+};

package/dist/extractors/index.js

@@ -3,11 +3,14 @@ import {
   pipeExtractors
 } from "./pipeExtractors.js";
 import { takumiExtractor } from "./presets/takumi.js";
+import { createKiwameExtractor, kiwameExtractor } from "./presets/kiwame.js";
 import { minimalFilter } from "./presets/minimal-filter.js";
 import { autoExtractor, createAutoExtractor } from "./presets/auto.js";
 export {
   autoExtractor,
   createAutoExtractor,
+  createKiwameExtractor,
+  kiwameExtractor,
   minimalFilter,
   pipeExtractors,
   takumiExtractor

package/dist/extractors/lib/block-features.js

@@ -0,0 +1,437 @@
+// src/extractors/lib/block-features.ts
+import { classList, matchString, stringProperty } from "../../utils/hast-fast.js";
+import { ancestorAt } from "./blocks.js";
+import { CONTENT_ROLES, REGEXPS, UNLIKELY_ROLES } from "./constants.js";
+import { findLandmarkChrome, unlikelyTier } from "./sanitize.js";
+import { buildScoringContext, scoreCandidates } from "./score.js";
+var FEATURE_NAMES = [
+  "bias",
+  "logLen",
+  "linkDensity",
+  "logLinks",
+  "hasImage",
+  "textless",
+  "punctDensity",
+  "digitRatio",
+  "sentenceEnds",
+  "tag_p",
+  "tag_li",
+  "tag_cell",
+  "tag_h1",
+  "tag_h2_6",
+  "tag_pre",
+  "tag_div",
+  "tag_dtdd",
+  "tag_blockquote",
+  "tag_figcaption",
+  "in_nav",
+  "in_header",
+  "in_footer",
+  "in_aside",
+  "in_article",
+  "in_main",
+  "in_form",
+  "in_table",
+  "in_list",
+  "role_chrome",
+  "role_content",
+  "cls_unlikely",
+  "cls_positive",
+  "cls_negative",
+  "cls_strong",
+  "cls_special",
+  "cls_byline",
+  "cls_comment",
+  "landmark",
+  "doom_strong",
+  "doom_weak",
+  "depth",
+  "relPos",
+  "relCharPos",
+  "prev_logLen",
+  "prev_linkDensity",
+  "next_logLen",
+  "next_linkDensity",
+  "win_linkDensity",
+  "win_logLen",
+  "parent_linkDensity",
+  "parent_logText",
+  "gparent_linkDensity",
+  "gparent_logText",
+  "inTopCandidate",
+  "candidateScore",
+  "inTopSibling",
+  "takumi_kept"
+];
+var FEATURE_COUNT = FEATURE_NAMES.length;
+var PUNCT = /* @__PURE__ */ new Set([...".,;:!?\u3001\u3002\uFF0C\uFF0E\uFF01\uFF1F\uFF1B\uFF1A"]);
+var SENTENCE_END = /* @__PURE__ */ new Set([...".!?\u3002\uFF01\uFF1F"]);
+var textStats = (text) => {
+  let punct = 0;
+  let digits = 0;
+  let sentences = 0;
+  for (let index = 0; index < text.length; index++) {
+    const char = text[index];
+    const code = text.charCodeAt(index);
+    if (code >= 48 && code <= 57) {
+      digits += 1;
+    } else if (PUNCT.has(char)) {
+      punct += 1;
+      const next = text.charCodeAt(index + 1);
+      if (SENTENCE_END.has(char) && (Number.isNaN(next) || next === 32 || next === 10 || code > 12288)) {
+        sentences += 1;
+      }
+    }
+  }
+  return { punct, digits, sentences };
+};
+var COMMENT = /comment|disqus|reply|respond/i;
+var log1p = (value) => Math.log(1 + value);
+var signalsOf = (element, cache) => {
+  let signals = cache.get(element);
+  if (signals) {
+    return signals;
+  }
+  const match = matchString(element);
+  const role = stringProperty(element, "role") ?? "";
+  const names = [...classList(element), stringProperty(element, "id") ?? ""];
+  signals = {
+    unlikely: REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match),
+    positive: REGEXPS.positive.test(match),
+    negative: REGEXPS.negative.test(match),
+    strong: names.some((name) => REGEXPS.stronglyUnlikely.test(name)),
+    special: REGEXPS.specialUnlikelyCandidates.test(match),
+    byline: REGEXPS.byline.test(match),
+    comment: COMMENT.test(match),
+    roleChrome: UNLIKELY_ROLES.has(role),
+    roleContent: CONTENT_ROLES.has(role)
+  };
+  cache.set(element, signals);
+  return signals;
+};
+var SIGNAL_LEVELS = 6;
+var SIGNAL_COUNT = 9;
+var RowWriter = class {
+  constructor(out) {
+    this.out = out;
+    this.offset = 0;
+  }
+  push(value) {
+    this.out[this.offset++] = value;
+  }
+  flag(condition) {
+    this.out[this.offset++] = condition ? 1 : 0;
+  }
+};
+var EMPTY_CHAIN = {
+  inNav: 0,
+  inHeader: 0,
+  inFooter: 0,
+  inAside: 0,
+  inArticle: 0,
+  inMain: 0,
+  inForm: 0,
+  inTable: 0,
+  inList: 0,
+  landmark: 0,
+  doomStrong: 0,
+  doomWeak: 0,
+  inTop: 0,
+  inTopSibling: 0,
+  nearestScore: 0,
+  distance: new Array(SIGNAL_COUNT).fill(Number.POSITIVE_INFINITY)
+};
+var signalFlags = (signals) => [
+  signals.roleChrome,
+  signals.roleContent,
+  signals.unlikely,
+  signals.positive,
+  signals.negative,
+  signals.strong,
+  signals.special,
+  signals.byline,
+  signals.comment
+];
+var any = (parent, condition) => parent || condition ? 1 : 0;
+var ChainResolver = class {
+  constructor(landmarks, candidateScore, parents, top) {
+    this.landmarks = landmarks;
+    this.candidateScore = candidateScore;
+    this.parents = parents;
+    this.top = top;
+  }
+  #chains = /* @__PURE__ */ new Map();
+  #signals = /* @__PURE__ */ new WeakMap();
+  /** Iterative, so a deeply nested page cannot exhaust the call stack. */
+  resolve(frame) {
+    const pending = [];
+    let current = frame;
+    let chain = EMPTY_CHAIN;
+    while (current) {
+      const memo = this.#chains.get(current.element);
+      if (memo) {
+        chain = memo;
+        break;
+      }
+      pending.push(current);
+      current = current.parent;
+    }
+    for (let index = pending.length - 1; index >= 0; index--) {
+      chain = this.#derive(chain, pending[index].element);
+      this.#chains.set(pending[index].element, chain);
+    }
+    return chain;
+  }
+  #derive(parent, element) {
+    const tag = element.tagName;
+    const tier = unlikelyTier(element);
+    const flags = signalFlags(signalsOf(element, this.#signals));
+    const top = this.top;
+    const topParent = top ? this.parents.get(top) : void 0;
+    return {
+      inNav: any(parent.inNav, tag === "nav"),
+      inHeader: any(parent.inHeader, tag === "header"),
+      inFooter: any(parent.inFooter, tag === "footer"),
+      inAside: any(parent.inAside, tag === "aside"),
+      inArticle: any(parent.inArticle, tag === "article"),
+      inMain: any(parent.inMain, tag === "main"),
+      inForm: any(parent.inForm, tag === "form"),
+      inTable: any(parent.inTable, tag === "table"),
+      inList: any(parent.inList, tag === "ul" || tag === "ol"),
+      landmark: any(parent.landmark, this.landmarks.has(element)),
+      doomStrong: any(parent.doomStrong, tier === "strong"),
+      doomWeak: any(parent.doomWeak, tier === "weak"),
+      inTop: any(parent.inTop, element === top),
+      inTopSibling: any(
+        parent.inTopSibling,
+        topParent !== void 0 && element !== top && this.parents.get(element) === topParent
+      ),
+      nearestScore: this.candidateScore.get(element) ?? parent.nearestScore,
+      distance: parent.distance.map((distance, signal) => flags[signal] ? 0 : distance + 1)
+    };
+  }
+};
+var evidence = (distance) => distance < SIGNAL_LEVELS ? 1 / (1 + distance) : 0;
+var writeText = (row, block, density) => {
+  const length = block.text.length;
+  const stats = textStats(block.text);
+  row.push(1);
+  row.push(log1p(length));
+  row.push(density);
+  row.push(log1p(block.links));
+  row.flag(block.images > 0);
+  row.flag(length === 0);
+  row.push(length === 0 ? 0 : stats.punct / length);
+  row.push(length === 0 ? 0 : stats.digits / length);
+  row.push(log1p(stats.sentences));
+};
+var writeTag = (row, tag) => {
+  row.flag(tag === "p");
+  row.flag(tag === "li");
+  row.flag(tag === "td" || tag === "th");
+  row.flag(tag === "h1");
+  row.flag(/^h[2-6]$/.test(tag));
+  row.flag(tag === "pre");
+  row.flag(tag === "div" || tag === "section");
+  row.flag(tag === "dt" || tag === "dd");
+  row.flag(tag === "blockquote");
+  row.flag(tag === "figcaption");
+};
+var writeChain = (row, chain) => {
+  for (const value of [
+    chain.inNav,
+    chain.inHeader,
+    chain.inFooter,
+    chain.inAside,
+    chain.inArticle,
+    chain.inMain,
+    chain.inForm,
+    chain.inTable,
+    chain.inList
+  ]) {
+    row.push(value);
+  }
+  for (const distance of chain.distance) {
+    row.push(evidence(distance));
+  }
+  row.push(chain.landmark);
+  row.push(chain.doomStrong);
+  row.push(chain.doomWeak);
+};
+var windowMeans = (textLen, density, index, radius) => {
+  let link = 0;
+  let len = 0;
+  let size = 0;
+  for (let near = Math.max(0, index - radius); near <= Math.min(textLen.length - 1, index + radius); near++) {
+    link += density[near];
+    len += log1p(textLen[near]);
+    size += 1;
+  }
+  return [link / size, len / size];
+};
+var writeNeighbourhood = (row, index, textLen, density) => {
+  const hasPrevious = index > 0;
+  const hasNext = index < textLen.length - 1;
+  row.push(hasPrevious ? log1p(textLen[index - 1]) : 0);
+  row.push(hasPrevious ? density[index - 1] : 0);
+  row.push(hasNext ? log1p(textLen[index + 1]) : 0);
+  row.push(hasNext ? density[index + 1] : 0);
+  const [windowLink, windowLen] = windowMeans(textLen, density, index, 2);
+  row.push(windowLink);
+  row.push(windowLen);
+};
+var writeContainers = (row, block, collector) => {
+  for (const container of [ancestorAt(block, 1), ancestorAt(block, 2)]) {
+    row.push(container ? collector.linkDensity(container) : 0);
+    row.push(container ? log1p(collector.textLength(container)) : 0);
+  }
+};
+var blockFeatures = ({ root, blocks, collector, lang, takumiKept }) => {
+  const count = blocks.length;
+  const out = new Float32Array(count * FEATURE_COUNT);
+  const context = buildScoringContext(root, collector, lang);
+  const candidates = scoreCandidates(root, context);
+  const topScore = candidates[0]?.score ?? 1;
+  const candidateScore = /* @__PURE__ */ new WeakMap();
+  for (const candidate of candidates) {
+    candidateScore.set(candidate.element, candidate.score / topScore);
+  }
+  const chains = new ChainResolver(findLandmarkChrome(root), candidateScore, context.parents, candidates[0]?.element);
+  const textLen = blocks.map((block) => block.text.length);
+  const density = blocks.map((block) => block.text.length === 0 ? 0 : block.linkChars / block.text.length);
+  const totalChars = textLen.reduce((sum, length) => sum + length, 0) || 1;
+  let charsBefore = 0;
+  for (let index = 0; index < count; index++) {
+    const block = blocks[index];
+    const row = new RowWriter(out.subarray(index * FEATURE_COUNT, (index + 1) * FEATURE_COUNT));
+    const chain = chains.resolve(block.frame);
+    writeText(row, block, density[index]);
+    writeTag(row, block.owner.tagName);
+    writeChain(row, chain);
+    row.push(Math.min(block.frame.depth, 40) / 40);
+    row.push(count <= 1 ? 0 : index / (count - 1));
+    row.push(charsBefore / totalChars);
+    charsBefore += textLen[index];
+    writeNeighbourhood(row, index, textLen, density);
+    writeContainers(row, block, collector);
+    row.push(chain.inTop);
+    row.push(chain.nearestScore);
+    row.push(chain.inTopSibling);
+    row.flag(takumiKept?.has(block.owner));
+  }
+  return out;
+};
+var STACK_FEATURE_NAMES = [
+  "s1_logit",
+  "s1_win1",
+  "s1_win3",
+  "s1_parent",
+  "s1_gparent",
+  "s1_ggparent",
+  "s1_parentShare",
+  "s1_distHigh",
+  "s1_docShare"
+];
+var STACKED_FEATURE_COUNT = FEATURE_COUNT + STACK_FEATURE_NAMES.length;
+var logit = (p) => Math.log(Math.max(1e-6, p) / Math.max(1e-6, 1 - p));
+var ancestorMass = (blocks, probabilities) => {
+  const mass = /* @__PURE__ */ new Map();
+  const add = (element, p, weight) => {
+    if (!element) {
+      return;
+    }
+    const entry = mass.get(element);
+    if (entry) {
+      entry.p += p * weight;
+      entry.w += weight;
+    } else {
+      mass.set(element, { p: p * weight, w: weight });
+    }
+  };
+  let docP = 0;
+  let docW = 0;
+  for (const block of blocks) {
+    const weight = block.text.length + 10;
+    const p = probabilities[block.index];
+    add(ancestorAt(block, 1), p, weight);
+    add(ancestorAt(block, 2), p, weight);
+    add(ancestorAt(block, 3), p, weight);
+    docP += p * weight;
+    docW += weight;
+  }
+  const parentWeightShare = (element) => {
+    const entry = element ? mass.get(element) : void 0;
+    return entry && docW > 0 ? entry.w / docW : 0;
+  };
+  return { mass, docShare: docW === 0 ? 0 : docP / docW, parentWeightShare };
+};
+var distanceToHigh = (probabilities) => {
+  const count = probabilities.length;
+  const distance = new Float32Array(count).fill(count);
+  let last = -1;
+  for (let index = 0; index < count; index++) {
+    if (probabilities[index] >= 0.5) {
+      last = index;
+    }
+    if (last >= 0) {
+      distance[index] = index - last;
+    }
+  }
+  last = -1;
+  for (let index = count - 1; index >= 0; index--) {
+    if (probabilities[index] >= 0.5) {
+      last = index;
+    }
+    if (last >= 0) {
+      distance[index] = Math.min(distance[index], last - index);
+    }
+  }
+  return distance;
+};
+var stackedFeatures = (blocks, base, probabilities) => {
+  const count = blocks.length;
+  const stride = STACKED_FEATURE_COUNT;
+  const out = new Float32Array(count * stride);
+  const { mass, docShare, parentWeightShare } = ancestorMass(blocks, probabilities);
+  const meanOf = (element, fallback) => {
+    const entry = element ? mass.get(element) : void 0;
+    return entry ? entry.p / entry.w : fallback;
+  };
+  const distance = distanceToHigh(probabilities);
+  for (let index = 0; index < count; index++) {
+    const row = index * stride;
+    out.set(base.subarray(index * FEATURE_COUNT, (index + 1) * FEATURE_COUNT), row);
+    const p = probabilities[index];
+    const window = (radius) => {
+      let sum = 0;
+      let n = 0;
+      for (let near = Math.max(0, index - radius); near <= Math.min(count - 1, index + radius); near++) {
+        if (near !== index) {
+          sum += probabilities[near];
+          n += 1;
+        }
+      }
+      return n === 0 ? p : sum / n;
+    };
+    const block = blocks[index];
+    let f = row + FEATURE_COUNT;
+    out[f++] = logit(p);
+    out[f++] = window(1);
+    out[f++] = window(3);
+    out[f++] = meanOf(ancestorAt(block, 1), p);
+    out[f++] = meanOf(ancestorAt(block, 2), p);
+    out[f++] = meanOf(ancestorAt(block, 3), p);
+    out[f++] = parentWeightShare(ancestorAt(block, 1));
+    out[f++] = Math.log1p(distance[index]);
+    out[f++] = docShare;
+  }
+  return out;
+};
+export {
+  FEATURE_COUNT,
+  FEATURE_NAMES,
+  STACKED_FEATURE_COUNT,
+  STACK_FEATURE_NAMES,
+  blockFeatures,
+  stackedFeatures
+};