pi-unsloth-webtools 0.2.4 → 0.3.0

package/pdf.ts

@@ -26,6 +26,7 @@ interface MupdfDocument {
 
 interface MupdfPage {
   toStructuredText(options?: string): MupdfStructuredText;
+  getBounds(): [number, number, number, number];
   getLinks(): { getBounds(): [number, number, number, number]; getURI(): string }[];
   destroy(): void;
 }
@@ -92,7 +93,6 @@ interface LinkInfo {
 interface TableBand {
   markdown: string;
   firstLineIndex: number;
-  lineIndexes: Set<number>;
 }
 
 interface HeaderInfo {
@@ -137,10 +137,13 @@ function markdownCorrupted(text: string): boolean {
   return shaped > threshold || (text.match(/\ufffd/g) ?? []).length > threshold;
 }
 
-function markdownIncomplete(markdown: string, plain: string): boolean {
-  const plainLetters = [...plain].filter((c) => /[\p{L}\p{N}]/u.test(c)).length;
+function countLetters(text: string): number {
+  return [...text].filter((c) => /[\p{L}\p{N}]/u.test(c)).length;
+}
+
+function markdownIncomplete(markdown: string, plainLetters: number): boolean {
   if (plainLetters < PDF_INCOMPLETE_MIN_LETTERS) return false;
-  const markdownLetters = [...markdown].filter((c) => /[\p{L}\p{N}]/u.test(c)).length;
+  const markdownLetters = countLetters(markdown);
   return markdownLetters < PDF_INCOMPLETE_RATIO * plainLetters;
 }
 
@@ -256,6 +259,84 @@ function getRawLines(json: JsonBlock[]): MergedLine[] {
   return nlines;
 }
 
+const RUNNING_EDGE_FRACTION = 0.12;
+const RUNNING_MIN_PAGES = 2;
+const RUNNING_MIN_FRACTION = 0.5;
+const RUNNING_POSITION_TOLERANCE = 5;
+const PAGE_NUMBER_RE = /^\d{1,4}$/;
+
+function lineText(line: MergedLine): string {
+  return line.spans.map((s) => s.text).join(" ").trim();
+}
+
+function lineAtEdge(line: MergedLine, pageHeight: number): boolean {
+  return (
+    line.lrect.y0 < pageHeight * RUNNING_EDGE_FRACTION ||
+    line.lrect.y1 > pageHeight * (1 - RUNNING_EDGE_FRACTION)
+  );
+}
+
+function linePositionKey(y0: number): number {
+  return Math.round(y0 / RUNNING_POSITION_TOLERANCE);
+}
+
+function countRunningLines(
+  pages: PageData[],
+  heights: number[],
+): { textCounts: Map<string, number>; numericPositionCounts: Map<number, number> } {
+  const textCounts = new Map<string, number>();
+  const numericPositionCounts = new Map<number, number>();
+  for (let i = 0; i < pages.length; i++) {
+    const height = heights[i];
+    const seenTexts = new Set<string>();
+    const seenPositions = new Set<number>();
+    for (const line of pages[i].lines) {
+      if (!lineAtEdge(line, height)) continue;
+      const text = lineText(line);
+      if (!text) continue;
+      const position = linePositionKey(line.lrect.y0);
+      const key = `${text}@${position}`;
+      if (!seenTexts.has(key)) {
+        seenTexts.add(key);
+        textCounts.set(key, (textCounts.get(key) ?? 0) + 1);
+      }
+      if (!seenPositions.has(position)) {
+        seenPositions.add(position);
+        if (PAGE_NUMBER_RE.test(text)) {
+          numericPositionCounts.set(position, (numericPositionCounts.get(position) ?? 0) + 1);
+        }
+      }
+    }
+  }
+  return { textCounts, numericPositionCounts };
+}
+
+function stripRunningLines(pages: PageData[], heights: number[]): number[] {
+  const { textCounts, numericPositionCounts } = countRunningLines(pages, heights);
+  const threshold = Math.max(RUNNING_MIN_PAGES, Math.ceil(pages.length * RUNNING_MIN_FRACTION));
+  const removedLetters: number[] = [];
+  for (let i = 0; i < pages.length; i++) {
+    const height = heights[i];
+    let removed = 0;
+    pages[i].lines = pages[i].lines.filter((line) => {
+      if (!lineAtEdge(line, height)) return true;
+      const text = lineText(line);
+      if (!text) return true;
+      const position = linePositionKey(line.lrect.y0);
+      const repeated = (textCounts.get(`${text}@${position}`) ?? 0) >= threshold;
+      const pageNumber =
+        PAGE_NUMBER_RE.test(text) && (numericPositionCounts.get(position) ?? 0) >= threshold;
+      if (repeated || pageNumber) {
+        removed += countLetters(text);
+        return false;
+      }
+      return true;
+    });
+    removedLetters.push(removed);
+  }
+  return removedLetters;
+}
+
 function findLink(links: LinkInfo[], span: SpanData): string | null {
   const midX = (span.x0 + span.x1) / 2;
   const midY = (span.y0 + span.y1) / 2;
@@ -317,8 +398,7 @@ function detectTableBands(lines: MergedLine[]): TableBand[] {
           for (const row of rows.slice(1)) {
             output += "|" + row.join("|") + "|\n";
           }
-          const indexes = new Set(band.map((l) => lines.indexOf(l)));
-          bands.push({ markdown: output + "\n", firstLineIndex: Math.min(...indexes), lineIndexes: indexes });
+          bands.push({ markdown: output + "\n", firstLineIndex: Math.min(...band.map((l) => lines.indexOf(l))) });
         }
       }
     }
@@ -436,10 +516,6 @@ function writeText(
     }
     out += "\n";
   }
-  while (emittedBands < tableBands.length) {
-    out += "\n" + tableBands[emittedBands].markdown;
-    emittedBands++;
-  }
   out += "\n";
   if (code) out += "```\n";
   out += "\n\n";
@@ -473,8 +549,11 @@ export async function extractPdfPages(
     const total = doc.countPages();
     const count = Math.min(total, MAX_WEB_PDF_PAGES);
     const pages: PageData[] = [];
+    const heights: number[] = [];
     for (let i = 0; i < count; i++) {
       const page = doc.loadPage(i);
+      const bounds = page.getBounds();
+      heights.push(bounds[3] - bounds[1]);
       let st: MupdfStructuredText | null = null;
       try {
         st = page.toStructuredText("");
@@ -496,10 +575,12 @@ export async function extractPdfPages(
         page.destroy();
       }
     }
+    const removedLetters = stripRunningLines(pages, heights);
     const info = identifyHeaders(pages.map((p) => p.lines));
     const extracted = pages.map((p, i) => {
       const markdown = renderPageMarkdown(p.lines, info, p.links);
-      const text = markdownCorrupted(markdown) || markdownIncomplete(markdown, p.plain) ? p.plain : markdown;
+      const plainLetters = Math.max(0, countLetters(p.plain) - removedLetters[i]);
+      const text = markdownCorrupted(markdown) || markdownIncomplete(markdown, plainLetters) ? p.plain : markdown;
       return { text, pageNumber: i + 1 };
     });
     return { pages: extracted, totalPages: total };

package/web-access.ts

@@ -77,11 +77,12 @@ function compressIpv6(ip: string): string {
 }
 
 function compressGroups(groups: string[]): string {
+  const normalized = groups.map((g) => (/^0+$/.test(g) ? "0" : g.replace(/^0+(?=[0-9a-f])/, "")));
   let bestStart = -1;
   let bestLen = 0;
   let runStart = -1;
-  for (let i = 0; i <= groups.length; i++) {
-    if (i < groups.length && groups[i] === "0") {
+  for (let i = 0; i <= normalized.length; i++) {
+    if (i < normalized.length && normalized[i] === "0") {
       if (runStart === -1) runStart = i;
     } else if (runStart !== -1) {
       const runLen = i - runStart;
@@ -92,10 +93,20 @@ function compressGroups(groups: string[]): string {
       runStart = -1;
     }
   }
-  if (bestLen < 2) return groups.map((g) => g.replace(/^0+(?=[0-9a-f])/, "")).join(":");
-  const head = groups.slice(0, bestStart).map((g) => g.replace(/^0+(?=[0-9a-f])/, ""));
-  const tail = groups.slice(bestStart + bestLen).map((g) => g.replace(/^0+(?=[0-9a-f])/, ""));
-  return [...head, "", ...tail].join(":");
+  if (bestLen < 2) return normalized.join(":");
+  const head = normalized.slice(0, bestStart);
+  const tail = normalized.slice(bestStart + bestLen);
+  return `${head.join(":")}::${tail.join(":")}`;
+}
+
+const PCP_ANYCAST = new Set(["2001:1::1", "2001:1::2"]);
+
+function isPcpAnycast(lower: string): boolean {
+  try {
+    return PCP_ANYCAST.has(compressIpv6(lower));
+  } catch {
+    return false;
+  }
 }
 
 export function normalizeWebsitePolicy(value: unknown): WebsitePolicy {
@@ -119,15 +130,7 @@ export function normalizeWebsitePolicy(value: unknown): WebsitePolicy {
     if (rawDomains.length > MAX_DOMAINS_PER_LIST) {
       throw new Error(`${key} supports at most ${MAX_DOMAINS_PER_LIST} domains`);
     }
-    const domains: string[] = [];
-    for (const rawDomain of rawDomains) {
-      if (typeof rawDomain !== "string" || rawDomain.length > MAX_CACHEABLE_DOMAIN_LEN) {
-        throw new Error(`${key} must contain only strings`);
-      }
-      const domain = normalizeDomain(rawDomain);
-      if (!domains.includes(domain)) domains.push(domain);
-    }
-    normalized[key] = domains;
+    normalized[key] = normalizeDomainList(rawDomains, key);
   }
   return normalized;
 }
@@ -136,12 +139,22 @@ function matchesDomain(hostname: string, domain: string): boolean {
   return hostname === domain || hostname.endsWith(`.${domain}`);
 }
 
-export function hostnameAllowed(hostname: string, policy: WebsitePolicy | null): boolean {
-  let host: string;
+function normalizeDomainList(domains: unknown[], listName: string): string[] {
+  const out: string[] = [];
+  for (const rawDomain of domains) {
+    if (typeof rawDomain !== "string" || rawDomain.length > MAX_CACHEABLE_DOMAIN_LEN) {
+      throw new Error(`${listName} must contain only strings`);
+    }
+    const domain = normalizeDomain(rawDomain);
+    if (!out.includes(domain)) out.push(domain);
+  }
+  return out;
+}
+
+function policyAllows(host: string, policy: WebsitePolicy | null): boolean {
   let normalized: WebsitePolicy;
   try {
-    host = normalizeDomain(hostname);
-    normalized = normalizePolicyMaybe(policy);
+    normalized = normalizeWebsitePolicy(policy);
   } catch {
     return false;
   }
@@ -150,22 +163,12 @@ export function hostnameAllowed(hostname: string, policy: WebsitePolicy | null):
   return allowed.length === 0 || allowed.some((domain) => matchesDomain(host, domain));
 }
 
-function normalizePolicyObject(policy: WebsitePolicy): WebsitePolicy {
-  const normalized: WebsitePolicy = { allowedDomains: [], blockedDomains: [] };
-  for (const key of ["allowedDomains", "blockedDomains"] as const) {
-    const domains: string[] = [];
-    for (const rawDomain of policy[key]) {
-      const domain = normalizeDomain(rawDomain);
-      if (!domains.includes(domain)) domains.push(domain);
-    }
-    normalized[key] = domains;
+export function hostnameAllowed(hostname: string, policy: WebsitePolicy | null): boolean {
+  try {
+    return policyAllows(normalizeDomain(hostname), policy);
+  } catch {
+    return false;
   }
-  return normalized;
-}
-
-function normalizePolicyMaybe(policy: WebsitePolicy | null | undefined): WebsitePolicy {
-  if (!policy) return { allowedDomains: [], blockedDomains: [] };
-  return normalizePolicyObject(policy);
 }
 
 export function checkUrlAccess(
@@ -211,7 +214,7 @@ export function checkUrlAccess(
   } catch {
     return [false, "Blocked: the URL has an invalid hostname or port.", ""];
   }
-  if (!hostnameAllowed(hostname, policy)) {
+  if (!policyAllows(hostname, policy)) {
     return [false, `Blocked: the website access policy disallows ${hostname}.`, hostname];
   }
   return [true, "", hostname];
@@ -315,7 +318,7 @@ const GITHUB_NON_OWNER_SEGMENTS = new Set([
 
 const GITHUB_NAME_RE = /^[A-Za-z0-9_.\-]{1,100}$/;
 
-export function githubRepoReadmeApiUrl(url: string): string | null {
+function githubRepoOwnerRepo(url: string): [string, string] | null {
   let parsed: URL;
   try {
     parsed = new URL(url);
@@ -330,7 +333,17 @@ export function githubRepoReadmeApiUrl(url: string): string | null {
   if (GITHUB_NON_OWNER_SEGMENTS.has(owner.toLowerCase())) return null;
   const cleanRepo = repo.endsWith(".git") ? repo.slice(0, -4) : repo;
   if (!GITHUB_NAME_RE.test(owner) || !GITHUB_NAME_RE.test(cleanRepo)) return null;
-  return `https://api.github.com/repos/${owner}/${cleanRepo}/readme`;
+  return [owner, cleanRepo];
+}
+
+export function githubRepoReadmeApiUrl(url: string): string | null {
+  const pair = githubRepoOwnerRepo(url);
+  return pair ? `https://api.github.com/repos/${pair[0]}/${pair[1]}/readme` : null;
+}
+
+export function githubRepoRawReadmeUrl(url: string): string | null {
+  const pair = githubRepoOwnerRepo(url);
+  return pair ? `https://raw.githubusercontent.com/${pair[0]}/${pair[1]}/HEAD/README.md` : null;
 }
 
 function ipv4Octets(ip: string): number[] | null {
@@ -353,6 +366,7 @@ export function isPublicIp(ip: string): boolean {
     if (o[0] === 172 && o[1] >= 16 && o[1] <= 31) return false;
     if (o[0] === 192 && o[1] === 0 && o[2] === 0) return false;
     if (o[0] === 192 && o[1] === 0 && o[2] === 2) return false;
+    if (o[0] === 192 && o[1] === 88 && o[2] === 99) return false;
     if (o[0] === 192 && o[1] === 168) return false;
     if (o[0] === 198 && (o[1] === 18 || o[1] === 19)) return false;
     if (o[0] === 198 && o[1] === 51 && o[2] === 100) return false;
@@ -360,18 +374,22 @@ export function isPublicIp(ip: string): boolean {
     if (o[0] >= 224) return false;
     return true;
   }
-  const lower = ip.toLowerCase();
-  if (lower === "::" || lower === "::1") return false;
-  if (lower.startsWith("fc") || lower.startsWith("fd")) return false;
-  if (/^fe[89ab][0-9a-f]:/.test(lower)) return false;
-  if (lower.startsWith("ff")) return false;
-  if (lower.startsWith("2001:db8")) return false;
-  if (lower.startsWith("64:ff9b:")) return false;
-  if (lower.startsWith("2001:10:")) return false;
-  if (lower.startsWith("2002:")) return false;
-  if (lower.startsWith("2001:0:") || lower.startsWith("2001::")) return false;
-  const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(lower);
-  if (mapped) return isPublicIp(mapped[1]);
-  if (lower.startsWith("::ffff:")) return false;
+  let canonical: string;
+  try {
+    canonical = compressIpv6(ip);
+  } catch {
+    return false;
+  }
+  if (canonical === "::1" || canonical.startsWith("::")) return false;
+  if (canonical.startsWith("fc") || canonical.startsWith("fd")) return false;
+  if (/^fe[89ab][0-9a-f]:/.test(canonical)) return false;
+  if (canonical.startsWith("ff")) return false;
+  if (canonical.startsWith("2001:db8")) return false;
+  if (canonical.startsWith("64:ff9b:")) return false;
+  if (canonical.startsWith("2001:10:")) return false;
+  if (canonical.startsWith("2002:")) return false;
+  if (canonical.startsWith("2001:0:") || canonical.startsWith("2001::")) return false;
+  if (canonical.startsWith("2001:2:")) return false;
+  if (isPcpAnycast(canonical)) return false;
   return true;
 }