token-goat 2.9.12 → 2.9.14

package/dist/token-goat-chunk-DBNY4RLN.mjs

@@ -0,0 +1,308 @@
+import { createRequire as __cjsRequire } from 'node:module';
+const require = __cjsRequire(import.meta.url);
+import {
+  DEFAULT_OCR_LANG,
+  isSupportedOcrLang,
+  loadConfig,
+  resolveOcrLangs
+} from "./token-goat-chunk-SAQ5PG4L.mjs";
+import {
+  ensureDirSync,
+  fenceUntrustedOcrText,
+  tokenGoatHome
+} from "./token-goat-chunk-AMYCQJX4.mjs";
+import {
+  init_define_import_meta_env
+} from "./token-goat-chunk-A37V4PBF.mjs";
+
+// src/image_ocr.ts
+init_define_import_meta_env();
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import * as fs from "node:fs";
+import { createRequire } from "node:module";
+import * as path from "node:path";
+
+// src/ocr_hashes.ts
+init_define_import_meta_env();
+function ocrLangPath(lang) {
+  return `https://cdn.jsdelivr.net/npm/@tesseract.js-data/${lang}@1.0.0/4.0.0_best_int`;
+}
+var OCR_LANG_HASHES = {
+  eng: "5dc5d8d640a212c9d6184921ba103b186f50e0fed9ee716c53e6b312b400d747",
+  fra: "bf83833fa957ff0076f6aa93f69e3bdf7b014dea829ea0c0d6be6b48a3ceef6d",
+  spa: "0062377729b81cc268b1822f09eb1c08c09f3f7f1c6b422540b51555a7eeea70",
+  deu: "a1b72cc25753eac167edfef5af4448a8dc34973503a2265c34c84520f896eb03",
+  ita: "36ec897f5f1f489b257801881286167a79b99a16040c6fbc7e3e30f03821b10b",
+  por: "42fab1f017aedab69b92bdecc01bbb11166cd3b177575612ee860f8e2825ece0",
+  nld: "363c360db9838838ff7ed3d8b885b33acc0d61d37165708303cf8a81e50164f3",
+  pol: "02b89cad819f1374631b4a3c92bdac79c214150f97a3686df78de6c8c30782db",
+  rus: "eb9be824435f6bb0f993925acb85fd842c8418d6db7613c818e749e619a1ad6d",
+  tur: "f0127d0f3745f9c65e2ae7ec6b23198fbe5aa186a61b35661426f5e1ef9dedd7",
+  swe: "a4c33cbd23d988c84b5f9d5d5b2417fc16da061e3dc7b16acb904a4566088b14",
+  ara: "e7d6494e2ef249ee97ad151eb01e0e6ae3aaf429256442ad6af534862a2a8c0f",
+  chi_sim: "9784f7c917c546424b690fcde708ce1f604a4393d08bb51ddab146d7d7c794e6",
+  chi_tra: "6abfb87cce5db0d09624f16eedd8a0b24173718856121f721b6d1214193d4dab",
+  jpn: "1a0175291ea145d4a66be681d1084496f10af938aacab247c5d40b31359a604e",
+  kor: "ec1749377d49ac38fb3d3cd05dd5e2a53359d329f359762bc638a81132109992"
+};
+var SUPPORTED_OCR_LANGS = Object.fromEntries(
+  Object.entries(OCR_LANG_HASHES).map(([lang, sha256]) => [lang, { langPath: ocrLangPath(lang), sha256 }])
+);
+function getOcrLangSpec(lang) {
+  const tokens = lang.split("+");
+  const primary = tokens.find((t) => t !== DEFAULT_OCR_LANG && isSupportedOcrLang(t)) ?? tokens[0] ?? DEFAULT_OCR_LANG;
+  const active = isSupportedOcrLang(primary) ? primary : DEFAULT_OCR_LANG;
+  const sha256 = OCR_LANG_HASHES[active];
+  return {
+    langPath: ocrLangPath(active),
+    sha256
+  };
+}
+
+// src/image_ocr.ts
+var _ocrTimeoutMs = 12e3;
+function setOcrTimeoutForTesting(ms) {
+  _ocrTimeoutMs = ms ?? 12e3;
+}
+function getActiveOcrLangs(lang) {
+  if (typeof lang === "string" && lang.trim()) {
+    return resolveOcrLangs(lang);
+  }
+  try {
+    const configured = loadConfig().image_shrink?.ocr_lang;
+    if (typeof configured === "string" && configured.trim()) {
+      return resolveOcrLangs(configured);
+    }
+  } catch {
+  }
+  return [DEFAULT_OCR_LANG];
+}
+function resolveOcrLang(lang) {
+  return getActiveOcrLangs(lang).join("+");
+}
+var OCR_LANG_FILE = `${DEFAULT_OCR_LANG}.traineddata`;
+var OCR_LANG_PATH = getOcrLangSpec("eng").langPath;
+var OCR_LANG_SHA256 = getOcrLangSpec("eng").sha256;
+function verifyOcrLangCache(lang) {
+  const activeLangs = getActiveOcrLangs(lang);
+  let anyAbsent = false;
+  for (const l of activeLangs) {
+    const spec = getOcrLangSpec(l);
+    const file = path.join(ocrCacheDir(), `${l}.traineddata`);
+    try {
+      if (!fs.existsSync(file)) {
+        anyAbsent = true;
+        continue;
+      }
+      const bytes = fs.readFileSync(file);
+      if (createHash("sha256").update(bytes).digest("hex") !== spec.sha256) {
+        return "mismatch";
+      }
+    } catch {
+      return "unreadable";
+    }
+  }
+  return anyAbsent ? "absent" : "ok";
+}
+function quarantineOcrLangCache(lang) {
+  const activeLangs = getActiveOcrLangs(lang);
+  for (const l of activeLangs) {
+    const spec = getOcrLangSpec(l);
+    const file = path.join(ocrCacheDir(), `${l}.traineddata`);
+    try {
+      if (fs.existsSync(file)) {
+        const bytes = fs.readFileSync(file);
+        if (createHash("sha256").update(bytes).digest("hex") !== spec.sha256) {
+          fs.rmSync(file, { force: true });
+        }
+      }
+    } catch {
+    }
+  }
+}
+var _ocrIntegrityFailed = false;
+function ocrIntegrityFailed() {
+  return _ocrIntegrityFailed;
+}
+function ocrBlockedOffline(lang) {
+  if (!loadConfig().network.offline) return false;
+  return verifyOcrLangCache(lang) === "absent";
+}
+function ocrCacheDir() {
+  return path.join(tokenGoatHome(), "ocr-cache");
+}
+function ensureOcrCacheDir() {
+  try {
+    ensureDirSync(ocrCacheDir());
+  } catch {
+  }
+}
+var _require = createRequire(import.meta.url);
+var _tesseractEntryPath;
+function resolveTesseractEntry() {
+  if (_tesseractEntryPath !== void 0) return _tesseractEntryPath;
+  try {
+    _tesseractEntryPath = _require.resolve("tesseract.js");
+  } catch {
+    _tesseractEntryPath = null;
+  }
+  return _tesseractEntryPath;
+}
+function isOcrEngineAvailable() {
+  return !_ocrUnavailableThisProcess && resolveTesseractEntry() !== null;
+}
+function setTesseractEntryForTesting(entryPath) {
+  _tesseractEntryPath = entryPath;
+}
+var _ocrUnavailableThisProcess = false;
+function resetOcrStateForTesting() {
+  _ocrUnavailableThisProcess = false;
+  _ocrIntegrityFailed = false;
+  _tesseractEntryPath = void 0;
+  _ocrTimeoutMs = 12e3;
+}
+function buildChildScript(entryPath, cacheDir, lang) {
+  const activeLangs = getActiveOcrLangs(lang);
+  const activeLangString = activeLangs.join("+");
+  const spec = getOcrLangSpec(lang ?? activeLangString);
+  return [
+    `const { createWorker } = require(${JSON.stringify(entryPath)});`,
+    "const chunks = [];",
+    "process.stdin.on('data', (c) => chunks.push(c));",
+    "process.stdin.on('end', async () => {",
+    "  try {",
+    "    const buf = Buffer.concat(chunks);",
+    `    const worker = await createWorker(${JSON.stringify(activeLangString)}, 1, { cachePath: ${JSON.stringify(cacheDir)}, langPath: ${JSON.stringify(spec.langPath)}, errorHandler: () => {} });`,
+    "    const { data } = await worker.recognize(buf);",
+    "    process.stdout.write(JSON.stringify({ text: data.text || '', confidence: data.confidence || 0 }));",
+    "    await worker.terminate();",
+    "    process.exit(0);",
+    "  } catch (e) {",
+    "    process.exit(1);",
+    "  }",
+    "});"
+  ].join("\n");
+}
+async function ocrImage(input, lang) {
+  if (_ocrUnavailableThisProcess) return null;
+  const activeLangs = getActiveOcrLangs(lang);
+  const entryPath = resolveTesseractEntry();
+  if (entryPath === null) return null;
+  if (ocrBlockedOffline(lang)) return null;
+  if (_ocrIntegrityFailed) return null;
+  if (verifyOcrLangCache(lang) === "mismatch") {
+    _ocrIntegrityFailed = true;
+    quarantineOcrLangCache(lang);
+    return null;
+  }
+  for (const l of activeLangs) {
+    if (l !== DEFAULT_OCR_LANG) {
+      const file = path.join(ocrCacheDir(), `${l}.traineddata`);
+      if (!fs.existsSync(file)) {
+        if (loadConfig().network.offline) return null;
+        try {
+          const res = await fetch(getOcrLangSpec(l).langPath + `/${l}.traineddata.gz`);
+          if (!res.ok) return null;
+          const gz = Buffer.from(await res.arrayBuffer());
+          const unzipped = (await import("node:zlib")).gunzipSync(gz);
+          if (createHash("sha256").update(unzipped).digest("hex") !== getOcrLangSpec(l).sha256) {
+            _ocrIntegrityFailed = true;
+            return null;
+          }
+          ensureOcrCacheDir();
+          fs.writeFileSync(file, unzipped);
+        } catch {
+          return null;
+        }
+      }
+    }
+  }
+  return new Promise((resolve) => {
+    let settled = false;
+    let child;
+    try {
+      ensureOcrCacheDir();
+      child = spawn(process.execPath, ["-e", buildChildScript(entryPath, ocrCacheDir(), lang)], {
+        stdio: ["pipe", "pipe", "ignore"]
+      });
+    } catch {
+      _ocrUnavailableThisProcess = true;
+      resolve(null);
+      return;
+    }
+    const chunks = [];
+    const finish = (result, subprocessBroken) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (subprocessBroken) _ocrUnavailableThisProcess = true;
+      try {
+        child.kill();
+      } catch {
+      }
+      resolve(result);
+    };
+    const timer = setTimeout(() => finish(null, true), _ocrTimeoutMs);
+    child.stdout?.on("data", (c) => chunks.push(c));
+    child.on("error", () => finish(null, true));
+    child.on("close", (code) => {
+      if (code !== 0) {
+        finish(null, false);
+        return;
+      }
+      if (verifyOcrLangCache(lang) === "mismatch") {
+        _ocrIntegrityFailed = true;
+        quarantineOcrLangCache(lang);
+        finish(null, false);
+        return;
+      }
+      try {
+        const raw = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+        const parsed = raw;
+        const text = typeof parsed.text === "string" ? parsed.text.trim() : "";
+        const confidence = typeof parsed.confidence === "number" ? parsed.confidence : 0;
+        finish({ text, confidence }, false);
+      } catch {
+        finish(null, false);
+      }
+    });
+    try {
+      child.stdin?.end(input);
+    } catch {
+      finish(null, false);
+    }
+  });
+}
+function isTextHeavy(result, minConfidence, minChars = 40) {
+  return result.confidence >= minConfidence && result.text.length >= minChars;
+}
+function formatOcrSummary(result, subject, originalBytes) {
+  const kb = Math.round(originalBytes / 1024);
+  const summary = `token-goat OCR'd ${subject} instead of shrinking it: text-heavy image detected (${Math.round(result.confidence)}% confidence), extracted ${result.text.length} chars of text from ${kb}kb of pixels.`;
+  return `${summary}
+
+${fenceUntrustedOcrText(result.text)}`;
+}
+
+export {
+  SUPPORTED_OCR_LANGS,
+  getOcrLangSpec,
+  setOcrTimeoutForTesting,
+  getActiveOcrLangs,
+  resolveOcrLang,
+  OCR_LANG_FILE,
+  OCR_LANG_PATH,
+  OCR_LANG_SHA256,
+  verifyOcrLangCache,
+  quarantineOcrLangCache,
+  ocrIntegrityFailed,
+  ocrBlockedOffline,
+  isOcrEngineAvailable,
+  setTesseractEntryForTesting,
+  resetOcrStateForTesting,
+  buildChildScript,
+  ocrImage,
+  isTextHeavy,
+  formatOcrSummary
+};

package/dist/token-goat-chunk-EG3663UT.mjs

@@ -0,0 +1,226 @@
+import { createRequire as __cjsRequire } from 'node:module';
+const require = __cjsRequire(import.meta.url);
+import {
+  parse
+} from "./token-goat-chunk-3BTK54F3.mjs";
+import {
+  displaySafeText,
+  filtersFilteredToEmptyNotice
+} from "./token-goat-chunk-AMYCQJX4.mjs";
+import {
+  compileGuardedRegexCached
+} from "./token-goat-chunk-GMOUBOX4.mjs";
+import {
+  init_define_import_meta_env
+} from "./token-goat-chunk-A37V4PBF.mjs";
+
+// src/csv_query.ts
+init_define_import_meta_env();
+var DELIMITER_CANDIDATES = [",", "	", ";", "|"];
+var DELIMITER_SAMPLE_ROWS = 5;
+function detectDelimiter(content) {
+  const slice = content.slice(0, 1e4);
+  if (slice.trim() === "") return ",";
+  for (const cand of DELIMITER_CANDIDATES) {
+    let rows;
+    try {
+      rows = parse(slice, { columns: false, skip_empty_lines: true, trim: true, delimiter: cand, bom: true, relax_quotes: true, to: DELIMITER_SAMPLE_ROWS });
+    } catch {
+      continue;
+    }
+    if ((rows[0]?.length ?? 0) < 2) continue;
+    return cand;
+  }
+  return ",";
+}
+function parseRecords(content, opts) {
+  const delimiter = opts.delimiter ?? detectDelimiter(content);
+  if (opts.noHeader === true) {
+    const rows = parse(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true, relax_quotes: true });
+    return rows.map((row) => Object.fromEntries(row.map((cell, i) => [`col${i + 1}`, cell])));
+  }
+  const header = csvHeader(content, { ...opts, delimiter });
+  const dupes = header.filter((name, i) => name !== "" && header.indexOf(name) !== i);
+  if (dupes.length > 0) {
+    const unique = [...new Set(dupes)];
+    throw new Error(
+      `duplicate column ${unique.length === 1 ? "name" : "names"} in header: ${unique.join(", ")} \u2014 rename the duplicates or pass --no-header to address columns positionally as col1, col2, \u2026`
+    );
+  }
+  return parse(content, { columns: true, skip_empty_lines: true, trim: true, delimiter, bom: true, relax_column_count: true, relax_quotes: true });
+}
+function csvHeader(content, opts) {
+  if (opts.noHeader === true) return [];
+  const delimiter = opts.delimiter ?? detectDelimiter(content);
+  try {
+    const rows = parse(content, { columns: false, skip_empty_lines: true, trim: true, delimiter, bom: true, to: 1, relax_quotes: true });
+    return rows[0] ?? [];
+  } catch {
+    return [];
+  }
+}
+var WHERE_SPEC_RE = /^((?:[^=<>~!]|!(?!=)|~(?!=))+)(!=|~=|>=|<=|=|>|<)(.*)$/;
+function parseWhereSpecs(specs) {
+  if (specs === void 0 || specs.length === 0) return void 0;
+  return specs.map((spec) => {
+    const m = WHERE_SPEC_RE.exec(spec);
+    if (!m) throw new Error(`invalid --where spec: ${spec} (expected col=value, col!=value, col>value, col<value, col>=value, or col<=value, or col~=regex)`);
+    const op = m[2];
+    const value = m[3];
+    if ((op === ">" || op === "<" || op === ">=" || op === "<=") && value.trim() === "") {
+      throw new Error(`invalid --where spec: ${spec} (missing comparison value after '${op}')`);
+    }
+    if (op === "~=") {
+      const guarded = compileGuardedRegexCached(value);
+      if (!guarded.ok) throw new Error(`invalid --where spec: ${spec} (the pattern ${guarded.reason})`);
+    }
+    return { column: m[1].trim(), op, value };
+  });
+}
+function resolveWhereColumn(where, allColumns) {
+  const rawSpec = where.column + where.op + where.value;
+  let best = where;
+  for (const col of allColumns) {
+    if (col.length <= best.column.length) continue;
+    if (!rawSpec.startsWith(col)) continue;
+    const rest = rawSpec.slice(col.length);
+    const m = /^(!=|~=|>=|<=|=|>|<)(.*)$/.exec(rest);
+    if (m) {
+      best = { column: col, op: m[1], value: m[2] };
+    }
+  }
+  return best;
+}
+function matchesWhere(row, where) {
+  const cell = row[where.column] ?? "";
+  switch (where.op) {
+    case "=":
+      return cell === where.value;
+    case "!=":
+      return cell !== where.value;
+    case "~=": {
+      const guarded = compileGuardedRegexCached(where.value);
+      return guarded.ok && guarded.re.test(cell);
+    }
+    case ">":
+    case "<":
+    case ">=":
+    case "<=": {
+      if (cell.trim() === "") return false;
+      const cellNum = Number(cell);
+      const valNum = Number(where.value);
+      const useNum = !Number.isNaN(cellNum) && !Number.isNaN(valNum);
+      const lhs = useNum ? cellNum : cell;
+      const rhs = useNum ? valNum : where.value;
+      switch (where.op) {
+        case ">":
+          return lhs > rhs;
+        case "<":
+          return lhs < rhs;
+        case ">=":
+          return lhs >= rhs;
+        case "<=":
+          return lhs <= rhs;
+      }
+    }
+  }
+}
+function queryCsv(content, opts) {
+  const records = parseRecords(content, opts);
+  const allColumns = records.length > 0 ? Object.keys(records[0]) : [];
+  const realHeader = records.length > 0 ? allColumns : csvHeader(content, opts);
+  const columns = opts.columns && opts.columns.length > 0 ? opts.columns : allColumns;
+  for (const c of columns) {
+    if (!realHeader.includes(c)) {
+      throw new Error(`unknown column: ${c} (available: ${realHeader.join(", ")})`);
+    }
+  }
+  const wheres = (opts.wheres ?? []).map((w) => resolveWhereColumn(w, realHeader));
+  for (const w of wheres) {
+    if (!realHeader.includes(w.column)) {
+      throw new Error(`unknown column: ${w.column} (available: ${realHeader.join(", ")})`);
+    }
+  }
+  let filtered = records;
+  if (wheres.length > 0) {
+    filtered = records.filter((r) => wheres.every((w) => matchesWhere(r, w)));
+  }
+  const totalRows = filtered.length;
+  const limited = opts.head !== void 0 ? filtered.slice(0, opts.head) : filtered;
+  const rows = limited.map((r) => columns.map((c) => r[c] ?? ""));
+  return { header: columns, rows, totalRows, preFilterRows: records.length };
+}
+function quoteCsvCell(cell) {
+  if (cell.includes(",") || cell.includes('"') || cell.includes("\n") || cell.includes("\r")) {
+    return `"${cell.replace(/"/g, '""')}"`;
+  }
+  return cell;
+}
+function formatCsvTable(result, activeFilters = []) {
+  const lines = [
+    result.header.map(quoteCsvCell).join(","),
+    ...result.rows.map((r) => r.map(quoteCsvCell).join(","))
+  ];
+  if (result.totalRows > result.rows.length) {
+    lines.push(`...(${result.totalRows - result.rows.length} more rows elided; use --head to see more)`);
+  }
+  if (result.totalRows === 0 && result.preFilterRows > 0) {
+    lines.push(filtersFilteredToEmptyNotice(result.preFilterRows, activeFilters, "data row", "data rows"));
+  }
+  return lines.join("\n");
+}
+function profileCsv(content, opts = {}) {
+  const records = parseRecords(content, opts);
+  const columns = records.length > 0 ? Object.keys(records[0]) : [];
+  return columns.map((col) => {
+    const values = records.map((r) => r[col] ?? "");
+    const nullCount = values.filter((v) => v.trim() === "").length;
+    const nonEmpty = values.filter((v) => v.trim() !== "");
+    const distinct = new Set(nonEmpty);
+    const isNumber = nonEmpty.length > 0 && nonEmpty.every((v) => v.trim() !== "" && !Number.isNaN(Number(v)));
+    const isDate = !isNumber && nonEmpty.length > 0 && nonEmpty.every((v) => !Number.isNaN(Date.parse(v)));
+    const inferredType = isNumber ? "number" : isDate ? "date" : "string";
+    const profile = { name: col, inferredType, nullCount, distinctCount: distinct.size };
+    if (nonEmpty.length > 0) {
+      if (isNumber) {
+        const nums = nonEmpty.map(Number);
+        profile.min = String(nums.reduce((a, b) => Math.min(a, b)));
+        profile.max = String(nums.reduce((a, b) => Math.max(a, b)));
+      } else if (isDate) {
+        const sorted = [...nonEmpty].sort((a, b) => Date.parse(a) - Date.parse(b));
+        profile.min = sorted[0];
+        profile.max = sorted[sorted.length - 1];
+      } else {
+        const sorted = [...nonEmpty].sort();
+        profile.min = sorted[0];
+        profile.max = sorted[sorted.length - 1];
+      }
+    }
+    if (distinct.size > 0 && distinct.size <= 10) {
+      const counts = /* @__PURE__ */ new Map();
+      for (const v of nonEmpty) counts.set(v, (counts.get(v) ?? 0) + 1);
+      profile.topValues = [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([value, count]) => ({ value, count }));
+    }
+    return profile;
+  });
+}
+function formatCsvProfile(profiles) {
+  return profiles.map((p) => {
+    const lines = [
+      `${displaySafeText(p.name)}  (${p.inferredType})`,
+      `  nulls: ${p.nullCount}  distinct: ${p.distinctCount}`
+    ];
+    if (p.min !== void 0) lines.push(`  range: ${displaySafeText(p.min)} .. ${displaySafeText(p.max ?? "")}`);
+    if (p.topValues !== void 0) lines.push(`  values: ${p.topValues.map((t) => `${displaySafeText(t.value)} (${t.count})`).join(", ")}`);
+    return lines.join("\n");
+  }).join("\n\n");
+}
+
+export {
+  parseWhereSpecs,
+  queryCsv,
+  quoteCsvCell,
+  formatCsvTable,
+  profileCsv,
+  formatCsvProfile
+};

package/dist/token-goat-chunk-EZNVAIR3.mjs

@@ -0,0 +1,21 @@
+import { createRequire as __cjsRequire } from 'node:module';
+const require = __cjsRequire(import.meta.url);
+import {
+  docxOutline,
+  docxTables,
+  docxText,
+  formatDocxTables
+} from "./token-goat-chunk-4SDX3QP3.mjs";
+import "./token-goat-chunk-YHGTGG6K.mjs";
+import "./token-goat-chunk-XTQAOTSO.mjs";
+import "./token-goat-chunk-AH6QILZM.mjs";
+import "./token-goat-chunk-Y4AFKTHK.mjs";
+import "./token-goat-chunk-AMYCQJX4.mjs";
+import "./token-goat-chunk-GMOUBOX4.mjs";
+import "./token-goat-chunk-A37V4PBF.mjs";
+export {
+  docxOutline,
+  docxTables,
+  docxText,
+  formatDocxTables
+};