token-goat 2.9.11 → 2.9.13

package/dist/token-goat-chunk-YOA4N6WA.mjs

@@ -0,0 +1,308 @@
+import { createRequire as __cjsRequire } from 'node:module';
+const require = __cjsRequire(import.meta.url);
+import {
+  DEFAULT_OCR_LANG,
+  isSupportedOcrLang,
+  loadConfig,
+  resolveOcrLangs
+} from "./token-goat-chunk-UMXJN7DI.mjs";
+import {
+  ensureDirSync,
+  fenceUntrustedOcrText,
+  tokenGoatHome
+} from "./token-goat-chunk-QKXBGBQR.mjs";
+import {
+  init_define_import_meta_env
+} from "./token-goat-chunk-A37V4PBF.mjs";
+
+// src/image_ocr.ts
+init_define_import_meta_env();
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import * as fs from "node:fs";
+import { createRequire } from "node:module";
+import * as path from "node:path";
+
+// src/ocr_hashes.ts
+init_define_import_meta_env();
+function ocrLangPath(lang) {
+  return `https://cdn.jsdelivr.net/npm/@tesseract.js-data/${lang}@1.0.0/4.0.0_best_int`;
+}
+var OCR_LANG_HASHES = {
+  eng: "5dc5d8d640a212c9d6184921ba103b186f50e0fed9ee716c53e6b312b400d747",
+  fra: "bf83833fa957ff0076f6aa93f69e3bdf7b014dea829ea0c0d6be6b48a3ceef6d",
+  spa: "0062377729b81cc268b1822f09eb1c08c09f3f7f1c6b422540b51555a7eeea70",
+  deu: "a1b72cc25753eac167edfef5af4448a8dc34973503a2265c34c84520f896eb03",
+  ita: "36ec897f5f1f489b257801881286167a79b99a16040c6fbc7e3e30f03821b10b",
+  por: "42fab1f017aedab69b92bdecc01bbb11166cd3b177575612ee860f8e2825ece0",
+  nld: "363c360db9838838ff7ed3d8b885b33acc0d61d37165708303cf8a81e50164f3",
+  pol: "02b89cad819f1374631b4a3c92bdac79c214150f97a3686df78de6c8c30782db",
+  rus: "eb9be824435f6bb0f993925acb85fd842c8418d6db7613c818e749e619a1ad6d",
+  tur: "f0127d0f3745f9c65e2ae7ec6b23198fbe5aa186a61b35661426f5e1ef9dedd7",
+  swe: "a4c33cbd23d988c84b5f9d5d5b2417fc16da061e3dc7b16acb904a4566088b14",
+  ara: "e7d6494e2ef249ee97ad151eb01e0e6ae3aaf429256442ad6af534862a2a8c0f",
+  chi_sim: "9784f7c917c546424b690fcde708ce1f604a4393d08bb51ddab146d7d7c794e6",
+  chi_tra: "6abfb87cce5db0d09624f16eedd8a0b24173718856121f721b6d1214193d4dab",
+  jpn: "1a0175291ea145d4a66be681d1084496f10af938aacab247c5d40b31359a604e",
+  kor: "ec1749377d49ac38fb3d3cd05dd5e2a53359d329f359762bc638a81132109992"
+};
+var SUPPORTED_OCR_LANGS = Object.fromEntries(
+  Object.entries(OCR_LANG_HASHES).map(([lang, sha256]) => [lang, { langPath: ocrLangPath(lang), sha256 }])
+);
+function getOcrLangSpec(lang) {
+  const tokens = lang.split("+");
+  const primary = tokens.find((t) => t !== DEFAULT_OCR_LANG && isSupportedOcrLang(t)) ?? tokens[0] ?? DEFAULT_OCR_LANG;
+  const active = isSupportedOcrLang(primary) ? primary : DEFAULT_OCR_LANG;
+  const sha256 = OCR_LANG_HASHES[active];
+  return {
+    langPath: ocrLangPath(active),
+    sha256
+  };
+}
+
+// src/image_ocr.ts
+var _ocrTimeoutMs = 12e3;
+function setOcrTimeoutForTesting(ms) {
+  _ocrTimeoutMs = ms ?? 12e3;
+}
+function getActiveOcrLangs(lang) {
+  if (typeof lang === "string" && lang.trim()) {
+    return resolveOcrLangs(lang);
+  }
+  try {
+    const configured = loadConfig().image_shrink?.ocr_lang;
+    if (typeof configured === "string" && configured.trim()) {
+      return resolveOcrLangs(configured);
+    }
+  } catch {
+  }
+  return [DEFAULT_OCR_LANG];
+}
+function resolveOcrLang(lang) {
+  return getActiveOcrLangs(lang).join("+");
+}
+var OCR_LANG_FILE = `${DEFAULT_OCR_LANG}.traineddata`;
+var OCR_LANG_PATH = getOcrLangSpec("eng").langPath;
+var OCR_LANG_SHA256 = getOcrLangSpec("eng").sha256;
+function verifyOcrLangCache(lang) {
+  const activeLangs = getActiveOcrLangs(lang);
+  let anyAbsent = false;
+  for (const l of activeLangs) {
+    const spec = getOcrLangSpec(l);
+    const file = path.join(ocrCacheDir(), `${l}.traineddata`);
+    try {
+      if (!fs.existsSync(file)) {
+        anyAbsent = true;
+        continue;
+      }
+      const bytes = fs.readFileSync(file);
+      if (createHash("sha256").update(bytes).digest("hex") !== spec.sha256) {
+        return "mismatch";
+      }
+    } catch {
+      return "unreadable";
+    }
+  }
+  return anyAbsent ? "absent" : "ok";
+}
+function quarantineOcrLangCache(lang) {
+  const activeLangs = getActiveOcrLangs(lang);
+  for (const l of activeLangs) {
+    const spec = getOcrLangSpec(l);
+    const file = path.join(ocrCacheDir(), `${l}.traineddata`);
+    try {
+      if (fs.existsSync(file)) {
+        const bytes = fs.readFileSync(file);
+        if (createHash("sha256").update(bytes).digest("hex") !== spec.sha256) {
+          fs.rmSync(file, { force: true });
+        }
+      }
+    } catch {
+    }
+  }
+}
+var _ocrIntegrityFailed = false;
+function ocrIntegrityFailed() {
+  return _ocrIntegrityFailed;
+}
+function ocrBlockedOffline(lang) {
+  if (!loadConfig().network.offline) return false;
+  return verifyOcrLangCache(lang) === "absent";
+}
+function ocrCacheDir() {
+  return path.join(tokenGoatHome(), "ocr-cache");
+}
+function ensureOcrCacheDir() {
+  try {
+    ensureDirSync(ocrCacheDir());
+  } catch {
+  }
+}
+var _require = createRequire(import.meta.url);
+var _tesseractEntryPath;
+function resolveTesseractEntry() {
+  if (_tesseractEntryPath !== void 0) return _tesseractEntryPath;
+  try {
+    _tesseractEntryPath = _require.resolve("tesseract.js");
+  } catch {
+    _tesseractEntryPath = null;
+  }
+  return _tesseractEntryPath;
+}
+function isOcrEngineAvailable() {
+  return !_ocrUnavailableThisProcess && resolveTesseractEntry() !== null;
+}
+function setTesseractEntryForTesting(entryPath) {
+  _tesseractEntryPath = entryPath;
+}
+var _ocrUnavailableThisProcess = false;
+function resetOcrStateForTesting() {
+  _ocrUnavailableThisProcess = false;
+  _ocrIntegrityFailed = false;
+  _tesseractEntryPath = void 0;
+  _ocrTimeoutMs = 12e3;
+}
+function buildChildScript(entryPath, cacheDir, lang) {
+  const activeLangs = getActiveOcrLangs(lang);
+  const activeLangString = activeLangs.join("+");
+  const spec = getOcrLangSpec(lang ?? activeLangString);
+  return [
+    `const { createWorker } = require(${JSON.stringify(entryPath)});`,
+    "const chunks = [];",
+    "process.stdin.on('data', (c) => chunks.push(c));",
+    "process.stdin.on('end', async () => {",
+    "  try {",
+    "    const buf = Buffer.concat(chunks);",
+    `    const worker = await createWorker(${JSON.stringify(activeLangString)}, 1, { cachePath: ${JSON.stringify(cacheDir)}, langPath: ${JSON.stringify(spec.langPath)}, errorHandler: () => {} });`,
+    "    const { data } = await worker.recognize(buf);",
+    "    process.stdout.write(JSON.stringify({ text: data.text || '', confidence: data.confidence || 0 }));",
+    "    await worker.terminate();",
+    "    process.exit(0);",
+    "  } catch (e) {",
+    "    process.exit(1);",
+    "  }",
+    "});"
+  ].join("\n");
+}
+async function ocrImage(input, lang) {
+  if (_ocrUnavailableThisProcess) return null;
+  const activeLangs = getActiveOcrLangs(lang);
+  const entryPath = resolveTesseractEntry();
+  if (entryPath === null) return null;
+  if (ocrBlockedOffline(lang)) return null;
+  if (_ocrIntegrityFailed) return null;
+  if (verifyOcrLangCache(lang) === "mismatch") {
+    _ocrIntegrityFailed = true;
+    quarantineOcrLangCache(lang);
+    return null;
+  }
+  for (const l of activeLangs) {
+    if (l !== DEFAULT_OCR_LANG) {
+      const file = path.join(ocrCacheDir(), `${l}.traineddata`);
+      if (!fs.existsSync(file)) {
+        if (loadConfig().network.offline) return null;
+        try {
+          const res = await fetch(getOcrLangSpec(l).langPath + `/${l}.traineddata.gz`);
+          if (!res.ok) return null;
+          const gz = Buffer.from(await res.arrayBuffer());
+          const unzipped = (await import("node:zlib")).gunzipSync(gz);
+          if (createHash("sha256").update(unzipped).digest("hex") !== getOcrLangSpec(l).sha256) {
+            _ocrIntegrityFailed = true;
+            return null;
+          }
+          ensureOcrCacheDir();
+          fs.writeFileSync(file, unzipped);
+        } catch {
+          return null;
+        }
+      }
+    }
+  }
+  return new Promise((resolve) => {
+    let settled = false;
+    let child;
+    try {
+      ensureOcrCacheDir();
+      child = spawn(process.execPath, ["-e", buildChildScript(entryPath, ocrCacheDir(), lang)], {
+        stdio: ["pipe", "pipe", "ignore"]
+      });
+    } catch {
+      _ocrUnavailableThisProcess = true;
+      resolve(null);
+      return;
+    }
+    const chunks = [];
+    const finish = (result, subprocessBroken) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (subprocessBroken) _ocrUnavailableThisProcess = true;
+      try {
+        child.kill();
+      } catch {
+      }
+      resolve(result);
+    };
+    const timer = setTimeout(() => finish(null, true), _ocrTimeoutMs);
+    child.stdout?.on("data", (c) => chunks.push(c));
+    child.on("error", () => finish(null, true));
+    child.on("close", (code) => {
+      if (code !== 0) {
+        finish(null, false);
+        return;
+      }
+      if (verifyOcrLangCache(lang) === "mismatch") {
+        _ocrIntegrityFailed = true;
+        quarantineOcrLangCache(lang);
+        finish(null, false);
+        return;
+      }
+      try {
+        const raw = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+        const parsed = raw;
+        const text = typeof parsed.text === "string" ? parsed.text.trim() : "";
+        const confidence = typeof parsed.confidence === "number" ? parsed.confidence : 0;
+        finish({ text, confidence }, false);
+      } catch {
+        finish(null, false);
+      }
+    });
+    try {
+      child.stdin?.end(input);
+    } catch {
+      finish(null, false);
+    }
+  });
+}
+function isTextHeavy(result, minConfidence, minChars = 40) {
+  return result.confidence >= minConfidence && result.text.length >= minChars;
+}
+function formatOcrSummary(result, subject, originalBytes) {
+  const kb = Math.round(originalBytes / 1024);
+  const summary = `token-goat OCR'd ${subject} instead of shrinking it: text-heavy image detected (${Math.round(result.confidence)}% confidence), extracted ${result.text.length} chars of text from ${kb}kb of pixels.`;
+  return `${summary}
+
+${fenceUntrustedOcrText(result.text)}`;
+}
+
+export {
+  SUPPORTED_OCR_LANGS,
+  getOcrLangSpec,
+  setOcrTimeoutForTesting,
+  getActiveOcrLangs,
+  resolveOcrLang,
+  OCR_LANG_FILE,
+  OCR_LANG_PATH,
+  OCR_LANG_SHA256,
+  verifyOcrLangCache,
+  quarantineOcrLangCache,
+  ocrIntegrityFailed,
+  ocrBlockedOffline,
+  isOcrEngineAvailable,
+  setTesseractEntryForTesting,
+  resetOcrStateForTesting,
+  buildChildScript,
+  ocrImage,
+  isTextHeavy,
+  formatOcrSummary
+};