tryharness 0.7.23 → 0.7.26

package/lib/ocr.mjs

@@ -14,24 +14,54 @@ import jpeg from "jpeg-js";
 export const MAX_PIXELS = 40e6;
 // The detector's own settings (its inference.yml: thresh 0.2, box_thresh 0.45, unclip_ratio 1.4) at a long side of
 // 960 to 2,048 pixels: a retina screenshot is shrunk to 2,048 across, a small image is enlarged.
-const DET = { min: 960, max: 2_048, thresh: 0.2, boxThresh: 0.45, unclip: 1.4, mean: [0.485, 0.456, 0.406], std: [0.229, 0.224, 0.225] };
+const DET = {
+  min: 960,
+  max: 2_048,
+  thresh: 0.2,
+  boxThresh: 0.45,
+  unclip: 1.4,
+  mean: [0.485, 0.456, 0.406],
+  std: [0.229, 0.224, 0.225],
+};
 const REC = { height: 48, mean: [0.5, 0.5, 0.5], std: [0.5, 0.5, 0.5] };
 // Boxes read per scheduler job, so a long image does not hold the ONNX lane from the firewall's checks.
 const REC_PER_JOB = 24;
 
-export const imageKind = (bytes) => bytes?.length > 8 && bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47 ? "png"
-  : bytes?.length > 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff ? "jpeg" : null;
+export const imageKind = (bytes) =>
+  bytes?.length > 8 &&
+  bytes[0] === 0x89 &&
+  bytes[1] === 0x50 &&
+  bytes[2] === 0x4e &&
+  bytes[3] === 0x47
+    ? "png"
+    : bytes?.length > 3 &&
+        bytes[0] === 0xff &&
+        bytes[1] === 0xd8 &&
+        bytes[2] === 0xff
+      ? "jpeg"
+      : null;
 
 /** { width, height, data: RGBA } from PNG or JPEG bytes; null for any other format. Throws on a broken or oversized image. */
 export function decodeImage(bytes) {
-  const buffer = Buffer.from(bytes.buffer ?? bytes, bytes.byteOffset ?? 0, bytes.byteLength ?? bytes.length);
+  const buffer = Buffer.from(
+    bytes.buffer ?? bytes,
+    bytes.byteOffset ?? 0,
+    bytes.byteLength ?? bytes.length
+  );
   const kind = imageKind(buffer);
   if (kind === "png") {
-    if (buffer.readUInt32BE(16) * buffer.readUInt32BE(20) > MAX_PIXELS) throw new Error("the image is too large to check");
+    if (buffer.readUInt32BE(16) * buffer.readUInt32BE(20) > MAX_PIXELS)
+      throw new Error("the image is too large to check");
     const { width, height, data } = pngjs.PNG.sync.read(buffer);
     return { width, height, data };
   }
-  if (kind === "jpeg") return jpeg.decode(buffer, { useTArray: true, formatAsRGBA: true, maxResolutionInMP: MAX_PIXELS / 1e6, maxMemoryUsageInMB: 1_024 });
+  if (kind === "jpeg")
+    return jpeg.decode(buffer, {
+      useTArray: true,
+      formatAsRGBA: true,
+      maxResolutionInMP: MAX_PIXELS / 1e6,
+      maxMemoryUsageInMB: 1_024,
+    });
   return null;
 }
 
@@ -39,13 +69,22 @@ export function decodeImage(bytes) {
 export function characterList(yml) {
   const lines = yml.split(/\r?\n/);
   const out = [];
-  for (const line of lines.slice(lines.findIndex((each) => /^\s*character_dict:\s*$/.test(each)) + 1)) {
+  for (const line of lines.slice(
+    lines.findIndex((each) => /^\s*character_dict:\s*$/.test(each)) + 1
+  )) {
     const item = line.match(/^\s*- (.*)$/);
     if (!item) break;
     const value = item[1];
-    out.push(value.startsWith("'") ? value.slice(1, -1).replace(/''/g, "'") : value.startsWith('"') ? JSON.parse(value) : value);
+    out.push(
+      value.startsWith("'")
+        ? value.slice(1, -1).replace(/''/g, "'")
+        : value.startsWith('"')
+          ? JSON.parse(value)
+          : value
+    );
   }
-  if (!out.length) throw new Error("the recogniser's character list is missing");
+  if (!out.length)
+    throw new Error("the recogniser's character list is missing");
   return out;
 }
 
@@ -57,10 +96,16 @@ function taps(start, length, count, limit) {
   for (let i = 0; i < count; i += 1) {
     const list = [];
     if (scale > 1) {
-      const a = start + i * scale, b = a + scale;
-      for (let p = Math.floor(a); p < b; p += 1) { const w = Math.min(b, p + 1) - Math.max(a, p); if (w > 0) list.push(at(p), w / scale); }
+      const a = start + i * scale,
+        b = a + scale;
+      for (let p = Math.floor(a); p < b; p += 1) {
+        const w = Math.min(b, p + 1) - Math.max(a, p);
+        if (w > 0) list.push(at(p), w / scale);
+      }
     } else {
-      const c = start + (i + 0.5) * scale - 0.5, p = Math.floor(c), f = c - p;
+      const c = start + (i + 0.5) * scale - 0.5,
+        p = Math.floor(c),
+        f = c - p;
       list.push(at(p), 1 - f, at(p + 1), f);
     }
     out[i] = list;
@@ -70,18 +115,30 @@ function taps(start, length, count, limit) {
 
 /** A region of the image resampled to width x height as a normalized BGR tensor (PaddleOCR's models read OpenCV's BGR); transparency is laid on white. */
 function tensorOf(image, [x0, y0, x1, y1], width, height, { mean, std }) {
-  const xs = taps(x0, x1 - x0, width, image.width), ys = taps(y0, y1 - y0, height, image.height);
-  const plane = width * height, out = new Float32Array(3 * plane), data = image.data, stride = image.width * 4;
+  const xs = taps(x0, x1 - x0, width, image.width),
+    ys = taps(y0, y1 - y0, height, image.height);
+  const plane = width * height,
+    out = new Float32Array(3 * plane),
+    data = image.data,
+    stride = image.width * 4;
   for (let j = 0; j < height; j += 1) {
     const ty = ys[j];
     for (let i = 0; i < width; i += 1) {
       const tx = xs[i];
-      let r = 0, g = 0, b = 0;
+      let r = 0,
+        g = 0,
+        b = 0;
       for (let v = 0; v < ty.length; v += 2) {
-        const row = ty[v] * stride, wy = ty[v + 1];
+        const row = ty[v] * stride,
+          wy = ty[v + 1];
         for (let u = 0; u < tx.length; u += 2) {
-          const p = row + tx[u] * 4, w = wy * tx[u + 1], a = data[p + 3] / 255, white = 255 * (1 - a);
-          r += (data[p] * a + white) * w; g += (data[p + 1] * a + white) * w; b += (data[p + 2] * a + white) * w;
+          const p = row + tx[u] * 4,
+            w = wy * tx[u + 1],
+            a = data[p + 3] / 255,
+            white = 255 * (1 - a);
+          r += (data[p] * a + white) * w;
+          g += (data[p + 1] * a + white) * w;
+          b += (data[p + 2] * a + white) * w;
         }
       }
       const k = j * width + i;
@@ -96,23 +153,43 @@ function tensorOf(image, [x0, y0, x1, y1], width, height, { mean, std }) {
 // DB post-processing: the regions above the threshold whose mean probability passes the box threshold, each grown by
 // area x unclip / perimeter on every side (the detector is trained on text shrunk by that much).
 function regions(map, width, height) {
-  const seen = new Uint8Array(width * height), stack = new Int32Array(width * height), boxes = [];
+  const seen = new Uint8Array(width * height),
+    stack = new Int32Array(width * height),
+    boxes = [];
   for (let start = 0; start < map.length; start += 1) {
     if (seen[start] || map[start] <= DET.thresh) continue;
     seen[start] = 1;
-    let top = 0, x0 = width, y0 = height, x1 = -1, y1 = -1, sum = 0, count = 0;
+    let top = 0,
+      x0 = width,
+      y0 = height,
+      x1 = -1,
+      y1 = -1,
+      sum = 0,
+      count = 0;
     stack[top++] = start;
-    const visit = (q) => { if (!seen[q] && map[q] > DET.thresh) { seen[q] = 1; stack[top++] = q; } };
+    const visit = (q) => {
+      if (!seen[q] && map[q] > DET.thresh) {
+        seen[q] = 1;
+        stack[top++] = q;
+      }
+    };
     while (top) {
-      const p = stack[--top], x = p % width, y = (p - x) / width;
-      sum += map[p]; count += 1;
-      if (x < x0) x0 = x; if (x > x1) x1 = x; if (y < y0) y0 = y; if (y > y1) y1 = y;
+      const p = stack[--top],
+        x = p % width,
+        y = (p - x) / width;
+      sum += map[p];
+      count += 1;
+      if (x < x0) x0 = x;
+      if (x > x1) x1 = x;
+      if (y < y0) y0 = y;
+      if (y > y1) y1 = y;
       if (x > 0) visit(p - 1);
       if (x < width - 1) visit(p + 1);
       if (y > 0) visit(p - width);
       if (y < height - 1) visit(p + width);
     }
-    const w = x1 - x0 + 1, h = y1 - y0 + 1;
+    const w = x1 - x0 + 1,
+      h = y1 - y0 + 1;
     if (Math.min(w, h) < 3 || sum / count < DET.boxThresh) continue;
     const d = (w * h * DET.unclip) / (2 * (w + h));
     boxes.push([x0 - d, y0 - d, x1 + 1 + d, y1 + 1 + d]);
@@ -126,37 +203,70 @@ function regions(map, width, height) {
 // the probability map is stitched from rows at least MARGIN from a band's edge. Lines are never cut: regions are found
 // on the whole map. On the image-redaction eval's screens this found what one pass did, and the OCR models' memory
 // levelled off at 544 MB instead of growing past 2 GB (160-row bands; 256 rows: 766 MB and about 10% faster).
-const BAND = 160, MARGIN = 32, WINDOW = BAND + 2 * MARGIN;
+const BAND = 160,
+  MARGIN = 32,
+  WINDOW = BAND + 2 * MARGIN;
 
 async function detect(image, { ort, graph }) {
   const long = Math.max(image.width, image.height);
   const ratio = Math.min(DET.max, Math.max(DET.min, long)) / long;
-  const width = Math.max(32, Math.round((image.width * ratio) / 32) * 32), height = Math.max(32, Math.round((image.height * ratio) / 32) * 32);
+  const width = Math.max(32, Math.round((image.width * ratio) / 32) * 32),
+    height = Math.max(32, Math.round((image.height * ratio) / 32) * 32);
   const columns = Math.ceil(width / 512) * 512;
   const session = await graph("det/inference.onnx");
-  const sx = image.width / width, sy = image.height / height;
+  const sx = image.width / width,
+    sy = image.height / height;
   const map = new Float32Array(width * height);
   for (let top = 0; top < height; top += BAND) {
-    const a = Math.max(0, Math.min(top - MARGIN, height - WINDOW)), rows = Math.min(WINDOW, height - a);
-    const pixels = tensorOf(image, [0, a * sy, image.width, (a + rows) * sy], width, rows, DET), input = new Float32Array(3 * WINDOW * columns);
+    const a = Math.max(0, Math.min(top - MARGIN, height - WINDOW)),
+      rows = Math.min(WINDOW, height - a);
+    const pixels = tensorOf(
+        image,
+        [0, a * sy, image.width, (a + rows) * sy],
+        width,
+        rows,
+        DET
+      ),
+      input = new Float32Array(3 * WINDOW * columns);
     // Padding repeats the edge pixels: a flat block beside the image made the detector see text in empty space.
-    for (let c = 0; c < 3; c += 1) for (let y = 0; y < WINDOW; y += 1) {
-      const from = (c * rows + Math.min(y, rows - 1)) * width, to = (c * WINDOW + y) * columns;
-      input.set(pixels.subarray(from, from + width), to);
-      input.fill(pixels[from + width - 1], to + width, to + columns);
-    }
-    const band = (await session.run({ [session.inputNames[0]]: new ort.Tensor("float32", input, [1, 3, WINDOW, columns]) }))[session.outputNames[0]].data;
-    for (let y = top; y < Math.min(height, top + BAND); y += 1) map.set(band.subarray((y - a) * columns, (y - a) * columns + width), y * width);
+    for (let c = 0; c < 3; c += 1)
+      for (let y = 0; y < WINDOW; y += 1) {
+        const from = (c * rows + Math.min(y, rows - 1)) * width,
+          to = (c * WINDOW + y) * columns;
+        input.set(pixels.subarray(from, from + width), to);
+        input.fill(pixels[from + width - 1], to + width, to + columns);
+      }
+    const band = (
+      await session.run({
+        [session.inputNames[0]]: new ort.Tensor("float32", input, [
+          1,
+          3,
+          WINDOW,
+          columns,
+        ]),
+      })
+    )[session.outputNames[0]].data;
+    for (let y = top; y < Math.min(height, top + BAND); y += 1)
+      map.set(
+        band.subarray((y - a) * columns, (y - a) * columns + width),
+        y * width
+      );
   }
   return regions(map, width, height)
-    .map(([x0, y0, x1, y1]) => [Math.max(0, x0 * sx), Math.max(0, y0 * sy), Math.min(image.width, x1 * sx), Math.min(image.height, y1 * sy)])
+    .map(([x0, y0, x1, y1]) => [
+      Math.max(0, x0 * sx),
+      Math.max(0, y0 * sy),
+      Math.min(image.width, x1 * sx),
+      Math.min(image.height, y1 * sy),
+    ])
     .filter(([x0, y0, x1, y1]) => x1 - x0 >= 4 && y1 - y0 >= 4);
 }
 
 // ONNX Runtime keeps memory for every input shape it has run (400 crop widths: 108 MB grew to 513 MB, October 14), so
 // crops are padded on the right to a few widths, with 0 after normalization as PaddleOCR's own batches pad. Over this
 // eval's screens that changed 16 of 845 lines, by spacing or a dash, none of a planted value.
-const bucket = (width) => (width <= 512 ? Math.ceil(width / 32) * 32 : Math.ceil(width / 64) * 64);
+const bucket = (width) =>
+  width <= 512 ? Math.ceil(width / 32) * 32 : Math.ceil(width / 64) * 64;
 
 // CTC greedy decoding that keeps, for each character, the output steps it was read over (each step is a fixed slice of
 // the crop's width).
@@ -164,32 +274,66 @@ async function recognise(image, box, characters, { ort, graph }) {
   const session = await graph("rec/inference.onnx");
   const [x0, y0, x1, y1] = box;
   const width = Math.max(8, Math.round((REC.height * (x1 - x0)) / (y1 - y0)));
-  const padded = bucket(width), crop = tensorOf(image, box, width, REC.height, REC), tensor = new Float32Array(3 * REC.height * padded);
-  for (let row = 0; row < 3 * REC.height; row += 1) tensor.set(crop.subarray(row * width, (row + 1) * width), row * padded);
+  const padded = bucket(width),
+    crop = tensorOf(image, box, width, REC.height, REC),
+    tensor = new Float32Array(3 * REC.height * padded);
+  for (let row = 0; row < 3 * REC.height; row += 1)
+    tensor.set(crop.subarray(row * width, (row + 1) * width), row * padded);
   const input = new ort.Tensor("float32", tensor, [1, 3, REC.height, padded]);
-  const out = (await session.run({ [session.inputNames[0]]: input }))[session.outputNames[0]];
-  const [, all, classes] = out.dims, data = out.data;
+  const out = (await session.run({ [session.inputNames[0]]: input }))[
+    session.outputNames[0]
+  ];
+  const [, all, classes] = out.dims,
+    data = out.data;
   const steps = Math.min(all, Math.ceil((all * width) / padded));
   const space = classes === characters.length + 2 ? classes - 1 : -1;
   const pixels = ((x1 - x0) / width) * (padded / all);
   const read = [];
-  let total = 0, previous = 0;
+  let total = 0,
+    previous = 0;
   for (let t = 0; t < steps; t += 1) {
-    let best = 0, p = data[t * classes];
-    for (let c = 1; c < classes; c += 1) if (data[t * classes + c] > p) { p = data[t * classes + c]; best = c; }
-    if (best && best === previous) read[read.length - 1].to = x0 + (t + 1) * pixels;
-    else if (best) { read.push({ ch: best === space ? " " : characters[best - 1] ?? "", from: x0 + t * pixels, to: x0 + (t + 1) * pixels }); total += p; }
+    let best = 0,
+      p = data[t * classes];
+    for (let c = 1; c < classes; c += 1)
+      if (data[t * classes + c] > p) {
+        p = data[t * classes + c];
+        best = c;
+      }
+    if (best && best === previous)
+      read[read.length - 1].to = x0 + (t + 1) * pixels;
+    else if (best) {
+      read.push({
+        ch: best === space ? " " : (characters[best - 1] ?? ""),
+        from: x0 + t * pixels,
+        to: x0 + (t + 1) * pixels,
+      });
+      total += p;
+    }
     previous = best;
   }
   // One entry per UTF-16 unit, so chars lines up with the text's indexes.
   const text = read.map((each) => each.ch).join("");
-  const chars = read.flatMap(({ ch, from, to }) => Array.from({ length: ch.length }, () => [Math.floor(from), Math.ceil(to)]));
-  const lead = text.length - text.trimStart().length, kept = text.trim().length;
-  return { text: text.trim(), box: [Math.floor(x0), Math.floor(y0), Math.ceil(x1), Math.ceil(y1)], score: read.length ? total / read.length : 0, chars: chars.slice(lead, lead + kept) };
+  const chars = read.flatMap(({ ch, from, to }) =>
+    Array.from({ length: ch.length }, () => [Math.floor(from), Math.ceil(to)])
+  );
+  const lead = text.length - text.trimStart().length,
+    kept = text.trim().length;
+  return {
+    text: text.trim(),
+    box: [Math.floor(x0), Math.floor(y0), Math.ceil(x1), Math.ceil(y1)],
+    score: read.length ? total / read.length : 0,
+    chars: chars.slice(lead, lead + kept),
+  };
 }
 
 // Top to bottom, left to right within a row (rows: boxes whose tops are within a quarter of their combined height).
-const readingOrder = (lines) => lines.sort((a, b) => (Math.abs(a.box[1] - b.box[1]) < (a.box[3] - a.box[1] + b.box[3] - b.box[1]) / 4 ? a.box[0] - b.box[0] : a.box[1] - b.box[1]));
+const readingOrder = (lines) =>
+  lines.sort((a, b) =>
+    Math.abs(a.box[1] - b.box[1]) <
+    (a.box[3] - a.box[1] + b.box[3] - b.box[1]) / 4
+      ? a.box[0] - b.box[0]
+      : a.box[1] - b.box[1]
+  );
 
 export function createOcr({ inference }) {
   let characters = null;
@@ -201,17 +345,36 @@ export function createOcr({ inference }) {
     const started = performance.now();
     const image = input?.data && input.width ? input : decodeImage(input);
     if (!image) throw new Error("only PNG and JPEG images can be read");
-    const boxes = await inference.onnx("ocr", (model) => detect(image, model), { lane });
+    const boxes = await inference.onnx("ocr", (model) => detect(image, model), {
+      lane,
+    });
     const lines = [];
     for (let i = 0; i < boxes.length; i += REC_PER_JOB) {
-      lines.push(...await inference.onnx("ocr", async (model) => {
-        characters ||= characterList(fs.readFileSync(path.join(model.dir, "rec", "inference.yml"), "utf8"));
-        const read = [];
-        for (const box of boxes.slice(i, i + REC_PER_JOB)) read.push(await recognise(image, box, characters, model));
-        return read;
-      }, { lane }));
+      lines.push(
+        ...(await inference.onnx(
+          "ocr",
+          async (model) => {
+            characters ||= characterList(
+              fs.readFileSync(
+                path.join(model.dir, "rec", "inference.yml"),
+                "utf8"
+              )
+            );
+            const read = [];
+            for (const box of boxes.slice(i, i + REC_PER_JOB))
+              read.push(await recognise(image, box, characters, model));
+            return read;
+          },
+          { lane }
+        ))
+      );
     }
-    return { width: image.width, height: image.height, lines: readingOrder(lines.filter((line) => line.text)), ms: Math.round(performance.now() - started) };
+    return {
+      width: image.width,
+      height: image.height,
+      lines: readingOrder(lines.filter((line) => line.text)),
+      ms: Math.round(performance.now() - started),
+    };
   };
   return { read };
 }