thai-ocr-web 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/LICENSE +177 -0
- package/NOTICE +20 -0
- package/README.md +259 -2
- package/models/NOTICE +11 -0
- package/models/config.json +21 -0
- package/models/det.onnx +0 -0
- package/models/det.yml +52 -0
- package/models/rec.onnx +0 -0
- package/models/rec2.onnx +0 -0
- package/models/rec_dict.txt +528 -0
- package/package.json +84 -4
- package/src/browser.d.ts +27 -0
- package/src/browser.js +132 -0
- package/src/index.d.ts +68 -0
- package/src/index.js +5 -0
- package/src/line-ocr.js +1108 -0
- package/src/pdf.d.ts +43 -0
- package/src/pdf.js +112 -0
- package/src/thai-norm.js +52 -0
- package/vendor/opencv/opencv-ocr.mjs +0 -0
package/src/pdf.d.ts
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { ImageDataLike, OcrResult } from "./index.js";
|
|
2
|
+
|
|
3
|
+
export interface PdfPage {
|
|
4
|
+
pageNumber: number;
|
|
5
|
+
pageCount: number;
|
|
6
|
+
/** "text-layer": the page's own text; "ocr": rendered and read with OCR */
|
|
7
|
+
source: "text-layer" | "ocr";
|
|
8
|
+
text: string;
|
|
9
|
+
/** present for OCR pages */
|
|
10
|
+
result?: OcrResult;
|
|
11
|
+
/** the rendered page, for OCR pages */
|
|
12
|
+
image?: ImageData;
|
|
13
|
+
width: number;
|
|
14
|
+
height: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface ReadPdfOptions {
|
|
18
|
+
/** OCR for one page image, e.g. (img) => ocr.read(img) */
|
|
19
|
+
read(image: ImageDataLike): Promise<{ text: string } & Partial<OcrResult>>;
|
|
20
|
+
/** URL of pdfjs-dist/build/pdf.worker.min.mjs */
|
|
21
|
+
workerSrc?: string;
|
|
22
|
+
/** a pdfjs-dist module, if you load it yourself */
|
|
23
|
+
pdfjs?: unknown;
|
|
24
|
+
/** passed to pdfjs.getDocument (cMapUrl, standardFontDataUrl, wasmUrl, …) */
|
|
25
|
+
documentOptions?: Record<string, unknown>;
|
|
26
|
+
/** take a page's own text when it has some (default true) */
|
|
27
|
+
useTextLayer?: boolean;
|
|
28
|
+
/** non-space characters a page needs to count as text (default 20) */
|
|
29
|
+
minTextChars?: number;
|
|
30
|
+
/** share of unmapped / broken characters above which a text layer is not trusted and the page is OCR'd (default 0.01) */
|
|
31
|
+
maxBrokenShare?: number;
|
|
32
|
+
/** also render text-layer pages (for display); they are still not OCR'd (default false) */
|
|
33
|
+
renderTextPages?: boolean;
|
|
34
|
+
/** called before each page is processed */
|
|
35
|
+
onPage?: (p: { pageNumber: number; pageCount: number }) => void;
|
|
36
|
+
/** pixels on the long side for rendering scanned pages (default 2200) */
|
|
37
|
+
targetLongSide?: number;
|
|
38
|
+
maxPages?: number;
|
|
39
|
+
signal?: AbortSignal;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export declare function readPdf(source: Blob | ArrayBuffer | Uint8Array, options: ReadPdfOptions): AsyncGenerator<PdfPage>;
|
|
43
|
+
export declare function isPdf(file: Blob & { name?: string }): Promise<boolean>;
|
package/src/pdf.js
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
// PDF input: read every page of a PDF with pdf.js (pdfjs-dist, an optional peer dependency).
|
|
2
|
+
//
|
|
3
|
+
// A page that already carries text (a PDF exported from Word, a web page saved as PDF) gives that text directly:
|
|
4
|
+
// exact and instant. A page without text (a scan, a photo) is rendered to an image and read with OCR.
|
|
5
|
+
//
|
|
6
|
+
// import { readPdf } from "thai-ocr-web/pdf";
|
|
7
|
+
// for await (const page of readPdf(file, { read: (image) => ocr.read(image), workerSrc })) {
|
|
8
|
+
// console.log(page.pageNumber, page.source, page.text);
|
|
9
|
+
// }
|
|
10
|
+
import { canonical } from "./thai-norm.js";
|
|
11
|
+
import { invalidShare } from "./line-ocr.js";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @param {Blob|ArrayBuffer|Uint8Array} source the PDF
|
|
15
|
+
* @param {object} o
|
|
16
|
+
* @param {(image: ImageData) => Promise<{text: string}>} o.read OCR for one page image, e.g. (img) => ocr.read(img)
|
|
17
|
+
* @param {string} [o.workerSrc] URL of pdfjs-dist/build/pdf.worker.min.mjs (pdf.js needs its worker)
|
|
18
|
+
* @param {object} [o.pdfjs] a pdfjs-dist module, if you load it yourself; otherwise it is imported here
|
|
19
|
+
* @param {object} [o.documentOptions] passed to pdfjs.getDocument (cMapUrl, standardFontDataUrl, wasmUrl, …)
|
|
20
|
+
* @param {boolean} [o.useTextLayer=true] take a page's own text when it has some; false = OCR every page
|
|
21
|
+
* @param {number} [o.minTextChars=20] a page needs at least this many non-space characters to count as text
|
|
22
|
+
* @param {number} [o.maxBrokenShare=0.01] a text layer with more unmapped characters (NUL, U+FFFD — Thai marks
|
|
23
|
+
* drawn with font-specific glyphs often lose their Unicode) or broken Thai spelling than this share is not
|
|
24
|
+
* trusted, and the page is read with OCR instead
|
|
25
|
+
* @param {number} [o.targetLongSide=2200] render scanned pages at this many pixels on the long side (~190 dpi on A4)
|
|
26
|
+
* @param {boolean} [o.renderTextPages=false] also render pages that have a text layer (to show them); they still
|
|
27
|
+
* are not OCR'd
|
|
28
|
+
* @param {(p: {pageNumber: number, pageCount: number}) => void} [o.onPage] called before each page is processed
|
|
29
|
+
* @param {number} [o.maxPages=Infinity]
|
|
30
|
+
* @param {AbortSignal} [o.signal]
|
|
31
|
+
* @yields {{pageNumber, pageCount, source: "text-layer"|"ocr", text, result?, image?, width, height}}
|
|
32
|
+
*/
|
|
33
|
+
export async function* readPdf(source, {
|
|
34
|
+
read, workerSrc, pdfjs, documentOptions = {}, useTextLayer = true, minTextChars = 20, maxBrokenShare = 0.01,
|
|
35
|
+
targetLongSide = 2200, maxPages = Infinity, renderTextPages = false, onPage, signal,
|
|
36
|
+
} = {}) {
|
|
37
|
+
if (typeof read !== "function") throw new Error("thai-ocr-web/pdf: pass { read: (imageData) => ocr.read(imageData) }");
|
|
38
|
+
const lib = pdfjs || (await import("pdfjs-dist"));
|
|
39
|
+
if (workerSrc) lib.GlobalWorkerOptions.workerSrc = workerSrc;
|
|
40
|
+
const data = source instanceof Uint8Array ? source
|
|
41
|
+
: new Uint8Array(source instanceof ArrayBuffer ? source : await source.arrayBuffer());
|
|
42
|
+
const task = lib.getDocument({ data, ...documentOptions });
|
|
43
|
+
const doc = await task.promise;
|
|
44
|
+
try {
|
|
45
|
+
const pageCount = doc.numPages;
|
|
46
|
+
for (let n = 1; n <= Math.min(pageCount, maxPages); n++) {
|
|
47
|
+
if (signal?.aborted) return;
|
|
48
|
+
onPage?.({ pageNumber: n, pageCount });
|
|
49
|
+
const page = await doc.getPage(n);
|
|
50
|
+
try {
|
|
51
|
+
const base = page.getViewport({ scale: 1 });
|
|
52
|
+
if (useTextLayer) {
|
|
53
|
+
const text = textOf(await page.getTextContent());
|
|
54
|
+
const chars = text.replace(/\s/g, "").length;
|
|
55
|
+
if (chars >= minTextChars && brokenShare(text) <= maxBrokenShare) {
|
|
56
|
+
const image = renderTextPages ? await render(page, base, targetLongSide) : undefined;
|
|
57
|
+
yield { pageNumber: n, pageCount, source: "text-layer", text, image,
|
|
58
|
+
width: image?.width ?? base.width, height: image?.height ?? base.height };
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (signal?.aborted) return;
|
|
63
|
+
const image = await render(page, base, targetLongSide);
|
|
64
|
+
const result = await read(image);
|
|
65
|
+
yield { pageNumber: n, pageCount, source: "ocr", text: result.text, result, image, width: image.width, height: image.height };
|
|
66
|
+
} finally {
|
|
67
|
+
page.cleanup();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
} finally {
|
|
71
|
+
await task.destroy();
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Render one page to ImageData on white, about targetLongSide pixels on its long side. */
|
|
76
|
+
async function render(page, base, targetLongSide) {
|
|
77
|
+
const scale = Math.min(4, targetLongSide / Math.max(base.width, base.height));
|
|
78
|
+
const viewport = page.getViewport({ scale });
|
|
79
|
+
const w = Math.round(viewport.width), h = Math.round(viewport.height);
|
|
80
|
+
const canvas = typeof OffscreenCanvas !== "undefined" ? new OffscreenCanvas(w, h)
|
|
81
|
+
: Object.assign(document.createElement("canvas"), { width: w, height: h });
|
|
82
|
+
const ctx = canvas.getContext("2d", { willReadFrequently: true });
|
|
83
|
+
ctx.fillStyle = "#ffffff"; // transparent PDF backgrounds read as white paper
|
|
84
|
+
ctx.fillRect(0, 0, w, h);
|
|
85
|
+
await page.render({ canvas, canvasContext: ctx, viewport }).promise;
|
|
86
|
+
return ctx.getImageData(0, 0, w, h);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Text of a pdf.js TextContent, one line per text line, Thai-normalised (PDFs often carry legacy code points). */
|
|
90
|
+
function textOf(content) {
|
|
91
|
+
let out = "";
|
|
92
|
+
for (const item of content.items) {
|
|
93
|
+
if (typeof item.str !== "string") continue;
|
|
94
|
+
out += item.str + (item.hasEOL ? "\n" : "");
|
|
95
|
+
}
|
|
96
|
+
return canonical(out);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Share of characters that show a broken text layer: unmapped glyphs, or Thai marks without a base letter. */
|
|
100
|
+
function brokenShare(text) {
|
|
101
|
+
const chars = [...text.replace(/\s/g, "")];
|
|
102
|
+
if (!chars.length) return 1;
|
|
103
|
+
const unmapped = chars.filter((c) => c === "\u0000" || c === "\uFFFD" || (c >= "\uE000" && c <= "\uF8FF")).length;
|
|
104
|
+
return Math.max(unmapped / chars.length, invalidShare(text));
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** True for a PDF file (by type, name or the %PDF magic bytes). */
|
|
108
|
+
export async function isPdf(file) {
|
|
109
|
+
if (file?.type === "application/pdf" || /\.pdf$/i.test(file?.name || "")) return true;
|
|
110
|
+
const head = new Uint8Array(await file.slice(0, 5).arrayBuffer());
|
|
111
|
+
return String.fromCharCode(...head) === "%PDF-";
|
|
112
|
+
}
|
package/src/thai-norm.js
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Port of runtime/thai_norm.py (OpenThai 2.1 OCR Text, Apache-2.0, iApp Technology), modified for JavaScript.
|
|
2
|
+
// One canonical form for Thai text; see the original for the rationale of every rule.
|
|
3
|
+
|
|
4
|
+
const PUA = {
|
|
5
|
+
0xf700: 0x0e10, 0xf701: 0x0e34, 0xf702: 0x0e35, 0xf703: 0x0e36, 0xf704: 0x0e37,
|
|
6
|
+
0xf705: 0x0e48, 0xf706: 0x0e49, 0xf707: 0x0e4a, 0xf708: 0x0e4b, 0xf709: 0x0e4c,
|
|
7
|
+
0xf70a: 0x0e48, 0xf70b: 0x0e49, 0xf70c: 0x0e4a, 0xf70d: 0x0e4b, 0xf70e: 0x0e4c,
|
|
8
|
+
0xf70f: 0x0e0d, 0xf710: 0x0e31, 0xf711: 0x0e4d, 0xf712: 0x0e47, 0xf713: 0x0e48,
|
|
9
|
+
0xf714: 0x0e49, 0xf715: 0x0e4a, 0xf716: 0x0e4b, 0xf717: 0x0e4c, 0xf718: 0x0e38,
|
|
10
|
+
0xf719: 0x0e39, 0xf71a: 0x0e3a,
|
|
11
|
+
};
|
|
12
|
+
const DROP = new Set(["", "", "", "", "", ""]);
|
|
13
|
+
|
|
14
|
+
const VOWEL_MARKS = new Set("ัิีึืฺุู็");
|
|
15
|
+
const TONE_MARKS = new Set("่้๊๋์๎");
|
|
16
|
+
const NIKHAHIT = "ํ";
|
|
17
|
+
const MARKS = new Set([...VOWEL_MARKS, ...TONE_MARKS, NIKHAHIT]);
|
|
18
|
+
const MARK_RUN = new RegExp("[" + [...MARKS].sort().join("") + "]{2,}", "gu");
|
|
19
|
+
const SARA_AM = /([่-๋]?)ํ([่-๋]?)า/gu;
|
|
20
|
+
const SPACES = /[ \t\f\v]+/g;
|
|
21
|
+
|
|
22
|
+
function translate(text) {
|
|
23
|
+
let out = "";
|
|
24
|
+
for (const ch of text) {
|
|
25
|
+
if (DROP.has(ch)) continue;
|
|
26
|
+
const cp = ch.codePointAt(0);
|
|
27
|
+
if (cp === 0xa0) out += " ";
|
|
28
|
+
else if (PUA[cp] !== undefined) out += String.fromCodePoint(PUA[cp]);
|
|
29
|
+
else out += ch;
|
|
30
|
+
}
|
|
31
|
+
return out;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function rank(c) {
|
|
35
|
+
return VOWEL_MARKS.has(c) ? 0 : TONE_MARKS.has(c) ? 1 : 2;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function order(run) {
|
|
39
|
+
const sorted = [...run].sort((a, b) => rank(a) - rank(b)); // Array.sort is stable, as Python's sorted
|
|
40
|
+
const out = [];
|
|
41
|
+
for (const ch of sorted) if (!out.length || out[out.length - 1] !== ch) out.push(ch);
|
|
42
|
+
return out.join("");
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function canonical(text) {
|
|
46
|
+
text = translate(text || "").normalize("NFC");
|
|
47
|
+
text = text.replace(MARK_RUN, order);
|
|
48
|
+
text = text.replace(SARA_AM, (_, a, b) => a + b + "ำ");
|
|
49
|
+
text = text.split("เเ").join("แ").split("ำำ").join("ำ");
|
|
50
|
+
text = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
|
|
51
|
+
return text.split("\n").map((l) => l.replace(SPACES, " ").trim()).join("\n").trim();
|
|
52
|
+
}
|
|
Binary file
|