rga-ocr 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rga_ocr/__init__.py +9 -0
- rga_ocr/adapter.py +290 -0
- rga_ocr/cli.py +1100 -0
- rga_ocr/engines/__init__.py +195 -0
- rga_ocr/engines/rapidocr.py +110 -0
- rga_ocr/engines/tesseract.py +107 -0
- rga_ocr/engines/vision.py +82 -0
- rga_ocr/engines/winocr.py +130 -0
- rga_ocr/paths.py +61 -0
- rga_ocr/setup_cmd.py +490 -0
- rga_ocr-0.1.1.dist-info/METADATA +618 -0
- rga_ocr-0.1.1.dist-info/RECORD +16 -0
- rga_ocr-0.1.1.dist-info/WHEEL +5 -0
- rga_ocr-0.1.1.dist-info/entry_points.txt +2 -0
- rga_ocr-0.1.1.dist-info/licenses/LICENSE +21 -0
- rga_ocr-0.1.1.dist-info/top_level.txt +1 -0
rga_ocr/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""rga-ocr: file-name search plus ripgrep-all content search with OCR adapters."""
|
|
2
|
+
|
|
3
|
+
PROG = "rga-ocr"
|
|
4
|
+
VERSION = "0.1.1"
|
|
5
|
+
ENV_PREFIX = "RGA_OCR_"
|
|
6
|
+
|
|
7
|
+
# names from before the rename, for migration hints only
|
|
8
|
+
LEGACY_PROG = "rga-vision"
|
|
9
|
+
LEGACY_ENV_PREFIX = "RGA_VISION_"
|
rga_ocr/adapter.py
ADDED
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""ripgrep-all custom adapter: stdin -> text.
|
|
3
|
+
|
|
4
|
+
Modes (argv[1]); argv[2] is the virtual path, for progress reports only:
|
|
5
|
+
pdf text layer per page via pypdfium2 (PDFium), OCR for pages without one;
|
|
6
|
+
pages separated by form feed (\\x0c) so rga's postprocpagebreaks
|
|
7
|
+
adds "Page N: " prefixes
|
|
8
|
+
image OCR (jpg, png, heic, tiff, webp, gif, bmp; decoded by Pillow)
|
|
9
|
+
office xlsx, pptx, ods, odp (zip of XML); pptx/odp slides as pages
|
|
10
|
+
|
|
11
|
+
Environment (set by rga-ocr from the profile, flags, or directly):
|
|
12
|
+
RGA_OCR_ENGINE OCR engine name (vision, winocr, rapidocr, tesseract,
|
|
13
|
+
or a registered one); unset: the platform's auto pick
|
|
14
|
+
RGA_OCR_LANG comma separated OCR languages, BCP 47 (de-DE,en-US);
|
|
15
|
+
unset or empty: the engine's default / auto-detection
|
|
16
|
+
RGA_OCR_FORCE_OCR "1": OCR every PDF page and append to the text layer
|
|
17
|
+
RGA_OCR_MIN_TEXT_CHARS PDF page with fewer text-layer chars is OCR'd (20)
|
|
18
|
+
RGA_OCR_RENDER_DPI PDF render resolution for OCR (200)
|
|
19
|
+
RGA_OCR_MAX_PIXELS longest image side handed to the OCR engine (3000)
|
|
20
|
+
RGA_OCR_PARAGRAPH_GAP line gap, in line heights, that starts a new output line (0.7)
|
|
21
|
+
RGA_OCR_REPORT path of a report file: one line is appended when a file
|
|
22
|
+
starts, one when it is done, one per error (rga-ocr's
|
|
23
|
+
status line and error forwarding read it)
|
|
24
|
+
|
|
25
|
+
Rules: never open the virtual path (archive members do not exist on disk),
|
|
26
|
+
never exit non-zero (rg would abort with exit 2), never emit NUL bytes.
|
|
27
|
+
rg discards the stderr of a preprocessor that exits 0, so errors are only
|
|
28
|
+
visible through the report file.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
import html
|
|
32
|
+
import io
|
|
33
|
+
import os
|
|
34
|
+
import re
|
|
35
|
+
import sys
|
|
36
|
+
import zipfile
|
|
37
|
+
|
|
38
|
+
from rga_ocr import ENV_PREFIX, PROG
|
|
39
|
+
|
|
40
|
+
TAG = PROG + " adapter"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _env_num(name, default, cast):
|
|
44
|
+
name = ENV_PREFIX + name
|
|
45
|
+
raw = os.environ.get(name, "")
|
|
46
|
+
try:
|
|
47
|
+
return cast(raw) if raw.strip() else default
|
|
48
|
+
except ValueError:
|
|
49
|
+
sys.stderr.write(f"{TAG}: ignoring {name}={raw!r}\n")
|
|
50
|
+
return default
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
MIN_TEXT_CHARS = _env_num("MIN_TEXT_CHARS", 20, int) # PDF page with fewer chars -> OCR
|
|
54
|
+
RENDER_DPI = _env_num("RENDER_DPI", 200, int)
|
|
55
|
+
MAX_PIXELS = _env_num("MAX_PIXELS", 3000, int) # longest image side handed to the engine
|
|
56
|
+
PARAGRAPH_GAP = _env_num("PARAGRAPH_GAP", 0.7, float)
|
|
57
|
+
PAGE_BREAK = "\x0c"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# ----------------------------------------------------------------- text
|
|
61
|
+
|
|
62
|
+
def clean(text):
|
|
63
|
+
"""De-hyphenate, join wrapped lines into one, drop NUL and form feeds."""
|
|
64
|
+
text = text.replace("\x00", "").replace("\x0c", "").replace("", "")
|
|
65
|
+
text = re.sub(r"-\n(?=[a-zäöüß])", "", text)
|
|
66
|
+
text = re.sub(r"\s+", " ", text)
|
|
67
|
+
return text.strip()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def write_out(text):
|
|
71
|
+
text = text.replace("\x00", "")
|
|
72
|
+
sys.stdout.buffer.write(text.encode("utf-8", "replace"))
|
|
73
|
+
sys.stdout.buffer.flush()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# ------------------------------------------------------------------ ocr
|
|
77
|
+
|
|
78
|
+
def decode(data):
|
|
79
|
+
"""Every frame of an encoded image as an RGB Pillow image: EXIF orientation
|
|
80
|
+
applied, longest side capped at MAX_PIXELS."""
|
|
81
|
+
from PIL import Image, ImageOps, ImageSequence, UnidentifiedImageError
|
|
82
|
+
try:
|
|
83
|
+
import pi_heif
|
|
84
|
+
pi_heif.register_heif_opener()
|
|
85
|
+
except ImportError:
|
|
86
|
+
pass
|
|
87
|
+
try:
|
|
88
|
+
src = Image.open(io.BytesIO(data))
|
|
89
|
+
except UnidentifiedImageError:
|
|
90
|
+
raise RuntimeError("not a decodable image") from None
|
|
91
|
+
frames = []
|
|
92
|
+
with src:
|
|
93
|
+
for frame in ImageSequence.Iterator(src):
|
|
94
|
+
img = ImageOps.exif_transpose(frame)
|
|
95
|
+
if img.mode != "RGB":
|
|
96
|
+
img = img.convert("RGB")
|
|
97
|
+
if max(img.size) > MAX_PIXELS:
|
|
98
|
+
img.thumbnail((MAX_PIXELS, MAX_PIXELS))
|
|
99
|
+
else:
|
|
100
|
+
img.load()
|
|
101
|
+
frames.append(img)
|
|
102
|
+
return frames
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class OCR:
|
|
106
|
+
"""The configured engine plus paragraph grouping of its lines."""
|
|
107
|
+
|
|
108
|
+
def __init__(self):
|
|
109
|
+
from rga_ocr import engines
|
|
110
|
+
langs = engines.parse_langs(os.environ.get(ENV_PREFIX + "LANG"))
|
|
111
|
+
name = os.environ.get(ENV_PREFIX + "ENGINE", "").strip() or "auto"
|
|
112
|
+
self.engine = engines.resolve(name, langs)(langs)
|
|
113
|
+
|
|
114
|
+
def text(self, img):
|
|
115
|
+
return paragraphs(self.engine.recognize(img))
|
|
116
|
+
|
|
117
|
+
def text_of_bytes(self, data):
|
|
118
|
+
"""OCR every frame of an encoded image; returns list of texts."""
|
|
119
|
+
return [self.text(img) for img in decode(data)]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def paragraphs(lines):
|
|
123
|
+
"""Join lines that sit directly beneath each other into one line.
|
|
124
|
+
lines: (top, x, height, text) in a top-down frame normalised to 0..1."""
|
|
125
|
+
if not lines:
|
|
126
|
+
return ""
|
|
127
|
+
lines = sorted(lines, key=lambda l: (l[0], l[1]))
|
|
128
|
+
out, cur = [], [lines[0][3]]
|
|
129
|
+
prev_top, _, prev_h, _ = lines[0]
|
|
130
|
+
prev_bottom = prev_top + prev_h
|
|
131
|
+
for top, _x, h, s in lines[1:]:
|
|
132
|
+
gap = top - prev_bottom
|
|
133
|
+
if gap < PARAGRAPH_GAP * max(prev_h, h):
|
|
134
|
+
cur.append(s)
|
|
135
|
+
else:
|
|
136
|
+
out.append("\n".join(cur))
|
|
137
|
+
cur = [s]
|
|
138
|
+
prev_bottom = max(prev_bottom, top + h) if gap < 0 else top + h
|
|
139
|
+
prev_h = h
|
|
140
|
+
out.append("\n".join(cur))
|
|
141
|
+
return "\n".join(clean(p) for p in out if p.strip())
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# ---------------------------------------------------------------- modes
|
|
145
|
+
|
|
146
|
+
def _text_layer(page):
|
|
147
|
+
"""The text lines of a PDF page, grouped like OCR output: one tuple per
|
|
148
|
+
pdfium text rectangle in the (top, x, height, text) frame paragraphs()
|
|
149
|
+
expects (top-down, normalised to 0..1)."""
|
|
150
|
+
width, height = page.get_size()
|
|
151
|
+
tp = page.get_textpage()
|
|
152
|
+
try:
|
|
153
|
+
lines = []
|
|
154
|
+
for i in range(tp.count_rects()):
|
|
155
|
+
left, bottom, right, top = tp.get_rect(i)
|
|
156
|
+
s = tp.get_text_bounded(left=left, bottom=bottom, right=right, top=top).strip()
|
|
157
|
+
if s:
|
|
158
|
+
lines.append(((height - top) / height, left / width, (top - bottom) / height, s))
|
|
159
|
+
finally:
|
|
160
|
+
tp.close()
|
|
161
|
+
return paragraphs(lines)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def mode_pdf(data):
|
|
165
|
+
import pypdfium2
|
|
166
|
+
force = os.environ.get(ENV_PREFIX + "FORCE_OCR") == "1"
|
|
167
|
+
ocr = None
|
|
168
|
+
pages = []
|
|
169
|
+
doc = pypdfium2.PdfDocument(data)
|
|
170
|
+
try:
|
|
171
|
+
for page in doc:
|
|
172
|
+
layer = _text_layer(page)
|
|
173
|
+
if force or len(layer.replace(" ", "")) < MIN_TEXT_CHARS:
|
|
174
|
+
if ocr is None:
|
|
175
|
+
ocr = OCR()
|
|
176
|
+
scale = min(RENDER_DPI / 72, MAX_PIXELS / max(page.get_size()))
|
|
177
|
+
img = page.render(scale=scale).to_pil()
|
|
178
|
+
if img.mode != "RGB":
|
|
179
|
+
img = img.convert("RGB")
|
|
180
|
+
text = ocr.text(img)
|
|
181
|
+
layer = (layer + "\n" + text).strip() if force else text
|
|
182
|
+
pages.append(layer)
|
|
183
|
+
page.close()
|
|
184
|
+
finally:
|
|
185
|
+
doc.close()
|
|
186
|
+
return PAGE_BREAK.join(pages)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def mode_image(data):
|
|
190
|
+
texts = OCR().text_of_bytes(data)
|
|
191
|
+
return "\n\n".join(t for t in texts if t)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _strip_xml(raw, para_tags, cell_sep=" "):
|
|
195
|
+
raw = re.sub(r"</(?:%s)>" % "|".join(para_tags), "\n", raw)
|
|
196
|
+
raw = re.sub(r"</(?:w:t|a:t|t|text:span)>", cell_sep, raw)
|
|
197
|
+
raw = re.sub(r"<[^>]+>", "", raw)
|
|
198
|
+
raw = html.unescape(raw)
|
|
199
|
+
lines = [clean(l) for l in raw.split("\n")]
|
|
200
|
+
return "\n".join(l for l in lines if l)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def mode_office(data):
|
|
204
|
+
pages = []
|
|
205
|
+
with zipfile.ZipFile(io.BytesIO(data)) as z:
|
|
206
|
+
names = z.namelist()
|
|
207
|
+
slides = sorted(
|
|
208
|
+
(n for n in names if re.fullmatch(r"ppt/slides/slide\d+\.xml", n)),
|
|
209
|
+
key=lambda n: int(re.search(r"(\d+)\.xml$", n).group(1)))
|
|
210
|
+
if slides: # pptx
|
|
211
|
+
for n in slides:
|
|
212
|
+
pages.append(_strip_xml(z.read(n).decode("utf-8", "replace"), ["a:p"]))
|
|
213
|
+
elif "xl/workbook.xml" in names: # xlsx
|
|
214
|
+
parts = []
|
|
215
|
+
if "xl/sharedStrings.xml" in names:
|
|
216
|
+
parts.append(_strip_xml(z.read("xl/sharedStrings.xml").decode("utf-8", "replace"), ["si"]))
|
|
217
|
+
for n in sorted(n for n in names if re.fullmatch(r"xl/worksheets/sheet\d+\.xml", n)):
|
|
218
|
+
raw = z.read(n).decode("utf-8", "replace")
|
|
219
|
+
raw = re.sub(r'<c\b[^>]*\bt="s"[^>]*>.*?</c>', "", raw, flags=re.S) # shared-string refs
|
|
220
|
+
parts.append(_strip_xml(raw, ["row"]))
|
|
221
|
+
pages.append("\n".join(p for p in parts if p))
|
|
222
|
+
elif "content.xml" in names: # ods / odp
|
|
223
|
+
raw = z.read("content.xml").decode("utf-8", "replace")
|
|
224
|
+
draw_pages = re.split(r"</draw:page>", raw)
|
|
225
|
+
if len(draw_pages) > 1: # odp
|
|
226
|
+
for p in draw_pages:
|
|
227
|
+
t = _strip_xml(p, ["text:p", "text:h"])
|
|
228
|
+
if t:
|
|
229
|
+
pages.append(t)
|
|
230
|
+
else: # ods
|
|
231
|
+
pages.append(_strip_xml(raw, ["text:p", "text:h", "table:table-row"]))
|
|
232
|
+
else:
|
|
233
|
+
raise RuntimeError("unrecognised office container")
|
|
234
|
+
return PAGE_BREAK.join(pages)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
MODES = {"pdf": mode_pdf, "image": mode_image, "office": mode_office}
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
REPORT = os.environ.get(ENV_PREFIX + "REPORT", "")
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def report(event, mode, path, detail=""):
|
|
244
|
+
"""Append one line to the report file in a single O_APPEND write, so the
|
|
245
|
+
lines of parallel adapters do not interleave (short writes stay atomic)."""
|
|
246
|
+
if not REPORT:
|
|
247
|
+
return
|
|
248
|
+
try:
|
|
249
|
+
fd = os.open(REPORT, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600)
|
|
250
|
+
try:
|
|
251
|
+
line = f"{event}\t{mode}\t{path}\t{detail}\n".replace("\r", " ")
|
|
252
|
+
os.write(fd, line.encode("utf-8", "replace")[:4000])
|
|
253
|
+
finally:
|
|
254
|
+
os.close(fd)
|
|
255
|
+
except OSError:
|
|
256
|
+
pass
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def main():
|
|
260
|
+
mode = sys.argv[1] if len(sys.argv) > 1 else ""
|
|
261
|
+
path = sys.argv[2] if len(sys.argv) > 2 else ""
|
|
262
|
+
fn = MODES.get(mode)
|
|
263
|
+
if fn is None:
|
|
264
|
+
sys.stderr.write(f"{TAG}: unknown mode {mode!r}\n")
|
|
265
|
+
return
|
|
266
|
+
data = sys.stdin.buffer.read()
|
|
267
|
+
if not data:
|
|
268
|
+
return
|
|
269
|
+
report("start", mode, path)
|
|
270
|
+
try:
|
|
271
|
+
write_out(fn(data))
|
|
272
|
+
except Exception as e: # noqa: BLE001
|
|
273
|
+
msg = f"{e.__class__.__name__}: {e}".replace("\n", " ")
|
|
274
|
+
sys.stderr.write(f"{TAG}[{mode}]: {msg}\n")
|
|
275
|
+
report("error", mode, path, msg)
|
|
276
|
+
report("done", mode, path)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
if __name__ == "__main__":
|
|
280
|
+
main()
|
|
281
|
+
# rg aborts the whole search (exit 2) when a preprocessor exits non-zero,
|
|
282
|
+
# and the OCR libraries run native teardown code at interpreter exit
|
|
283
|
+
# (seen: onnxruntime's telemetry shutdown spinning forever). Everything is
|
|
284
|
+
# written and flushed by now, so end the process here.
|
|
285
|
+
for stream in (sys.stdout, sys.stderr):
|
|
286
|
+
try:
|
|
287
|
+
stream.flush()
|
|
288
|
+
except (OSError, ValueError):
|
|
289
|
+
pass
|
|
290
|
+
os._exit(0)
|