rga-ocr 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rga_ocr/__init__.py ADDED
@@ -0,0 +1,9 @@
1
+ """rga-ocr: file-name search plus ripgrep-all content search with OCR adapters."""
2
+
3
+ PROG = "rga-ocr"
4
+ VERSION = "0.1.1"
5
+ ENV_PREFIX = "RGA_OCR_"
6
+
7
+ # names from before the rename, for migration hints only
8
+ LEGACY_PROG = "rga-vision"
9
+ LEGACY_ENV_PREFIX = "RGA_VISION_"
rga_ocr/adapter.py ADDED
@@ -0,0 +1,290 @@
1
+ #!/usr/bin/env python3
2
+ """ripgrep-all custom adapter: stdin -> text.
3
+
4
+ Modes (argv[1]); argv[2] is the virtual path, for progress reports only:
5
+ pdf text layer per page via pypdfium2 (PDFium), OCR for pages without one;
6
+ pages separated by form feed (\\x0c) so rga's postprocpagebreaks
7
+ adds "Page N: " prefixes
8
+ image OCR (jpg, png, heic, tiff, webp, gif, bmp; decoded by Pillow)
9
+ office xlsx, pptx, ods, odp (zip of XML); pptx/odp slides as pages
10
+
11
+ Environment (set by rga-ocr from the profile, flags, or directly):
12
+ RGA_OCR_ENGINE OCR engine name (vision, winocr, rapidocr, tesseract,
13
+ or a registered one); unset: the platform's auto pick
14
+ RGA_OCR_LANG comma separated OCR languages, BCP 47 (de-DE,en-US);
15
+ unset or empty: the engine's default / auto-detection
16
+ RGA_OCR_FORCE_OCR "1": OCR every PDF page and append to the text layer
17
+ RGA_OCR_MIN_TEXT_CHARS PDF page with fewer text-layer chars is OCR'd (20)
18
+ RGA_OCR_RENDER_DPI PDF render resolution for OCR (200)
19
+ RGA_OCR_MAX_PIXELS longest image side handed to the OCR engine (3000)
20
+ RGA_OCR_PARAGRAPH_GAP line gap, in line heights, that starts a new output line (0.7)
21
+ RGA_OCR_REPORT path of a report file: one line is appended when a file
22
+ starts, one when it is done, one per error (rga-ocr's
23
+ status line and error forwarding read it)
24
+
25
+ Rules: never open the virtual path (archive members do not exist on disk),
26
+ never exit non-zero (rg would abort with exit 2), never emit NUL bytes.
27
+ rg discards the stderr of a preprocessor that exits 0, so errors are only
28
+ visible through the report file.
29
+ """
30
+
31
+ import html
32
+ import io
33
+ import os
34
+ import re
35
+ import sys
36
+ import zipfile
37
+
38
+ from rga_ocr import ENV_PREFIX, PROG
39
+
40
+ TAG = PROG + " adapter"
41
+
42
+
43
+ def _env_num(name, default, cast):
44
+ name = ENV_PREFIX + name
45
+ raw = os.environ.get(name, "")
46
+ try:
47
+ return cast(raw) if raw.strip() else default
48
+ except ValueError:
49
+ sys.stderr.write(f"{TAG}: ignoring {name}={raw!r}\n")
50
+ return default
51
+
52
+
53
+ MIN_TEXT_CHARS = _env_num("MIN_TEXT_CHARS", 20, int) # PDF page with fewer chars -> OCR
54
+ RENDER_DPI = _env_num("RENDER_DPI", 200, int)
55
+ MAX_PIXELS = _env_num("MAX_PIXELS", 3000, int) # longest image side handed to the engine
56
+ PARAGRAPH_GAP = _env_num("PARAGRAPH_GAP", 0.7, float)
57
+ PAGE_BREAK = "\x0c"
58
+
59
+
60
+ # ----------------------------------------------------------------- text
61
+
62
+ def clean(text):
63
+ """De-hyphenate, join wrapped lines into one, drop NUL and form feeds."""
64
+ text = text.replace("\x00", "").replace("\x0c", "").replace("­", "")
65
+ text = re.sub(r"-\n(?=[a-zäöüß])", "", text)
66
+ text = re.sub(r"\s+", " ", text)
67
+ return text.strip()
68
+
69
+
70
+ def write_out(text):
71
+ text = text.replace("\x00", "")
72
+ sys.stdout.buffer.write(text.encode("utf-8", "replace"))
73
+ sys.stdout.buffer.flush()
74
+
75
+
76
+ # ------------------------------------------------------------------ ocr
77
+
78
+ def decode(data):
79
+ """Every frame of an encoded image as an RGB Pillow image: EXIF orientation
80
+ applied, longest side capped at MAX_PIXELS."""
81
+ from PIL import Image, ImageOps, ImageSequence, UnidentifiedImageError
82
+ try:
83
+ import pi_heif
84
+ pi_heif.register_heif_opener()
85
+ except ImportError:
86
+ pass
87
+ try:
88
+ src = Image.open(io.BytesIO(data))
89
+ except UnidentifiedImageError:
90
+ raise RuntimeError("not a decodable image") from None
91
+ frames = []
92
+ with src:
93
+ for frame in ImageSequence.Iterator(src):
94
+ img = ImageOps.exif_transpose(frame)
95
+ if img.mode != "RGB":
96
+ img = img.convert("RGB")
97
+ if max(img.size) > MAX_PIXELS:
98
+ img.thumbnail((MAX_PIXELS, MAX_PIXELS))
99
+ else:
100
+ img.load()
101
+ frames.append(img)
102
+ return frames
103
+
104
+
105
+ class OCR:
106
+ """The configured engine plus paragraph grouping of its lines."""
107
+
108
+ def __init__(self):
109
+ from rga_ocr import engines
110
+ langs = engines.parse_langs(os.environ.get(ENV_PREFIX + "LANG"))
111
+ name = os.environ.get(ENV_PREFIX + "ENGINE", "").strip() or "auto"
112
+ self.engine = engines.resolve(name, langs)(langs)
113
+
114
+ def text(self, img):
115
+ return paragraphs(self.engine.recognize(img))
116
+
117
+ def text_of_bytes(self, data):
118
+ """OCR every frame of an encoded image; returns list of texts."""
119
+ return [self.text(img) for img in decode(data)]
120
+
121
+
122
+ def paragraphs(lines):
123
+ """Join lines that sit directly beneath each other into one line.
124
+ lines: (top, x, height, text) in a top-down frame normalised to 0..1."""
125
+ if not lines:
126
+ return ""
127
+ lines = sorted(lines, key=lambda l: (l[0], l[1]))
128
+ out, cur = [], [lines[0][3]]
129
+ prev_top, _, prev_h, _ = lines[0]
130
+ prev_bottom = prev_top + prev_h
131
+ for top, _x, h, s in lines[1:]:
132
+ gap = top - prev_bottom
133
+ if gap < PARAGRAPH_GAP * max(prev_h, h):
134
+ cur.append(s)
135
+ else:
136
+ out.append("\n".join(cur))
137
+ cur = [s]
138
+ prev_bottom = max(prev_bottom, top + h) if gap < 0 else top + h
139
+ prev_h = h
140
+ out.append("\n".join(cur))
141
+ return "\n".join(clean(p) for p in out if p.strip())
142
+
143
+
144
+ # ---------------------------------------------------------------- modes
145
+
146
+ def _text_layer(page):
147
+ """The text lines of a PDF page, grouped like OCR output: one tuple per
148
+ pdfium text rectangle in the (top, x, height, text) frame paragraphs()
149
+ expects (top-down, normalised to 0..1)."""
150
+ width, height = page.get_size()
151
+ tp = page.get_textpage()
152
+ try:
153
+ lines = []
154
+ for i in range(tp.count_rects()):
155
+ left, bottom, right, top = tp.get_rect(i)
156
+ s = tp.get_text_bounded(left=left, bottom=bottom, right=right, top=top).strip()
157
+ if s:
158
+ lines.append(((height - top) / height, left / width, (top - bottom) / height, s))
159
+ finally:
160
+ tp.close()
161
+ return paragraphs(lines)
162
+
163
+
164
+ def mode_pdf(data):
165
+ import pypdfium2
166
+ force = os.environ.get(ENV_PREFIX + "FORCE_OCR") == "1"
167
+ ocr = None
168
+ pages = []
169
+ doc = pypdfium2.PdfDocument(data)
170
+ try:
171
+ for page in doc:
172
+ layer = _text_layer(page)
173
+ if force or len(layer.replace(" ", "")) < MIN_TEXT_CHARS:
174
+ if ocr is None:
175
+ ocr = OCR()
176
+ scale = min(RENDER_DPI / 72, MAX_PIXELS / max(page.get_size()))
177
+ img = page.render(scale=scale).to_pil()
178
+ if img.mode != "RGB":
179
+ img = img.convert("RGB")
180
+ text = ocr.text(img)
181
+ layer = (layer + "\n" + text).strip() if force else text
182
+ pages.append(layer)
183
+ page.close()
184
+ finally:
185
+ doc.close()
186
+ return PAGE_BREAK.join(pages)
187
+
188
+
189
+ def mode_image(data):
190
+ texts = OCR().text_of_bytes(data)
191
+ return "\n\n".join(t for t in texts if t)
192
+
193
+
194
+ def _strip_xml(raw, para_tags, cell_sep=" "):
195
+ raw = re.sub(r"</(?:%s)>" % "|".join(para_tags), "\n", raw)
196
+ raw = re.sub(r"</(?:w:t|a:t|t|text:span)>", cell_sep, raw)
197
+ raw = re.sub(r"<[^>]+>", "", raw)
198
+ raw = html.unescape(raw)
199
+ lines = [clean(l) for l in raw.split("\n")]
200
+ return "\n".join(l for l in lines if l)
201
+
202
+
203
+ def mode_office(data):
204
+ pages = []
205
+ with zipfile.ZipFile(io.BytesIO(data)) as z:
206
+ names = z.namelist()
207
+ slides = sorted(
208
+ (n for n in names if re.fullmatch(r"ppt/slides/slide\d+\.xml", n)),
209
+ key=lambda n: int(re.search(r"(\d+)\.xml$", n).group(1)))
210
+ if slides: # pptx
211
+ for n in slides:
212
+ pages.append(_strip_xml(z.read(n).decode("utf-8", "replace"), ["a:p"]))
213
+ elif "xl/workbook.xml" in names: # xlsx
214
+ parts = []
215
+ if "xl/sharedStrings.xml" in names:
216
+ parts.append(_strip_xml(z.read("xl/sharedStrings.xml").decode("utf-8", "replace"), ["si"]))
217
+ for n in sorted(n for n in names if re.fullmatch(r"xl/worksheets/sheet\d+\.xml", n)):
218
+ raw = z.read(n).decode("utf-8", "replace")
219
+ raw = re.sub(r'<c\b[^>]*\bt="s"[^>]*>.*?</c>', "", raw, flags=re.S) # shared-string refs
220
+ parts.append(_strip_xml(raw, ["row"]))
221
+ pages.append("\n".join(p for p in parts if p))
222
+ elif "content.xml" in names: # ods / odp
223
+ raw = z.read("content.xml").decode("utf-8", "replace")
224
+ draw_pages = re.split(r"</draw:page>", raw)
225
+ if len(draw_pages) > 1: # odp
226
+ for p in draw_pages:
227
+ t = _strip_xml(p, ["text:p", "text:h"])
228
+ if t:
229
+ pages.append(t)
230
+ else: # ods
231
+ pages.append(_strip_xml(raw, ["text:p", "text:h", "table:table-row"]))
232
+ else:
233
+ raise RuntimeError("unrecognised office container")
234
+ return PAGE_BREAK.join(pages)
235
+
236
+
237
+ MODES = {"pdf": mode_pdf, "image": mode_image, "office": mode_office}
238
+
239
+
240
+ REPORT = os.environ.get(ENV_PREFIX + "REPORT", "")
241
+
242
+
243
+ def report(event, mode, path, detail=""):
244
+ """Append one line to the report file in a single O_APPEND write, so the
245
+ lines of parallel adapters do not interleave (short writes stay atomic)."""
246
+ if not REPORT:
247
+ return
248
+ try:
249
+ fd = os.open(REPORT, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600)
250
+ try:
251
+ line = f"{event}\t{mode}\t{path}\t{detail}\n".replace("\r", " ")
252
+ os.write(fd, line.encode("utf-8", "replace")[:4000])
253
+ finally:
254
+ os.close(fd)
255
+ except OSError:
256
+ pass
257
+
258
+
259
+ def main():
260
+ mode = sys.argv[1] if len(sys.argv) > 1 else ""
261
+ path = sys.argv[2] if len(sys.argv) > 2 else ""
262
+ fn = MODES.get(mode)
263
+ if fn is None:
264
+ sys.stderr.write(f"{TAG}: unknown mode {mode!r}\n")
265
+ return
266
+ data = sys.stdin.buffer.read()
267
+ if not data:
268
+ return
269
+ report("start", mode, path)
270
+ try:
271
+ write_out(fn(data))
272
+ except Exception as e: # noqa: BLE001
273
+ msg = f"{e.__class__.__name__}: {e}".replace("\n", " ")
274
+ sys.stderr.write(f"{TAG}[{mode}]: {msg}\n")
275
+ report("error", mode, path, msg)
276
+ report("done", mode, path)
277
+
278
+
279
+ if __name__ == "__main__":
280
+ main()
281
+ # rg aborts the whole search (exit 2) when a preprocessor exits non-zero,
282
+ # and the OCR libraries run native teardown code at interpreter exit
283
+ # (seen: onnxruntime's telemetry shutdown spinning forever). Everything is
284
+ # written and flushed by now, so end the process here.
285
+ for stream in (sys.stdout, sys.stderr):
286
+ try:
287
+ stream.flush()
288
+ except (OSError, ValueError):
289
+ pass
290
+ os._exit(0)