ocr-stream 2.3.2__tar.gz → 2.3.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/PKG-INFO +2 -2
- ocr_stream-2.3.8/ocr_stream/__init__.py +356 -0
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/ocr_stream/bin_tess/__init__.py +3 -0
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/ocr_stream/modules/__init__.py +4 -1
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/ocr_stream/ocr/__init__.py +13 -6
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/ocr_stream/recognize.py +3 -2
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/ocr_stream.egg-info/PKG-INFO +2 -2
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/ocr_stream.egg-info/requires.txt +1 -1
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/pyproject.toml +2 -2
- ocr_stream-2.3.2/ocr_stream/__init__.py +0 -269
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/README.md +0 -0
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/ocr_stream.egg-info/SOURCES.txt +0 -0
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/ocr_stream.egg-info/dependency_links.txt +0 -0
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/ocr_stream.egg-info/top_level.txt +0 -0
- {ocr_stream-2.3.2 → ocr_stream-2.3.8}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ocr_stream
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.8
|
|
4
4
|
Summary: Extrair textos de documentos digitalizados e imagens.
|
|
5
5
|
Author: Bruno Chaves
|
|
6
6
|
Requires-Python: >=3.11
|
|
@@ -12,7 +12,7 @@ Requires-Dist: pandas
|
|
|
12
12
|
Requires-Dist: requests>=2.32.4
|
|
13
13
|
Requires-Dist: opencv-python
|
|
14
14
|
Requires-Dist: pytesseract
|
|
15
|
-
Requires-Dist: convert_stream>=2.3.
|
|
15
|
+
Requires-Dist: convert_stream>=2.3.8
|
|
16
16
|
Requires-Dist: soup_files>=1.4
|
|
17
17
|
|
|
18
18
|
# ocrlib
|
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
from io import BytesIO
|
|
4
|
+
import threading
|
|
5
|
+
from ocr_stream.modules import *
|
|
6
|
+
from ocr_stream.bin_tess import BinTesseract, get_path_tesseract_sys
|
|
7
|
+
from ocr_stream.ocr import TesseractOcr, TextRecognized
|
|
8
|
+
from ocr_stream.recognize import RecognizeImage, RecognizePdf
|
|
9
|
+
from convert_stream import DocumentPdf, PageDocumentPdf
|
|
10
|
+
from convert_stream.image import ImageObject, CollectionImages
|
|
11
|
+
from convert_stream.mod_types import (
|
|
12
|
+
ColumnsTable, ColumnBody, HeadCell, HeadValues, create_void_dict_map
|
|
13
|
+
)
|
|
14
|
+
from time import sleep
|
|
15
|
+
from convert_stream.text import FindText
|
|
16
|
+
from convert_stream.pdf import ConvertPdfToImages, CollectionPagePdf
|
|
17
|
+
import pandas as pd
|
|
18
|
+
from soup_files import (
|
|
19
|
+
JsonConvert, ProgressBarAdapter, ABCProgressBar, CreatePbar, Directory, File, InputFiles, LibraryDocs,
|
|
20
|
+
ProgressBarSimple
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class VoidPbar(ABCProgressBar):
|
|
25
|
+
|
|
26
|
+
def __init__(self):
|
|
27
|
+
super().__init__()
|
|
28
|
+
|
|
29
|
+
def set_percent(self, percent: float):
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
def set_text(self, text: str):
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class ProgressVoid(ProgressBarAdapter):
|
|
37
|
+
|
|
38
|
+
def __init__(self, progress_bar: ABCProgressBar = VoidPbar()):
|
|
39
|
+
super().__init__(progress_bar)
|
|
40
|
+
|
|
41
|
+
def update(self, percent: float, status: str = "-"):
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
def update_percent(self, percent: float = 0):
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
def update_text(self, text: str = "-"):
|
|
48
|
+
pass
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def read_image(img: ImageObject, rec: RecognizeImage = RecognizeImage()) -> TextRecognized:
|
|
52
|
+
_txt = rec.image_recognize(img)
|
|
53
|
+
_txt.metadata = img.metadata
|
|
54
|
+
return _txt
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def read_images(
|
|
58
|
+
images: list[ImageObject], *,
|
|
59
|
+
rec: RecognizeImage = RecognizeImage(),
|
|
60
|
+
pbar: ProgressBarAdapter = CreatePbar().get()
|
|
61
|
+
) -> DocumentPdf:
|
|
62
|
+
pbar.start()
|
|
63
|
+
print()
|
|
64
|
+
max_num = len(images)
|
|
65
|
+
pages = []
|
|
66
|
+
|
|
67
|
+
for idx, image in enumerate(images):
|
|
68
|
+
pbar.update(
|
|
69
|
+
((idx + 1) / max_num) * 100,
|
|
70
|
+
f'[OCR] {idx + 1} / {max_num}',
|
|
71
|
+
)
|
|
72
|
+
out = read_image(image, rec=rec).to_page_pdf()
|
|
73
|
+
pages.append(out)
|
|
74
|
+
tmp_doc = DocumentPdf.create_from_pages(pages)
|
|
75
|
+
print()
|
|
76
|
+
pbar.stop()
|
|
77
|
+
return tmp_doc
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def read_document(
|
|
81
|
+
document: DocumentPdf, *,
|
|
82
|
+
dpi: int = 300,
|
|
83
|
+
pbar: ProgressBarAdapter = CreatePbar().get()
|
|
84
|
+
) -> DocumentPdf:
|
|
85
|
+
pbar.start()
|
|
86
|
+
print()
|
|
87
|
+
_metadata = document.metadata
|
|
88
|
+
recognized_items: list[TextRecognized] = []
|
|
89
|
+
pages_pdf: list[PageDocumentPdf] = document.to_pages()
|
|
90
|
+
total = len(pages_pdf)
|
|
91
|
+
void_pbar = ProgressVoid()
|
|
92
|
+
for num, page in enumerate(pages_pdf):
|
|
93
|
+
pbar.update(
|
|
94
|
+
((num + 1) / total) * 100,
|
|
95
|
+
f'[OCR PDF] {num + 1}/{total} {page.metadata.name}',
|
|
96
|
+
)
|
|
97
|
+
convert: ConvertPdfToImages = ConvertPdfToImages(DocumentPdf.create_from_pages([page]))
|
|
98
|
+
convert.set_pbar(void_pbar)
|
|
99
|
+
images: list[ImageObject] = convert.to_images(dpi=dpi)
|
|
100
|
+
for im in images:
|
|
101
|
+
recognized_items.append(read_image(im))
|
|
102
|
+
print()
|
|
103
|
+
final_pages: list[PageDocumentPdf] = [x.to_page_pdf() for x in recognized_items]
|
|
104
|
+
final_doc: DocumentPdf = DocumentPdf.create_from_pages(final_pages)
|
|
105
|
+
final_doc.metadata = _metadata
|
|
106
|
+
return final_doc
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class SearchableTextImage(object):
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
return {
|
|
113
|
+
ColumnsTable.KEY.value: ColumnBody(ColumnsTable.KEY.value, ListString([])),
|
|
114
|
+
ColumnsTable.NUM_PAGE.value: ColumnBody(ColumnsTable.NUM_PAGE.value, ListString([])),
|
|
115
|
+
ColumnsTable.NUM_LINE.value: ColumnBody(ColumnsTable.NUM_LINE.value, ListString([])),
|
|
116
|
+
ColumnsTable.TEXT.value: ColumnBody(ColumnsTable.TEXT.value, ListString([])),
|
|
117
|
+
ColumnsTable.FILE_NAME.value: ColumnBody(ColumnsTable.FILE_NAME.value, ListString([])),
|
|
118
|
+
ColumnsTable.FILETYPE.value: ColumnBody(ColumnsTable.FILETYPE.value, ListString([])),
|
|
119
|
+
ColumnsTable.FILE_PATH.value: ColumnBody(ColumnsTable.FILE_PATH.value, ListString([])),
|
|
120
|
+
ColumnsTable.DIR.value: ColumnBody(ColumnsTable.DIR.value, ListString([])),
|
|
121
|
+
}
|
|
122
|
+
"""
|
|
123
|
+
def __init__(self, silent: bool = False):
|
|
124
|
+
self.silent = silent
|
|
125
|
+
self.elements: dict[str, ColumnBody] = create_void_dict_map()
|
|
126
|
+
|
|
127
|
+
def __repr__(self):
|
|
128
|
+
return f'SearchableTextPdf: {self.elements}'
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def length(self) -> int:
|
|
132
|
+
return len(self.elements[ColumnsTable.TEXT.value])
|
|
133
|
+
|
|
134
|
+
def is_empty(self) -> bool:
|
|
135
|
+
return len(self.elements[ColumnsTable.TEXT.value]) == 0
|
|
136
|
+
|
|
137
|
+
@property
|
|
138
|
+
def columns(self) -> HeadValues:
|
|
139
|
+
return HeadValues([HeadCell(x) for x in list(self.elements.keys())])
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def first(self) -> dict[str, str]:
|
|
143
|
+
if self.is_empty():
|
|
144
|
+
return {}
|
|
145
|
+
cols = self.columns
|
|
146
|
+
_first = {}
|
|
147
|
+
for c in cols:
|
|
148
|
+
_first[c] = self.elements[c][0]
|
|
149
|
+
return _first
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def last(self) -> dict[str, str]:
|
|
153
|
+
if self.is_empty():
|
|
154
|
+
return {}
|
|
155
|
+
if self.is_empty():
|
|
156
|
+
return {}
|
|
157
|
+
cols = self.columns
|
|
158
|
+
_last = {}
|
|
159
|
+
for c in cols:
|
|
160
|
+
_last[c] = self.elements[c][-1]
|
|
161
|
+
return _last
|
|
162
|
+
|
|
163
|
+
def get_item(self, idx: int) -> dict[str, str]:
|
|
164
|
+
_current_item = {}
|
|
165
|
+
cols = self.columns
|
|
166
|
+
try:
|
|
167
|
+
for col in cols:
|
|
168
|
+
_current_item[col] = self.elements[col][idx]
|
|
169
|
+
except Exception as err:
|
|
170
|
+
if not self.silent:
|
|
171
|
+
print(err)
|
|
172
|
+
return _current_item
|
|
173
|
+
else:
|
|
174
|
+
return _current_item
|
|
175
|
+
|
|
176
|
+
def add_line(self, line: dict[str, str]) -> None:
|
|
177
|
+
cols: HeadValues = self.columns
|
|
178
|
+
cols_line: HeadValues = HeadValues([HeadCell(y) for y in list(line.keys())])
|
|
179
|
+
for col in cols:
|
|
180
|
+
if cols_line.contains(col, case=True, iqual=True):
|
|
181
|
+
self.elements[col].append(line[col])
|
|
182
|
+
|
|
183
|
+
def clear(self) -> None:
|
|
184
|
+
for _k in self.elements.keys():
|
|
185
|
+
self.elements[_k].clear()
|
|
186
|
+
|
|
187
|
+
def to_string(self) -> str:
|
|
188
|
+
"""
|
|
189
|
+
Retorna o texto da coluna TEXT em formato de string
|
|
190
|
+
ou 'nas' em caso de erro nas = Not a String
|
|
191
|
+
"""
|
|
192
|
+
try:
|
|
193
|
+
return ' '.join(self.elements[ColumnsTable.TEXT.value])
|
|
194
|
+
except Exception as e:
|
|
195
|
+
print(e)
|
|
196
|
+
return 'nas'
|
|
197
|
+
|
|
198
|
+
def to_data_frame(self) -> pd.DataFrame:
|
|
199
|
+
return pd.DataFrame.from_dict(self.elements)
|
|
200
|
+
|
|
201
|
+
def to_file_json(self, file: File):
|
|
202
|
+
"""Exporta os dados da busca para um arquivo JSON"""
|
|
203
|
+
dt = JsonConvert.from_dict(self.elements).to_json_data()
|
|
204
|
+
dt.to_file(file)
|
|
205
|
+
|
|
206
|
+
def to_file_excel(self, file: File):
|
|
207
|
+
"""Exporta os dados da busca para um arquivo .XLSX"""
|
|
208
|
+
self.to_data_frame().to_excel(file.absolute(), index=False)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class ImageFinder(object):
|
|
212
|
+
"""
|
|
213
|
+
Classe para Filtrar texto em Imagens
|
|
214
|
+
"""
|
|
215
|
+
|
|
216
|
+
def __init__(self):
|
|
217
|
+
self.docs_collection: dict[str, ImageObject] = {}
|
|
218
|
+
self.recognize: RecognizeImage = RecognizeImage()
|
|
219
|
+
self.pbar: ProgressBarAdapter = ProgressBarAdapter()
|
|
220
|
+
|
|
221
|
+
def is_empty(self) -> bool:
|
|
222
|
+
return len(self.docs_collection) == 0
|
|
223
|
+
|
|
224
|
+
def clear(self) -> None:
|
|
225
|
+
self.docs_collection.clear()
|
|
226
|
+
|
|
227
|
+
def add_image(self, img: File | bytes | BytesIO | str | ImageObject) -> None:
|
|
228
|
+
if isinstance(img, File):
|
|
229
|
+
img = ImageObject(img)
|
|
230
|
+
elif isinstance(img, ImageObject):
|
|
231
|
+
pass
|
|
232
|
+
elif isinstance(img, bytes):
|
|
233
|
+
img = ImageObject(img)
|
|
234
|
+
elif isinstance(img, str):
|
|
235
|
+
img = ImageObject(img)
|
|
236
|
+
elif isinstance(img, BytesIO):
|
|
237
|
+
img = ImageObject(img)
|
|
238
|
+
|
|
239
|
+
if not img.metadata.file_path.is_empty:
|
|
240
|
+
self.docs_collection[img.metadata.file_path] = img
|
|
241
|
+
else:
|
|
242
|
+
self.docs_collection[img.metadata.name] = img
|
|
243
|
+
|
|
244
|
+
def add_images(self, images: list[File] | list[ImageObject] | list[bytes] | list[BytesIO]) -> None:
|
|
245
|
+
for f in images:
|
|
246
|
+
self.add_image(f)
|
|
247
|
+
|
|
248
|
+
def add_directory_images(self, dir_pdf: Directory) -> None:
|
|
249
|
+
files = InputFiles(dir_pdf).get_files(file_type=LibraryDocs.IMAGE)
|
|
250
|
+
if len(files) > 0:
|
|
251
|
+
self.add_images(files)
|
|
252
|
+
|
|
253
|
+
def find(
|
|
254
|
+
self, text: str,
|
|
255
|
+
separator: str = '\n',
|
|
256
|
+
iqual: bool = False,
|
|
257
|
+
case: bool = False,
|
|
258
|
+
silent: bool = False,
|
|
259
|
+
) -> SearchableTextImage:
|
|
260
|
+
"""
|
|
261
|
+
Filtrar texto retornando a primeira ocorrência do Documento PDF.
|
|
262
|
+
"""
|
|
263
|
+
_searchable = SearchableTextImage(silent)
|
|
264
|
+
if self.is_empty():
|
|
265
|
+
return _searchable
|
|
266
|
+
|
|
267
|
+
total_num: int = len(self.docs_collection.keys())
|
|
268
|
+
for n, file_key in enumerate(self.docs_collection.keys()):
|
|
269
|
+
self.pbar.update(
|
|
270
|
+
((n + 1) / total_num) * 100,
|
|
271
|
+
f'[PESQUISANDO]: {n+1}/{total_num}',
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
current_img: ImageObject = self.docs_collection[file_key]
|
|
275
|
+
txt_in_img = self.recognize.image_to_string(current_img)
|
|
276
|
+
if (txt_in_img == '') or (txt_in_img is None):
|
|
277
|
+
self.pbar.update_text(f'ERRO: {n+1} {current_img.metadata.name}')
|
|
278
|
+
continue
|
|
279
|
+
try:
|
|
280
|
+
fd = FindText(txt_in_img, separator=separator)
|
|
281
|
+
idx = fd.find_index(text, iqual=iqual, case=case)
|
|
282
|
+
if idx is None:
|
|
283
|
+
continue
|
|
284
|
+
math_text: str | None = fd.get_index(idx)
|
|
285
|
+
except Exception as err:
|
|
286
|
+
print(f'{__class__.__name__} {err}')
|
|
287
|
+
else:
|
|
288
|
+
new_line: dict[str, str] = {
|
|
289
|
+
ColumnsTable.KEY.value: f'{idx}',
|
|
290
|
+
ColumnsTable.NUM_PAGE.value: 'nan',
|
|
291
|
+
ColumnsTable.NUM_LINE.value: f'{idx + 1}',
|
|
292
|
+
ColumnsTable.TEXT.value: math_text,
|
|
293
|
+
ColumnsTable.FILE_NAME.value: current_img.metadata.file_path,
|
|
294
|
+
ColumnsTable.FILETYPE.value: current_img.metadata.extension,
|
|
295
|
+
ColumnsTable.FILE_PATH.value: current_img.metadata.dir_path,
|
|
296
|
+
ColumnsTable.DIR.value: current_img.metadata.dir_path,
|
|
297
|
+
}
|
|
298
|
+
_searchable.add_line(new_line)
|
|
299
|
+
return _searchable
|
|
300
|
+
return _searchable
|
|
301
|
+
|
|
302
|
+
def find_all(
|
|
303
|
+
self, text: str,
|
|
304
|
+
separator: str = '\n',
|
|
305
|
+
iqual: bool = False,
|
|
306
|
+
case: bool = False,
|
|
307
|
+
silent: bool = False,
|
|
308
|
+
) -> SearchableTextImage:
|
|
309
|
+
"""
|
|
310
|
+
Filtrar texto em documento PDF e retorna todas as ocorrências do texto
|
|
311
|
+
encontradas no documento, incluindo o número da linha, página e nome do arquivo
|
|
312
|
+
em cada ocorrência.
|
|
313
|
+
"""
|
|
314
|
+
_searchable = SearchableTextImage(silent)
|
|
315
|
+
if self.is_empty():
|
|
316
|
+
return _searchable
|
|
317
|
+
|
|
318
|
+
total_num: int = len(self.docs_collection.keys())
|
|
319
|
+
for n, file_key in enumerate(self.docs_collection.keys()):
|
|
320
|
+
self.pbar.update(
|
|
321
|
+
((n + 1) / total_num) * 100,
|
|
322
|
+
f'[PESQUISANDO]: {n + 1}/{total_num}',
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
current_img: ImageObject = self.docs_collection[file_key]
|
|
326
|
+
txt_in_img = self.recognize.image_to_string(current_img)
|
|
327
|
+
if (txt_in_img == '') or (txt_in_img is None):
|
|
328
|
+
self.pbar.update_text(f'ERRO: {n + 1} {current_img.metadata.name}')
|
|
329
|
+
continue
|
|
330
|
+
try:
|
|
331
|
+
fd = FindText(txt_in_img, separator=separator)
|
|
332
|
+
idx = fd.find_index(text, iqual=iqual, case=case)
|
|
333
|
+
if idx is None:
|
|
334
|
+
continue
|
|
335
|
+
math_text: str | None = fd.get_index(idx)
|
|
336
|
+
except Exception as err:
|
|
337
|
+
print(f'{__class__.__name__} {err}')
|
|
338
|
+
else:
|
|
339
|
+
new_line: dict[str, str] = {
|
|
340
|
+
ColumnsTable.KEY.value: f'{idx}',
|
|
341
|
+
ColumnsTable.NUM_PAGE.value: 'nan',
|
|
342
|
+
ColumnsTable.NUM_LINE.value: f'{idx + 1}',
|
|
343
|
+
ColumnsTable.TEXT.value: math_text,
|
|
344
|
+
ColumnsTable.FILE_NAME.value: current_img.metadata.file_path,
|
|
345
|
+
ColumnsTable.FILETYPE.value: current_img.metadata.extension,
|
|
346
|
+
ColumnsTable.FILE_PATH.value: current_img.metadata.dir_path,
|
|
347
|
+
ColumnsTable.DIR.value: current_img.metadata.dir_path,
|
|
348
|
+
}
|
|
349
|
+
_searchable.add_line(new_line)
|
|
350
|
+
return _searchable
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
__all__ = [
|
|
354
|
+
'ImageFinder', 'SearchableTextImage', 'read_images',
|
|
355
|
+
'read_document', 'RecognizeImage', 'RecognizePdf'
|
|
356
|
+
]
|
|
@@ -29,7 +29,7 @@ try:
|
|
|
29
29
|
except Exception as err:
|
|
30
30
|
print(err)
|
|
31
31
|
pyocr = object
|
|
32
|
-
|
|
32
|
+
|
|
33
33
|
|
|
34
34
|
if MOD_PYTESSERACT and MOD_PYOCR:
|
|
35
35
|
ModuleOcr: TypeAlias = Union[pyocr, pytesseract]
|
|
@@ -43,3 +43,6 @@ elif MOD_PYOCR:
|
|
|
43
43
|
else:
|
|
44
44
|
ModuleOcr: TypeAlias = None
|
|
45
45
|
DEFAULT_LIB_OCR = LibOcr.NOT_IMPLEMENTED
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
__all__ = ['LibOcr', 'DEFAULT_LIB_OCR']
|
|
@@ -7,10 +7,10 @@ from typing import Union
|
|
|
7
7
|
from io import BytesIO
|
|
8
8
|
from pandas import DataFrame
|
|
9
9
|
from soup_files import File
|
|
10
|
-
from convert_stream import
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
from convert_stream.mod_types import
|
|
10
|
+
from convert_stream import DocumentPdf, PageDocumentPdf
|
|
11
|
+
from convert_stream.mod_types.modules import DEFAULT_LIB_IMAGE, LibImage
|
|
12
|
+
from convert_stream.image import ImageObject
|
|
13
|
+
from convert_stream.mod_types import MetaDataItem, MetaDataFile
|
|
14
14
|
from convert_stream.mod_types.table_types import ColumnBody
|
|
15
15
|
from ocr_stream.modules import (
|
|
16
16
|
LibOcr, DEFAULT_LIB_OCR, pytesseract, Output, pyocr,
|
|
@@ -31,8 +31,9 @@ class TextRecognized(object):
|
|
|
31
31
|
"""
|
|
32
32
|
|
|
33
33
|
def __init__(self, bytes_recognized: bytes):
|
|
34
|
-
self.metadata:
|
|
34
|
+
self.metadata: MetaDataFile = MetaDataFile()
|
|
35
35
|
self.bytes_recognized: bytes = bytes_recognized
|
|
36
|
+
|
|
36
37
|
self.list_bad_char: list[str] = [
|
|
37
38
|
':', ',', ';', '$', '=',
|
|
38
39
|
'!', '}', '{', '(', ')',
|
|
@@ -51,7 +52,10 @@ class TextRecognized(object):
|
|
|
51
52
|
return doc.to_pages()[0]
|
|
52
53
|
|
|
53
54
|
def to_document(self) -> DocumentPdf:
|
|
54
|
-
|
|
55
|
+
tmp_doc = DocumentPdf(BytesIO(self.bytes_recognized))
|
|
56
|
+
tmp_doc.metadata.file_path = self.metadata.file_path
|
|
57
|
+
tmp_doc.metadata.dir_path = self.metadata.dir_path
|
|
58
|
+
return tmp_doc
|
|
55
59
|
|
|
56
60
|
def to_dict(self, separator: str = '\n') -> dict[str, ColumnBody]:
|
|
57
61
|
return self.to_page_pdf().to_dict(separator=separator)
|
|
@@ -205,3 +209,6 @@ class TesseractOcr(object):
|
|
|
205
209
|
else:
|
|
206
210
|
raise ValueError(f'{lib_ocr} is not supported')
|
|
207
211
|
return cls(md)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
__all__ = ['TesseractOcr', 'TextRecognized']
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
from __future__ import annotations
|
|
3
|
-
from convert_stream import
|
|
3
|
+
from convert_stream import DocumentPdf, PageDocumentPdf
|
|
4
|
+
from convert_stream.pdf import ConvertPdfToImages
|
|
5
|
+
from convert_stream.image import ImageObject
|
|
4
6
|
from ocr_stream.bin_tess import BinTesseract
|
|
5
7
|
from ocr_stream.modules import LibOcr, DEFAULT_LIB_OCR
|
|
6
8
|
from ocr_stream.ocr import TesseractOcr, TextRecognized
|
|
@@ -121,4 +123,3 @@ class RecognizePdf(object):
|
|
|
121
123
|
recognized_doc = self.recognize_document(file_doc, dpi=dpi)
|
|
122
124
|
del file_doc
|
|
123
125
|
return recognized_doc
|
|
124
|
-
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ocr_stream
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.8
|
|
4
4
|
Summary: Extrair textos de documentos digitalizados e imagens.
|
|
5
5
|
Author: Bruno Chaves
|
|
6
6
|
Requires-Python: >=3.11
|
|
@@ -12,7 +12,7 @@ Requires-Dist: pandas
|
|
|
12
12
|
Requires-Dist: requests>=2.32.4
|
|
13
13
|
Requires-Dist: opencv-python
|
|
14
14
|
Requires-Dist: pytesseract
|
|
15
|
-
Requires-Dist: convert_stream>=2.3.
|
|
15
|
+
Requires-Dist: convert_stream>=2.3.8
|
|
16
16
|
Requires-Dist: soup_files>=1.4
|
|
17
17
|
|
|
18
18
|
# ocrlib
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ocr_stream"
|
|
7
|
-
version = "2.3.
|
|
7
|
+
version = "2.3.8"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name="Bruno Chaves"},
|
|
10
10
|
]
|
|
@@ -21,7 +21,7 @@ dependencies = [
|
|
|
21
21
|
"requests>=2.32.4",
|
|
22
22
|
"opencv-python",
|
|
23
23
|
"pytesseract",
|
|
24
|
-
"convert_stream>=2.3.
|
|
24
|
+
"convert_stream>=2.3.8",
|
|
25
25
|
"soup_files>=1.4",
|
|
26
26
|
]
|
|
27
27
|
|
|
@@ -1,269 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
|
|
3
|
-
from ocr_stream.modules import *
|
|
4
|
-
from ocr_stream.bin_tess import BinTesseract, get_path_tesseract_sys
|
|
5
|
-
from ocr_stream.ocr import TesseractOcr, TextRecognized, DEFAULT_LIB_OCR
|
|
6
|
-
from ocr_stream.recognize import RecognizeImage, RecognizePdf
|
|
7
|
-
from soup_files import File
|
|
8
|
-
from convert_stream import (
|
|
9
|
-
ImageObject, DocumentPdf, ConvertPdfToImages, ColumnsTable, CollectionImages, FindText
|
|
10
|
-
)
|
|
11
|
-
from pandas import DataFrame
|
|
12
|
-
from soup_files import JsonConvert, ProgressBarAdapter, CreatePbar
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def read_images(
|
|
16
|
-
images: list[ImageObject], *,
|
|
17
|
-
pbar: ProgressBarAdapter = CreatePbar().get()
|
|
18
|
-
) -> DocumentPdf:
|
|
19
|
-
pbar.start()
|
|
20
|
-
print()
|
|
21
|
-
max_num = len(images)
|
|
22
|
-
pages = []
|
|
23
|
-
rec = RecognizeImage()
|
|
24
|
-
for idx, image in enumerate(images):
|
|
25
|
-
pbar.update(
|
|
26
|
-
((idx + 1) / max_num) * 100,
|
|
27
|
-
f'[OCR] {idx + 1} / {max_num}',
|
|
28
|
-
)
|
|
29
|
-
out = rec.image_recognize(image).to_page_pdf()
|
|
30
|
-
pages.append(out)
|
|
31
|
-
tmp_doc = DocumentPdf.create_from_pages(pages)
|
|
32
|
-
print()
|
|
33
|
-
pbar.stop()
|
|
34
|
-
return tmp_doc
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def read_document(
|
|
38
|
-
document: DocumentPdf, *,
|
|
39
|
-
dpi: int = 300,
|
|
40
|
-
pbar: ProgressBarAdapter = CreatePbar().get()
|
|
41
|
-
) -> DocumentPdf:
|
|
42
|
-
pbar.start()
|
|
43
|
-
print()
|
|
44
|
-
convert = ConvertPdfToImages.create(document)
|
|
45
|
-
convert.set_pbar(pbar)
|
|
46
|
-
images = convert.to_images(dpi=dpi)
|
|
47
|
-
print()
|
|
48
|
-
return read_images(images, pbar=pbar)
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
class SearchableTextImage(object):
|
|
52
|
-
"""
|
|
53
|
-
Esta classe serve para armazenar textos filtrados em documentos PDF
|
|
54
|
-
guardando o texto buscado e outros elementos como a página em que o texto foi
|
|
55
|
-
encontrado, linha e o nome do respetivo arquivo.
|
|
56
|
-
"""
|
|
57
|
-
def __init__(self, silent: bool = False):
|
|
58
|
-
self.silent = silent
|
|
59
|
-
self.elements: dict[str, list[str]] = {
|
|
60
|
-
ColumnsTable.NUM_LINE.value: [],
|
|
61
|
-
ColumnsTable.TEXT.value: [],
|
|
62
|
-
ColumnsTable.FILE_PATH.value: [],
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
def __repr__(self):
|
|
66
|
-
return f'{__class__.__name__}: {self.elements}'
|
|
67
|
-
|
|
68
|
-
def is_empty(self) -> bool:
|
|
69
|
-
return len(self.elements[ColumnsTable.TEXT.value]) == 0
|
|
70
|
-
|
|
71
|
-
@property
|
|
72
|
-
def first(self) -> dict[str, str]:
|
|
73
|
-
if self.is_empty():
|
|
74
|
-
return {}
|
|
75
|
-
return {
|
|
76
|
-
ColumnsTable.NUM_LINE.value: self.elements[ColumnsTable.NUM_LINE.value][0],
|
|
77
|
-
ColumnsTable.TEXT.value: self.elements[ColumnsTable.TEXT.value][0],
|
|
78
|
-
ColumnsTable.FILE_PATH.value: self.elements[ColumnsTable.FILE_PATH.value][0],
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
@property
|
|
82
|
-
def last(self) -> dict[str, str]:
|
|
83
|
-
if self.is_empty():
|
|
84
|
-
return {}
|
|
85
|
-
return {
|
|
86
|
-
ColumnsTable.NUM_LINE.value: self.elements[ColumnsTable.NUM_LINE.value][-1],
|
|
87
|
-
ColumnsTable.TEXT.value: self.elements[ColumnsTable.TEXT.value][-1],
|
|
88
|
-
ColumnsTable.FILE_PATH.value: self.elements[ColumnsTable.FILE_PATH.value][-1],
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
@property
|
|
92
|
-
def length(self) -> int:
|
|
93
|
-
return len(self.elements[ColumnsTable.TEXT.value])
|
|
94
|
-
|
|
95
|
-
@property
|
|
96
|
-
def files(self) -> list[str]:
|
|
97
|
-
if self.is_empty():
|
|
98
|
-
return []
|
|
99
|
-
return self.elements[ColumnsTable.FILE_PATH.value]
|
|
100
|
-
|
|
101
|
-
def get_item(self, idx: int) -> dict[str, str]:
|
|
102
|
-
try:
|
|
103
|
-
return {
|
|
104
|
-
ColumnsTable.NUM_LINE.value: self.elements[ColumnsTable.NUM_LINE.value][idx],
|
|
105
|
-
ColumnsTable.TEXT.value: self.elements[ColumnsTable.TEXT.value][idx],
|
|
106
|
-
ColumnsTable.FILE_PATH.value: self.elements[ColumnsTable.FILE_PATH.value][idx],
|
|
107
|
-
}
|
|
108
|
-
except Exception as err:
|
|
109
|
-
if not self.silent:
|
|
110
|
-
print(err)
|
|
111
|
-
return {}
|
|
112
|
-
|
|
113
|
-
def add_line(self, text, *, num_line: str, file: str) -> None:
|
|
114
|
-
if not self.silent:
|
|
115
|
-
print('----------------------')
|
|
116
|
-
print(f'{file} => {text}')
|
|
117
|
-
self.elements[ColumnsTable.NUM_LINE.value].append(num_line)
|
|
118
|
-
self.elements[ColumnsTable.TEXT.value].append(text)
|
|
119
|
-
self.elements[ColumnsTable.FILE_PATH.value].append(file)
|
|
120
|
-
|
|
121
|
-
def clear(self) -> None:
|
|
122
|
-
for _k in self.elements.keys():
|
|
123
|
-
self.elements[_k].clear()
|
|
124
|
-
|
|
125
|
-
def to_string(self) -> str:
|
|
126
|
-
"""
|
|
127
|
-
Retorna o texto da coluna TEXT em formato de string
|
|
128
|
-
ou 'nas' em caso de erro nas = Not a String
|
|
129
|
-
"""
|
|
130
|
-
try:
|
|
131
|
-
return ' '.join(self.elements[ColumnsTable.TEXT.value])
|
|
132
|
-
except Exception as e:
|
|
133
|
-
print(e)
|
|
134
|
-
return 'nas'
|
|
135
|
-
|
|
136
|
-
def to_data_frame(self) -> DataFrame:
|
|
137
|
-
return DataFrame.from_dict(self.elements)
|
|
138
|
-
|
|
139
|
-
def to_file_json(self, file: File):
|
|
140
|
-
"""Exporta os dados da busca para arquivo .JSON"""
|
|
141
|
-
dt = JsonConvert.from_dict(self.elements).to_json_data()
|
|
142
|
-
dt.to_file(file)
|
|
143
|
-
|
|
144
|
-
def to_file_excel(self, file: File):
|
|
145
|
-
"""Exporta os dados da busca para um arquivo .XLSX"""
|
|
146
|
-
try:
|
|
147
|
-
self.to_data_frame().to_excel(file.absolute())
|
|
148
|
-
except Exception as e:
|
|
149
|
-
print(e)
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
class FindTextImages(object):
|
|
153
|
-
|
|
154
|
-
def __init__(
|
|
155
|
-
self, collection_images: CollectionImages,
|
|
156
|
-
*,
|
|
157
|
-
rec_img: RecognizeImage = RecognizeImage(),
|
|
158
|
-
pbar: ProgressBarAdapter = CreatePbar().get(),
|
|
159
|
-
):
|
|
160
|
-
self.collection_images: CollectionImages = collection_images
|
|
161
|
-
self._rec_img: RecognizeImage = rec_img
|
|
162
|
-
self.pbar: ProgressBarAdapter = pbar
|
|
163
|
-
|
|
164
|
-
def find(
|
|
165
|
-
self, text: str,
|
|
166
|
-
separator: str = '\n',
|
|
167
|
-
iqual: bool = False,
|
|
168
|
-
case: bool = False,
|
|
169
|
-
silent: bool = False,
|
|
170
|
-
) -> SearchableTextImage:
|
|
171
|
-
"""
|
|
172
|
-
Filtrar texto retornando a primeira ocorrência do Documento PDF.
|
|
173
|
-
"""
|
|
174
|
-
print()
|
|
175
|
-
self.pbar.start()
|
|
176
|
-
_searchable = SearchableTextImage(silent)
|
|
177
|
-
|
|
178
|
-
max_num: int = self.collection_images.length
|
|
179
|
-
for idx, image in enumerate(self.collection_images):
|
|
180
|
-
self.pbar.update(
|
|
181
|
-
((idx + 1) / max_num) * 100,
|
|
182
|
-
f'[Pesquisando]: {idx + 1} / {max_num} Aguarde...',
|
|
183
|
-
)
|
|
184
|
-
text_str_in_page: str = self._rec_img.image_to_string(image)
|
|
185
|
-
|
|
186
|
-
if (text_str_in_page == 'nas') or (text_str_in_page is None):
|
|
187
|
-
continue
|
|
188
|
-
try:
|
|
189
|
-
fd = FindText(text_str_in_page, separator=separator)
|
|
190
|
-
idx = fd.find_index(text, iqual=iqual, case=case)
|
|
191
|
-
if idx is None:
|
|
192
|
-
continue
|
|
193
|
-
math_text = fd.get_index(idx)
|
|
194
|
-
except Exception as err:
|
|
195
|
-
print(f'{__class__.__name__} {err}')
|
|
196
|
-
else:
|
|
197
|
-
_searchable.add_line(
|
|
198
|
-
math_text,
|
|
199
|
-
num_line=str(idx + 1),
|
|
200
|
-
file=image.name,
|
|
201
|
-
)
|
|
202
|
-
return _searchable
|
|
203
|
-
self.pbar.stop()
|
|
204
|
-
print()
|
|
205
|
-
return _searchable
|
|
206
|
-
|
|
207
|
-
def find_all(
|
|
208
|
-
self, text: str,
|
|
209
|
-
separator: str = '\n',
|
|
210
|
-
iqual: bool = False,
|
|
211
|
-
case: bool = False,
|
|
212
|
-
silent: bool = False,
|
|
213
|
-
) -> SearchableTextImage:
|
|
214
|
-
"""
|
|
215
|
-
Filtrar texto em documento PDF e retorna todas as ocorrências do texto
|
|
216
|
-
encontradas no documento, incluindo o número da linha, página e nome do arquivo
|
|
217
|
-
em cada ocorrência.
|
|
218
|
-
"""
|
|
219
|
-
print()
|
|
220
|
-
self.pbar.start()
|
|
221
|
-
_searchable = SearchableTextImage(silent)
|
|
222
|
-
max_num: int = self.collection_images.length
|
|
223
|
-
for idx, image in enumerate(self.collection_images):
|
|
224
|
-
self.pbar.update(
|
|
225
|
-
((idx + 1) / max_num) * 100,
|
|
226
|
-
f'[Pesquisando]: {idx + 1} / {max_num} Aguarde...',
|
|
227
|
-
)
|
|
228
|
-
|
|
229
|
-
text_str_in_page = self._rec_img.image_to_string(image)
|
|
230
|
-
if (text_str_in_page == 'nas') or (text_str_in_page is None):
|
|
231
|
-
continue
|
|
232
|
-
try:
|
|
233
|
-
_values = text_str_in_page.split(separator)
|
|
234
|
-
for num, item in enumerate(_values):
|
|
235
|
-
if case:
|
|
236
|
-
if iqual:
|
|
237
|
-
if text == item:
|
|
238
|
-
_searchable.add_line(
|
|
239
|
-
item,
|
|
240
|
-
num_line=str(num + 1),
|
|
241
|
-
file=image.name,
|
|
242
|
-
)
|
|
243
|
-
else:
|
|
244
|
-
if text in item:
|
|
245
|
-
_searchable.add_line(
|
|
246
|
-
item,
|
|
247
|
-
num_line=str(num + 1),
|
|
248
|
-
file=image.name,
|
|
249
|
-
)
|
|
250
|
-
else:
|
|
251
|
-
if iqual:
|
|
252
|
-
if text.lower() == item.lower():
|
|
253
|
-
_searchable.add_line(
|
|
254
|
-
item,
|
|
255
|
-
num_line=str(num + 1),
|
|
256
|
-
file=image.name,
|
|
257
|
-
)
|
|
258
|
-
else:
|
|
259
|
-
if text.lower() in item.lower():
|
|
260
|
-
_searchable.add_line(
|
|
261
|
-
item,
|
|
262
|
-
num_line=str(num + 1),
|
|
263
|
-
file=image.name,
|
|
264
|
-
)
|
|
265
|
-
except Exception as e:
|
|
266
|
-
print(f'{__class__.__name__} {e}')
|
|
267
|
-
self.pbar.stop()
|
|
268
|
-
print()
|
|
269
|
-
return _searchable
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|