czytajpdf 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- czytajpdf-0.3.0/LICENSE +202 -0
- czytajpdf-0.3.0/PKG-INFO +119 -0
- czytajpdf-0.3.0/README.md +87 -0
- czytajpdf-0.3.0/pyproject.toml +25 -0
- czytajpdf-0.3.0/setup.cfg +4 -0
- czytajpdf-0.3.0/src/czytajpdf/__init__.py +67 -0
- czytajpdf-0.3.0/src/czytajpdf/_ocr.py +72 -0
- czytajpdf-0.3.0/src/czytajpdf/_spell.py +241 -0
- czytajpdf-0.3.0/src/czytajpdf/_tesseract.py +48 -0
- czytajpdf-0.3.0/src/czytajpdf.egg-info/PKG-INFO +119 -0
- czytajpdf-0.3.0/src/czytajpdf.egg-info/SOURCES.txt +13 -0
- czytajpdf-0.3.0/src/czytajpdf.egg-info/dependency_links.txt +1 -0
- czytajpdf-0.3.0/src/czytajpdf.egg-info/requires.txt +24 -0
- czytajpdf-0.3.0/src/czytajpdf.egg-info/top_level.txt +1 -0
- czytajpdf-0.3.0/tests/test_czytajpdf.py +195 -0
czytajpdf-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
czytajpdf-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: czytajpdf
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Page-wise Polish and English PDF text extraction with optional OCR and spelling repair
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
Project-URL: Homepage, https://github.com/praha6248/CzytajPDF
|
|
7
|
+
Project-URL: Repository, https://github.com/praha6248/CzytajPDF
|
|
8
|
+
Project-URL: Issues, https://github.com/praha6248/CzytajPDF/issues
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: pypdf>=5
|
|
13
|
+
Provides-Extra: ocr
|
|
14
|
+
Requires-Dist: tesserocr<3,>=2.8; extra == "ocr"
|
|
15
|
+
Requires-Dist: pooch<2,>=1.8; extra == "ocr"
|
|
16
|
+
Requires-Dist: rapidocr<4,>=3.10; extra == "ocr"
|
|
17
|
+
Requires-Dist: onnxruntime>=1.20; extra == "ocr"
|
|
18
|
+
Requires-Dist: pypdfium2>=4.30; extra == "ocr"
|
|
19
|
+
Requires-Dist: numpy>=1.24; extra == "ocr"
|
|
20
|
+
Requires-Dist: pillow>=10; extra == "ocr"
|
|
21
|
+
Provides-Extra: polish
|
|
22
|
+
Requires-Dist: morfeusz2>=1.99; extra == "polish"
|
|
23
|
+
Requires-Dist: wordfreq>=3.1; extra == "polish"
|
|
24
|
+
Provides-Extra: pl
|
|
25
|
+
Requires-Dist: morfeusz2>=1.99; extra == "pl"
|
|
26
|
+
Requires-Dist: wordfreq>=3.1; extra == "pl"
|
|
27
|
+
Provides-Extra: english
|
|
28
|
+
Requires-Dist: wordfreq>=3.1; extra == "english"
|
|
29
|
+
Provides-Extra: en
|
|
30
|
+
Requires-Dist: wordfreq>=3.1; extra == "en"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# CzytajPDF
|
|
34
|
+
|
|
35
|
+
[](https://github.com/praha6248/CzytajPDF/actions/workflows/ci.yml)
|
|
36
|
+
|
|
37
|
+
**Polskie i angielskie PDF-y oraz skany w jednym wywołaniu Pythona.** CzytajPDF odczytuje dokument strona po stronie. Gdy pismo ma warstwę tekstową, a załącznik jest skanem, dostajesz jeden wynik z informacją, skąd pochodzi tekst każdej strony. Opcjonalna korekta przywraca polskie znaki lub naprawia typowe błędy OCR po angielsku, zachowując oryginalny odczyt do porównania.
|
|
38
|
+
|
|
39
|
+
> **English summary:** Page-wise Python library (Apache-2.0) for hybrid Polish and English PDFs — reads embedded text via `pypdf`, automatically falls back to local OCR (`tesserocr` / `rapidocr`) on scanned pages, and optionally applies Polish (`morfeusz2` + `wordfreq`) or English (`wordfreq`) OCR repair with dual-resolution reconciliation while preserving `source` and `raw_text`. Also packaged as `readpdf-pl` and `readpdf-en` from the same repository. See [README.en.md](README.en.md).
|
|
40
|
+
|
|
41
|
+
## Pakiety w repozytorium
|
|
42
|
+
|
|
43
|
+
Jedno repozytorium dostarcza wspólną bibliotekę bazową oraz dwie paczki językowe o prostszej konfiguracji domyślnej:
|
|
44
|
+
|
|
45
|
+
- **`czytajpdf`** — pełna biblioteka z parametrem `lang="pl"` (domyślnie) lub `lang="en"`
|
|
46
|
+
- **`readpdf-pl`** (`packages/readpdf-pl`) — paczka dla dokumentów polskich (domyślnie `lang="pl"`)
|
|
47
|
+
- **`readpdf-en`** (`packages/readpdf-en`) — paczka dla dokumentów angielskich (domyślnie `lang="en"`, bez zależności od `morfeusz2`)
|
|
48
|
+
|
|
49
|
+
## Pierwszy odczyt
|
|
50
|
+
|
|
51
|
+
Python 3.10+. Instalacja z GitHuba z obsługą OCR i korekty:
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
# Polski i angielski (pełna instalacja)
|
|
55
|
+
python -m pip install 'czytajpdf[ocr,polish] @ git+https://github.com/praha6248/CzytajPDF.git'
|
|
56
|
+
|
|
57
|
+
# Tylko angielski (bez morfeusz2)
|
|
58
|
+
python -m pip install 'czytajpdf[ocr,english] @ git+https://github.com/praha6248/CzytajPDF.git'
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from czytajpdf import extract
|
|
63
|
+
|
|
64
|
+
dokument = extract("pismo-z-zalacznikami.pdf", ocr=True, correct=True)
|
|
65
|
+
for strona in dokument.pages:
|
|
66
|
+
print(strona.number, strona.source, strona.text)
|
|
67
|
+
|
|
68
|
+
english_doc = extract("filing-with-exhibits.pdf", ocr=True, correct=True, lang="en")
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Do PDF-ów z warstwą tekstową wystarczy instalacja bez dodatków i wywołanie `extract("pismo.pdf")`. Cały tekst jest dostępny jako `dokument.text`. Numeracja stron zaczyna się od 1. Z paczek językowych można korzystać przez `from readpdf_pl import extract` oraz `from readpdf_en import extract`.
|
|
72
|
+
|
|
73
|
+
## Statystyki odczytu
|
|
74
|
+
|
|
75
|
+
| Dokument | Język | Strony | Strony z warstwy tekstowej | Strony z OCR | Znaki z OCR |
|
|
76
|
+
| --- | --- | ---: | ---: | ---: | ---: |
|
|
77
|
+
| [Dz.U. 2024 poz. 1](https://api.sejm.gov.pl/eli/acts/DU/2024/1/text.pdf) | `pl` | 8 | 8 | 0 | 0 |
|
|
78
|
+
| [Dz.U. 1945 nr 5 poz. 17](https://api.sejm.gov.pl/eli/acts/DU/1945/17/text.pdf) | `pl` | 2 | 2 | 0 | 0 |
|
|
79
|
+
| PDF mieszany z pierwszych stron obu polskich dokumentów | `pl` | 2 | 1 | 1 | 2716 |
|
|
80
|
+
| [SCOTUS No. 22-451](https://www.supremecourt.gov/opinions/23pdf/22-451_7m58.pdf) | `en` | 114 | 114 | 0 | 0 |
|
|
81
|
+
| [Brown v. Board of Education, 347 U.S. 483](https://cdn.loc.gov/service/ll/usrep/usrep347/usrep347483/usrep347483.pdf) | `en` | 14 | 14 | 0 | 0 |
|
|
82
|
+
| PDF mieszany z pierwszych stron obu angielskich dokumentów | `en` | 2 | 1 | 1 | 2204 |
|
|
83
|
+
|
|
84
|
+
## Skuteczność korekty polskiego OCR
|
|
85
|
+
|
|
86
|
+
Pomiar obejmuje nagłówki z 18 archiwalnych skanów Dziennika Ustaw. Na pierwotnej próbce 12 nagłówków **liczba błędów słów spadła z 79 do 2**. Do porównania użyto tych samych wycinków i transkrypcji.
|
|
87
|
+
|
|
88
|
+
| Próbka i wersja biblioteki | Błędy znaków (CER) po korekcie | Błędy słów (WER) po korekcie |
|
|
89
|
+
| --- | ---: | ---: |
|
|
90
|
+
| 12 pierwotnych nagłówków · 0.1.0, RapidOCR | 382 / 2129 · 17,94% | 79 / 293 · 26,96% |
|
|
91
|
+
| Te same 12 nagłówków · 0.2.0 / 0.3.0, Tesseract | 2 / 2129 · 0,09% | 2 / 293 · 0,68% |
|
|
92
|
+
| 3 dodatkowe nagłówki użyte do sprawdzenia i poprawienia obsługi dzielenia słów · 0.2.0 / 0.3.0 | 0 / 593 · 0,00% | 0 / 87 · 0,00% |
|
|
93
|
+
| 3 osobne nagłówki użyte do końcowej weryfikacji · 0.2.0 / 0.3.0 | 1 / 430 · 0,23% | 1 / 67 · 1,49% |
|
|
94
|
+
|
|
95
|
+
Na pierwotnej próbce surowy Tesseract popełnił 5 błędów słów, a drugi odczyt z korektą usunął 3. Pozostały pomyłki `r.` → `r,` oraz `i` → `1`. W końcowej weryfikacji OCR dodał kropkę po słowie `lutego`.
|
|
96
|
+
|
|
97
|
+
CER i WER uwzględniają zamiany, wstawienia i pominięcia względem transkrypcji z obrazu. Ujednolicono białe znaki, zachowując interpunkcję i wielkość liter. Wynik dotyczy wycinków z drukowanymi nagłówkami. [Pomiar 0.1.0](benchmarks/rapidocr-0.1.0.json) · [Pomiar 0.2.0/0.3.0](benchmarks/results.json) · [Próbka dodatkowa](benchmarks/holdout.json) · [Weryfikacja końcowa](benchmarks/validation.json) · [Skrypt pomiarowy](benchmarks/score.py)
|
|
98
|
+
|
|
99
|
+
## Skuteczność angielskiego OCR (`lang="en"`)
|
|
100
|
+
|
|
101
|
+
Pomiar obejmuje 9 archiwalnych skanów *United States Reports* z Library of Congress (lata 1937–1989). Na 6 dokumentach próbki głównej korekta naprawiła błędy `Amendfnent` → `Amendment` oraz `eriminal` → `criminal` (spadek WER z 17 do 15 na 897 słowach), a na 3 osobnych dokumentach walidacyjnych nie wprowadziła żadnych fałszywych poprawek.
|
|
102
|
+
|
|
103
|
+
| Próbka angielska (0.3.0, Tesseract `eng`) | Surowy CER / WER | Po korekcie CER | Po korekcie WER |
|
|
104
|
+
| --- | ---: | ---: | ---: |
|
|
105
|
+
| 6 skanów *U.S. Reports* (1937–1984) | 0,35% / 1,90% (17 / 897) | 17 / 5745 · 0,30% | 15 / 897 · 1,67% |
|
|
106
|
+
| 3 osobne skany walidacyjne (1943, 1964, 1989) | 0,32% / 1,99% (6 / 301) | 6 / 1871 · 0,32% | 6 / 301 · 1,99% |
|
|
107
|
+
|
|
108
|
+
[Pomiar angielski](benchmarks/english_results.json) · [Walidacja angielska](benchmarks/english_validation.json)
|
|
109
|
+
|
|
110
|
+
## Jak wybierany jest sposób odczytu
|
|
111
|
+
|
|
112
|
+
- Domyślnie działa pypdf. Przy `ocr=True` strony z mniej niż 40 znakami po pominięciu białych znaków trafiają do modelu Tesseract dla wybranego języka (`lang="pl"` albo `lang="en"`). Aby wybrać RapidOCR, przekaż `engine="rapidocr"`. Próg zmienisz przez `min_chars`. Długa, uszkodzona warstwa tekstowa może przejść ten próg.
|
|
113
|
+
- `source` przyjmuje wartość `pdf`, `tesseract` lub `rapidocr`. Pusty wynik OCR pozostawia tekst z warstwy PDF. Wynik zachowuje podział na strony, bez współrzędnych słów ani struktury tabel.
|
|
114
|
+
- `correct=True` włącza korektę wyników OCR: dla `lang="pl"` używa `wordfreq` i `Morfeusza 2`, a dla `lang="en"` `wordfreq` i ostrożnych zamian typowych pomyłek OCR. Tesseract może ponownie odczytać stronę w rozdzielczości 180 DPI, aby zweryfikować nierozpoznane słowa. Pierwszy odczyt pozostaje dostępny w `raw_text`. Przy nazwiskach i nazwach własnych warto porównać wynik z `raw_text`.
|
|
115
|
+
- Rozpoznawanie odbywa się lokalnie. Przy pierwszym użyciu Tesseract pobiera odpowiedni model `tessdata_best` (około 12 MB dla `pol` lub 15 MB dla `eng`), sprawdza SHA-256 i zapisuje plik w pamięci podręcznej. Kolejne odczyty działają offline. RapidOCR korzysta z osobnych modeli.
|
|
116
|
+
|
|
117
|
+
## Licencja i pochodzenie
|
|
118
|
+
|
|
119
|
+
Kod biblioteki jest dostępny na [Apache-2.0](LICENSE). Projekt wykorzystuje logikę z [odwolania2json](https://github.com/praha6248/odwolania2Json). Wagi OCR, słowniki i dane częstości są osobnymi zależnościami na własnych licencjach.
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# CzytajPDF
|
|
2
|
+
|
|
3
|
+
[](https://github.com/praha6248/CzytajPDF/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
**Polskie i angielskie PDF-y oraz skany w jednym wywołaniu Pythona.** CzytajPDF odczytuje dokument strona po stronie. Gdy pismo ma warstwę tekstową, a załącznik jest skanem, dostajesz jeden wynik z informacją, skąd pochodzi tekst każdej strony. Opcjonalna korekta przywraca polskie znaki lub naprawia typowe błędy OCR po angielsku, zachowując oryginalny odczyt do porównania.
|
|
6
|
+
|
|
7
|
+
> **English summary:** Page-wise Python library (Apache-2.0) for hybrid Polish and English PDFs — reads embedded text via `pypdf`, automatically falls back to local OCR (`tesserocr` / `rapidocr`) on scanned pages, and optionally applies Polish (`morfeusz2` + `wordfreq`) or English (`wordfreq`) OCR repair with dual-resolution reconciliation while preserving `source` and `raw_text`. Also packaged as `readpdf-pl` and `readpdf-en` from the same repository. See [README.en.md](README.en.md).
|
|
8
|
+
|
|
9
|
+
## Pakiety w repozytorium
|
|
10
|
+
|
|
11
|
+
Jedno repozytorium dostarcza wspólną bibliotekę bazową oraz dwie paczki językowe o prostszej konfiguracji domyślnej:
|
|
12
|
+
|
|
13
|
+
- **`czytajpdf`** — pełna biblioteka z parametrem `lang="pl"` (domyślnie) lub `lang="en"`
|
|
14
|
+
- **`readpdf-pl`** (`packages/readpdf-pl`) — paczka dla dokumentów polskich (domyślnie `lang="pl"`)
|
|
15
|
+
- **`readpdf-en`** (`packages/readpdf-en`) — paczka dla dokumentów angielskich (domyślnie `lang="en"`, bez zależności od `morfeusz2`)
|
|
16
|
+
|
|
17
|
+
## Pierwszy odczyt
|
|
18
|
+
|
|
19
|
+
Python 3.10+. Instalacja z GitHuba z obsługą OCR i korekty:
|
|
20
|
+
|
|
21
|
+
```sh
|
|
22
|
+
# Polski i angielski (pełna instalacja)
|
|
23
|
+
python -m pip install 'czytajpdf[ocr,polish] @ git+https://github.com/praha6248/CzytajPDF.git'
|
|
24
|
+
|
|
25
|
+
# Tylko angielski (bez morfeusz2)
|
|
26
|
+
python -m pip install 'czytajpdf[ocr,english] @ git+https://github.com/praha6248/CzytajPDF.git'
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from czytajpdf import extract
|
|
31
|
+
|
|
32
|
+
dokument = extract("pismo-z-zalacznikami.pdf", ocr=True, correct=True)
|
|
33
|
+
for strona in dokument.pages:
|
|
34
|
+
print(strona.number, strona.source, strona.text)
|
|
35
|
+
|
|
36
|
+
english_doc = extract("filing-with-exhibits.pdf", ocr=True, correct=True, lang="en")
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Do PDF-ów z warstwą tekstową wystarczy instalacja bez dodatków i wywołanie `extract("pismo.pdf")`. Cały tekst jest dostępny jako `dokument.text`. Numeracja stron zaczyna się od 1. Z paczek językowych można korzystać przez `from readpdf_pl import extract` oraz `from readpdf_en import extract`.
|
|
40
|
+
|
|
41
|
+
## Statystyki odczytu
|
|
42
|
+
|
|
43
|
+
| Dokument | Język | Strony | Strony z warstwy tekstowej | Strony z OCR | Znaki z OCR |
|
|
44
|
+
| --- | --- | ---: | ---: | ---: | ---: |
|
|
45
|
+
| [Dz.U. 2024 poz. 1](https://api.sejm.gov.pl/eli/acts/DU/2024/1/text.pdf) | `pl` | 8 | 8 | 0 | 0 |
|
|
46
|
+
| [Dz.U. 1945 nr 5 poz. 17](https://api.sejm.gov.pl/eli/acts/DU/1945/17/text.pdf) | `pl` | 2 | 2 | 0 | 0 |
|
|
47
|
+
| PDF mieszany z pierwszych stron obu polskich dokumentów | `pl` | 2 | 1 | 1 | 2716 |
|
|
48
|
+
| [SCOTUS No. 22-451](https://www.supremecourt.gov/opinions/23pdf/22-451_7m58.pdf) | `en` | 114 | 114 | 0 | 0 |
|
|
49
|
+
| [Brown v. Board of Education, 347 U.S. 483](https://cdn.loc.gov/service/ll/usrep/usrep347/usrep347483/usrep347483.pdf) | `en` | 14 | 14 | 0 | 0 |
|
|
50
|
+
| PDF mieszany z pierwszych stron obu angielskich dokumentów | `en` | 2 | 1 | 1 | 2204 |
|
|
51
|
+
|
|
52
|
+
## Skuteczność korekty polskiego OCR
|
|
53
|
+
|
|
54
|
+
Pomiar obejmuje nagłówki z 18 archiwalnych skanów Dziennika Ustaw. Na pierwotnej próbce 12 nagłówków **liczba błędów słów spadła z 79 do 2**. Do porównania użyto tych samych wycinków i transkrypcji.
|
|
55
|
+
|
|
56
|
+
| Próbka i wersja biblioteki | Błędy znaków (CER) po korekcie | Błędy słów (WER) po korekcie |
|
|
57
|
+
| --- | ---: | ---: |
|
|
58
|
+
| 12 pierwotnych nagłówków · 0.1.0, RapidOCR | 382 / 2129 · 17,94% | 79 / 293 · 26,96% |
|
|
59
|
+
| Te same 12 nagłówków · 0.2.0 / 0.3.0, Tesseract | 2 / 2129 · 0,09% | 2 / 293 · 0,68% |
|
|
60
|
+
| 3 dodatkowe nagłówki użyte do sprawdzenia i poprawienia obsługi dzielenia słów · 0.2.0 / 0.3.0 | 0 / 593 · 0,00% | 0 / 87 · 0,00% |
|
|
61
|
+
| 3 osobne nagłówki użyte do końcowej weryfikacji · 0.2.0 / 0.3.0 | 1 / 430 · 0,23% | 1 / 67 · 1,49% |
|
|
62
|
+
|
|
63
|
+
Na pierwotnej próbce surowy Tesseract popełnił 5 błędów słów, a drugi odczyt z korektą usunął 3. Pozostały pomyłki `r.` → `r,` oraz `i` → `1`. W końcowej weryfikacji OCR dodał kropkę po słowie `lutego`.
|
|
64
|
+
|
|
65
|
+
CER i WER uwzględniają zamiany, wstawienia i pominięcia względem transkrypcji z obrazu. Ujednolicono białe znaki, zachowując interpunkcję i wielkość liter. Wynik dotyczy wycinków z drukowanymi nagłówkami. [Pomiar 0.1.0](benchmarks/rapidocr-0.1.0.json) · [Pomiar 0.2.0/0.3.0](benchmarks/results.json) · [Próbka dodatkowa](benchmarks/holdout.json) · [Weryfikacja końcowa](benchmarks/validation.json) · [Skrypt pomiarowy](benchmarks/score.py)
|
|
66
|
+
|
|
67
|
+
## Skuteczność angielskiego OCR (`lang="en"`)
|
|
68
|
+
|
|
69
|
+
Pomiar obejmuje 9 archiwalnych skanów *United States Reports* z Library of Congress (lata 1937–1989). Na 6 dokumentach próbki głównej korekta naprawiła błędy `Amendfnent` → `Amendment` oraz `eriminal` → `criminal` (spadek WER z 17 do 15 na 897 słowach), a na 3 osobnych dokumentach walidacyjnych nie wprowadziła żadnych fałszywych poprawek.
|
|
70
|
+
|
|
71
|
+
| Próbka angielska (0.3.0, Tesseract `eng`) | Surowy CER / WER | Po korekcie CER | Po korekcie WER |
|
|
72
|
+
| --- | ---: | ---: | ---: |
|
|
73
|
+
| 6 skanów *U.S. Reports* (1937–1984) | 0,35% / 1,90% (17 / 897) | 17 / 5745 · 0,30% | 15 / 897 · 1,67% |
|
|
74
|
+
| 3 osobne skany walidacyjne (1943, 1964, 1989) | 0,32% / 1,99% (6 / 301) | 6 / 1871 · 0,32% | 6 / 301 · 1,99% |
|
|
75
|
+
|
|
76
|
+
[Pomiar angielski](benchmarks/english_results.json) · [Walidacja angielska](benchmarks/english_validation.json)
|
|
77
|
+
|
|
78
|
+
## Jak wybierany jest sposób odczytu
|
|
79
|
+
|
|
80
|
+
- Domyślnie działa pypdf. Przy `ocr=True` strony z mniej niż 40 znakami po pominięciu białych znaków trafiają do modelu Tesseract dla wybranego języka (`lang="pl"` albo `lang="en"`). Aby wybrać RapidOCR, przekaż `engine="rapidocr"`. Próg zmienisz przez `min_chars`. Długa, uszkodzona warstwa tekstowa może przejść ten próg.
|
|
81
|
+
- `source` przyjmuje wartość `pdf`, `tesseract` lub `rapidocr`. Pusty wynik OCR pozostawia tekst z warstwy PDF. Wynik zachowuje podział na strony, bez współrzędnych słów ani struktury tabel.
|
|
82
|
+
- `correct=True` włącza korektę wyników OCR: dla `lang="pl"` używa `wordfreq` i `Morfeusza 2`, a dla `lang="en"` `wordfreq` i ostrożnych zamian typowych pomyłek OCR. Tesseract może ponownie odczytać stronę w rozdzielczości 180 DPI, aby zweryfikować nierozpoznane słowa. Pierwszy odczyt pozostaje dostępny w `raw_text`. Przy nazwiskach i nazwach własnych warto porównać wynik z `raw_text`.
|
|
83
|
+
- Rozpoznawanie odbywa się lokalnie. Przy pierwszym użyciu Tesseract pobiera odpowiedni model `tessdata_best` (około 12 MB dla `pol` lub 15 MB dla `eng`), sprawdza SHA-256 i zapisuje plik w pamięci podręcznej. Kolejne odczyty działają offline. RapidOCR korzysta z osobnych modeli.
|
|
84
|
+
|
|
85
|
+
## Licencja i pochodzenie
|
|
86
|
+
|
|
87
|
+
Kod biblioteki jest dostępny na [Apache-2.0](LICENSE). Projekt wykorzystuje logikę z [odwolania2json](https://github.com/praha6248/odwolania2Json). Wagi OCR, słowniki i dane częstości są osobnymi zależnościami na własnych licencjach.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "czytajpdf"
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
description = "Page-wise Polish and English PDF text extraction with optional OCR and spelling repair"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "Apache-2.0"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
dependencies = ["pypdf>=5"]
|
|
14
|
+
|
|
15
|
+
[project.optional-dependencies]
|
|
16
|
+
ocr = ["tesserocr>=2.8,<3", "pooch>=1.8,<2", "rapidocr>=3.10,<4", "onnxruntime>=1.20", "pypdfium2>=4.30", "numpy>=1.24", "pillow>=10"]
|
|
17
|
+
polish = ["morfeusz2>=1.99", "wordfreq>=3.1"]
|
|
18
|
+
pl = ["morfeusz2>=1.99", "wordfreq>=3.1"]
|
|
19
|
+
english = ["wordfreq>=3.1"]
|
|
20
|
+
en = ["wordfreq>=3.1"]
|
|
21
|
+
|
|
22
|
+
[project.urls]
|
|
23
|
+
Homepage = "https://github.com/praha6248/CzytajPDF"
|
|
24
|
+
Repository = "https://github.com/praha6248/CzytajPDF"
|
|
25
|
+
Issues = "https://github.com/praha6248/CzytajPDF/issues"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Read Polish and English PDF text, preserving the origin and original wording of each page."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Literal
|
|
6
|
+
|
|
7
|
+
from pypdf import PdfReader
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class Page:
|
|
12
|
+
number: int
|
|
13
|
+
source: Literal["pdf", "rapidocr", "tesseract"]
|
|
14
|
+
raw_text: str
|
|
15
|
+
text: str
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class Document:
|
|
20
|
+
pages: tuple[Page, ...]
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def text(self) -> str:
|
|
24
|
+
return "\n".join(page.text for page in self.pages)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def extract(
|
|
28
|
+
path: str | Path,
|
|
29
|
+
*,
|
|
30
|
+
ocr: bool = False,
|
|
31
|
+
correct: bool = False,
|
|
32
|
+
min_chars: int = 40,
|
|
33
|
+
engine: Literal["tesseract", "rapidocr"] = "tesseract",
|
|
34
|
+
lang: Literal["pl", "en"] = "pl",
|
|
35
|
+
) -> Document:
|
|
36
|
+
"""Read all pages. When enabled, try OCR below min_chars non-whitespace characters."""
|
|
37
|
+
if lang not in ("pl", "en"):
|
|
38
|
+
raise ValueError("lang must be pl or en")
|
|
39
|
+
if engine not in ("rapidocr", "tesseract"):
|
|
40
|
+
raise ValueError("engine must be tesseract or rapidocr")
|
|
41
|
+
if min_chars < 1:
|
|
42
|
+
raise ValueError("min_chars must be positive")
|
|
43
|
+
if correct and not ocr:
|
|
44
|
+
raise ValueError("correct=True requires ocr=True")
|
|
45
|
+
path = Path(path)
|
|
46
|
+
with path.open("rb") as stream:
|
|
47
|
+
reader = PdfReader(stream)
|
|
48
|
+
native = [page.extract_text() or "" for page in reader.pages]
|
|
49
|
+
|
|
50
|
+
recognized = {}
|
|
51
|
+
if ocr:
|
|
52
|
+
indices = [i for i, text in enumerate(native) if len("".join(text.split())) < min_chars]
|
|
53
|
+
if indices:
|
|
54
|
+
try:
|
|
55
|
+
from ._ocr import recognize
|
|
56
|
+
except ImportError as exc:
|
|
57
|
+
raise ImportError("OCR requires czytajpdf[ocr]") from exc
|
|
58
|
+
recognized = recognize(path, indices, engine, correct, lang)
|
|
59
|
+
|
|
60
|
+
pages = []
|
|
61
|
+
for index, original in enumerate(native):
|
|
62
|
+
candidate, text = recognized.get(index, ("", ""))
|
|
63
|
+
if not candidate.strip():
|
|
64
|
+
pages.append(Page(index + 1, "pdf", original, original))
|
|
65
|
+
continue
|
|
66
|
+
pages.append(Page(index + 1, engine, candidate, text))
|
|
67
|
+
return Document(tuple(pages))
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Rasterization and Polish/English recognition adapted from odwolania2json (Apache-2.0)."""
|
|
2
|
+
|
|
3
|
+
from contextlib import ExitStack
|
|
4
|
+
from functools import lru_cache
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from threading import Lock
|
|
7
|
+
|
|
8
|
+
import pypdfium2 as pdfium
|
|
9
|
+
|
|
10
|
+
_lock = Lock()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@lru_cache(maxsize=2)
|
|
14
|
+
def _engine(lang: str = "pl"):
|
|
15
|
+
from rapidocr import ModelType, OCRVersion, RapidOCR
|
|
16
|
+
|
|
17
|
+
return RapidOCR(params={
|
|
18
|
+
"Rec.lang_type": lang,
|
|
19
|
+
"Rec.ocr_version": OCRVersion.PPOCRV6,
|
|
20
|
+
"Rec.model_type": ModelType.SMALL,
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _render(page, dpi):
|
|
25
|
+
bitmap = page.render(scale=dpi / 72)
|
|
26
|
+
try:
|
|
27
|
+
return bitmap.to_pil().convert("RGB")
|
|
28
|
+
finally:
|
|
29
|
+
bitmap.close()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def recognize(
|
|
33
|
+
path: Path,
|
|
34
|
+
indices: list[int],
|
|
35
|
+
engine: str,
|
|
36
|
+
correct: bool,
|
|
37
|
+
lang: str = "pl",
|
|
38
|
+
) -> dict[int, tuple[str, str]]:
|
|
39
|
+
result = {}
|
|
40
|
+
if correct:
|
|
41
|
+
from ._spell import correct_text, ensure_dependencies, needs_second_read, reconcile_words
|
|
42
|
+
|
|
43
|
+
ensure_dependencies(lang)
|
|
44
|
+
# PDFium is not thread-safe. Serialize access to it and the shared OCR model.
|
|
45
|
+
with _lock, pdfium.PdfDocument(path) as pdf, ExitStack() as stack:
|
|
46
|
+
if engine == "tesseract":
|
|
47
|
+
try:
|
|
48
|
+
from ._tesseract import open_engine, read_page
|
|
49
|
+
except ImportError as exc:
|
|
50
|
+
raise ImportError("OCR requires czytajpdf[ocr]") from exc
|
|
51
|
+
|
|
52
|
+
tesseract = stack.enter_context(open_engine(lang))
|
|
53
|
+
else:
|
|
54
|
+
import numpy as np
|
|
55
|
+
|
|
56
|
+
for index in indices:
|
|
57
|
+
page = pdf[index]
|
|
58
|
+
try:
|
|
59
|
+
if engine == "tesseract":
|
|
60
|
+
raw = read_page(tesseract, _render(page, 300), 300)
|
|
61
|
+
text = raw
|
|
62
|
+
if correct and needs_second_read(raw, lang):
|
|
63
|
+
alternate = read_page(tesseract, _render(page, 180), 180)
|
|
64
|
+
text = reconcile_words(raw, alternate, lang)
|
|
65
|
+
else:
|
|
66
|
+
image = np.asarray(_render(page, 180))[:, :, ::-1].copy()
|
|
67
|
+
output = _engine(lang)(image)
|
|
68
|
+
raw = text = "\n".join(output.txts or ())
|
|
69
|
+
result[index] = raw, correct_text(text, lang=lang) if correct else text
|
|
70
|
+
finally:
|
|
71
|
+
page.close()
|
|
72
|
+
return result
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
"""Polish and English OCR spelling repair adapted from odwolania2json under Apache-2.0."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from difflib import SequenceMatcher
|
|
5
|
+
from functools import lru_cache
|
|
6
|
+
import itertools
|
|
7
|
+
import re
|
|
8
|
+
import unicodedata
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def fold(text: str) -> str:
|
|
12
|
+
text = text.replace("ł", "l").replace("Ł", "L")
|
|
13
|
+
return "".join(c for c in unicodedata.normalize("NFD", text) if not unicodedata.combining(c))
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
WORD = re.compile(r"[A-Za-zÀ-žĄĆĘŁŃÓŚŹŻąćęłńóśźż]{3,}")
|
|
17
|
+
L_LOOKALIKES = "tfh"
|
|
18
|
+
MAX_SUBSTITUTIONS = 3
|
|
19
|
+
ACCENT_MARGIN = 0.5
|
|
20
|
+
ACCENTS = {"a": "ą", "e": "ę", "l": "ł", "t": "ł", "o": "ó", "s": "ś", "c": "ć", "n": "ń", "z": "żź"}
|
|
21
|
+
|
|
22
|
+
# High-precision visual OCR confusions for English tokens; avoids corrupting proper nouns.
|
|
23
|
+
EN_OCR_PAIRS = (
|
|
24
|
+
("rn", "m"),
|
|
25
|
+
("fn", "m"),
|
|
26
|
+
("hi", "m"),
|
|
27
|
+
("in", "m"),
|
|
28
|
+
("ni", "m"),
|
|
29
|
+
("vv", "w"),
|
|
30
|
+
("cl", "d"),
|
|
31
|
+
("e", "c"),
|
|
32
|
+
("c", "e"),
|
|
33
|
+
("f", "t"),
|
|
34
|
+
("t", "f"),
|
|
35
|
+
("l", "i"),
|
|
36
|
+
("i", "l"),
|
|
37
|
+
("v", "y"),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def ensure_dependencies(lang: str = "pl") -> None:
|
|
42
|
+
"""Verify optional spelling dependencies for the selected language."""
|
|
43
|
+
try:
|
|
44
|
+
import wordfreq # noqa: F401
|
|
45
|
+
except ImportError as exc:
|
|
46
|
+
extra = "polish" if lang == "pl" else "english"
|
|
47
|
+
label = "Polish" if lang == "pl" else "English"
|
|
48
|
+
raise ImportError(f"{label} spelling repair requires czytajpdf[{extra}]") from exc
|
|
49
|
+
if lang == "pl":
|
|
50
|
+
try:
|
|
51
|
+
import morfeusz2 # noqa: F401
|
|
52
|
+
except ImportError as exc:
|
|
53
|
+
raise ImportError("Polish spelling repair requires czytajpdf[polish]") from exc
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@lru_cache(maxsize=2)
|
|
57
|
+
def lexicon(lang: str = "pl", size: int = 400000) -> tuple[frozenset[str], dict[str, str], dict[str, str]]:
|
|
58
|
+
"""Return known words and (for Polish) accent-folded lookup tables."""
|
|
59
|
+
import wordfreq
|
|
60
|
+
|
|
61
|
+
words = wordfreq.top_n_list(lang, size)
|
|
62
|
+
if lang != "pl":
|
|
63
|
+
return frozenset(words), {}, {}
|
|
64
|
+
index: dict[str, str] = {}
|
|
65
|
+
accented: dict[str, str] = {}
|
|
66
|
+
for word in words:
|
|
67
|
+
key = fold(word)
|
|
68
|
+
index.setdefault(key, word)
|
|
69
|
+
if word != key:
|
|
70
|
+
accented.setdefault(key, word)
|
|
71
|
+
return frozenset(words), index, accented
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@lru_cache(maxsize=1)
|
|
75
|
+
def _morfeusz():
|
|
76
|
+
import morfeusz2
|
|
77
|
+
|
|
78
|
+
return morfeusz2.Morfeusz(generate=False)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@lru_cache(maxsize=65536)
|
|
82
|
+
def is_word(word: str, lang: str = "pl") -> bool:
|
|
83
|
+
"""Tell whether a form exists in the frequency lexicon (or Morfeusz2 for Polish)."""
|
|
84
|
+
known = lexicon(lang)[0]
|
|
85
|
+
if word.lower() in known:
|
|
86
|
+
return True
|
|
87
|
+
if lang != "pl":
|
|
88
|
+
return False
|
|
89
|
+
analyser = _morfeusz()
|
|
90
|
+
return any(item[2][2] != "ign" for item in analyser.analyse(word))
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _frequency(word: str, lang: str = "pl") -> float:
|
|
94
|
+
"""Zipf frequency of a word (0 when unknown)."""
|
|
95
|
+
import wordfreq
|
|
96
|
+
|
|
97
|
+
return wordfreq.zipf_frequency(word, lang)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _variants(key: str) -> list[str]:
|
|
101
|
+
"""Folded variants where up to MAX_SUBSTITUTIONS look-alike letters are read as 'l' (folded 'ł')."""
|
|
102
|
+
positions = [i for i, c in enumerate(key) if c in L_LOOKALIKES]
|
|
103
|
+
out = []
|
|
104
|
+
for count in range(1, min(MAX_SUBSTITUTIONS, len(positions)) + 1):
|
|
105
|
+
for chosen in itertools.combinations(positions, count):
|
|
106
|
+
chars = list(key)
|
|
107
|
+
for i in chosen:
|
|
108
|
+
chars[i] = "l"
|
|
109
|
+
out.append("".join(chars))
|
|
110
|
+
return out
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _restore_with_morfeusz(lower: str, max_changes: int = 2) -> str | None:
|
|
114
|
+
"""Find a Polish dictionary form differing by at most two diacritic or 'ł' restorations."""
|
|
115
|
+
if len(lower) > 24:
|
|
116
|
+
return None
|
|
117
|
+
positions = [i for i, c in enumerate(lower) if c in ACCENTS]
|
|
118
|
+
if len(positions) > 8:
|
|
119
|
+
max_changes = 1
|
|
120
|
+
for count in range(1, max_changes + 1):
|
|
121
|
+
for chosen in itertools.combinations(positions, count):
|
|
122
|
+
for letters in itertools.product(*(ACCENTS[lower[i]] for i in chosen)):
|
|
123
|
+
chars = list(lower)
|
|
124
|
+
for i, letter in zip(chosen, letters):
|
|
125
|
+
chars[i] = letter
|
|
126
|
+
candidate = "".join(chars)
|
|
127
|
+
if is_word(candidate, "pl"):
|
|
128
|
+
return candidate
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _match_case(original: str, word: str) -> str:
|
|
133
|
+
"""Apply the capitalization pattern of the original token to the replacement."""
|
|
134
|
+
if original.isupper() and len(original) > 1:
|
|
135
|
+
return word.upper()
|
|
136
|
+
if original[0].isupper():
|
|
137
|
+
return word[0].upper() + word[1:]
|
|
138
|
+
return word
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _correct_english_word(token: str, known: frozenset[str], min_zipf: float) -> str:
|
|
142
|
+
lower = token.lower()
|
|
143
|
+
if len(lower) < 5 or lower in known:
|
|
144
|
+
return token
|
|
145
|
+
candidates = set()
|
|
146
|
+
for old, new in EN_OCR_PAIRS:
|
|
147
|
+
start = 0
|
|
148
|
+
while (idx := lower.find(old, start)) != -1:
|
|
149
|
+
candidate = lower[:idx] + new + lower[idx + len(old):]
|
|
150
|
+
if candidate in known:
|
|
151
|
+
candidates.add(candidate)
|
|
152
|
+
start = idx + 1
|
|
153
|
+
if not candidates:
|
|
154
|
+
return token
|
|
155
|
+
best = max(candidates, key=lambda word: _frequency(word, "en"))
|
|
156
|
+
# Require stronger evidence for title-cased words so proper nouns stay intact.
|
|
157
|
+
threshold = max(min_zipf, 3.5) if token[0].isupper() and not token.isupper() else min_zipf
|
|
158
|
+
if _frequency(best, "en") < threshold:
|
|
159
|
+
return token
|
|
160
|
+
return _match_case(token, best)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _correct_polish_word(token: str, known: frozenset[str], index: dict[str, str],
|
|
164
|
+
accented: dict[str, str], min_zipf: float) -> str:
|
|
165
|
+
lower = token.lower()
|
|
166
|
+
key = fold(lower)
|
|
167
|
+
if lower in known:
|
|
168
|
+
better = accented.get(key)
|
|
169
|
+
if better and lower == key and _frequency(better, "pl") - _frequency(lower, "pl") >= ACCENT_MARGIN:
|
|
170
|
+
return _match_case(token, better)
|
|
171
|
+
return token
|
|
172
|
+
if key in index and index[key] != lower:
|
|
173
|
+
return _match_case(token, index[key])
|
|
174
|
+
if is_word(token, "pl"):
|
|
175
|
+
return token
|
|
176
|
+
candidates = [index[variant] for variant in _variants(key) if variant in index]
|
|
177
|
+
if not candidates:
|
|
178
|
+
restored = _restore_with_morfeusz(lower)
|
|
179
|
+
return _match_case(token, restored) if restored else token
|
|
180
|
+
best = max(candidates, key=lambda word: _frequency(word, "pl"))
|
|
181
|
+
if _frequency(best, "pl") < min_zipf:
|
|
182
|
+
return token
|
|
183
|
+
return _match_case(token, best)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def correct_word(token: str, min_zipf: float | None = None, lang: str = "pl") -> str:
|
|
187
|
+
"""Return the most likely intended word for an OCR token, or the token itself."""
|
|
188
|
+
if len(token) > 48:
|
|
189
|
+
return token
|
|
190
|
+
known, index, accented = lexicon(lang)
|
|
191
|
+
if not known:
|
|
192
|
+
return token
|
|
193
|
+
if lang == "en":
|
|
194
|
+
return _correct_english_word(token, known, 1.5 if min_zipf is None else min_zipf)
|
|
195
|
+
return _correct_polish_word(token, known, index, accented, 2.0 if min_zipf is None else min_zipf)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
# A line-ending hyphen splits one lexical item into two apparent words.
|
|
199
|
+
# Keep both fragments intact, including their original line separator.
|
|
200
|
+
SPELL_TOKEN = re.compile(r"(?P<split>[^\W\d_]+[-‐]\s*\n[ \t]*[^\W\d_]+)|" + WORD.pattern)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def correct_text(text: str, min_zipf: float | None = None, lang: str = "pl") -> str:
|
|
204
|
+
"""Repair every word of an OCR text that is not in the lexicon."""
|
|
205
|
+
return SPELL_TOKEN.sub(
|
|
206
|
+
lambda m: m[0] if m["split"] else correct_word(m[0], min_zipf, lang=lang),
|
|
207
|
+
text,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def needs_second_read(text: str, lang: str = "pl") -> bool:
|
|
212
|
+
return any(not is_word(word, lang) for word in WORD.findall(text))
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def reconcile_words(text: str, second_read: str, lang: str = "pl") -> str:
|
|
216
|
+
"""Repair unknown words only when a second reading supplies a similar known form."""
|
|
217
|
+
tokens = list(re.finditer(r"\S+", text))
|
|
218
|
+
first = [token.group() for token in tokens]
|
|
219
|
+
second = second_read.split()
|
|
220
|
+
pieces = []
|
|
221
|
+
last = 0
|
|
222
|
+
for tag, a, b, c, d in SequenceMatcher(None, first, second, autojunk=False).get_opcodes():
|
|
223
|
+
if tag != "replace" or b - a != d - c:
|
|
224
|
+
continue
|
|
225
|
+
for token, candidate in zip(tokens[a:b], second[c:d]):
|
|
226
|
+
original = token.group()
|
|
227
|
+
match = WORD.search(original)
|
|
228
|
+
other = WORD.search(candidate)
|
|
229
|
+
if not match or not other:
|
|
230
|
+
continue
|
|
231
|
+
before, after = match.group(), other.group()
|
|
232
|
+
if (original[:match.start()] != candidate[:other.start()]
|
|
233
|
+
or original[match.end():] != candidate[other.end():]):
|
|
234
|
+
continue
|
|
235
|
+
if len(before) != len(after) or sum(x != y for x, y in zip(before, after)) > 1:
|
|
236
|
+
continue
|
|
237
|
+
if not is_word(before, lang) and is_word(after, lang):
|
|
238
|
+
pieces.extend((text[last:token.start()], candidate))
|
|
239
|
+
last = token.end()
|
|
240
|
+
pieces.append(text[last:])
|
|
241
|
+
return "".join(pieces)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Offline recognition after a one-time download of tessdata_best LSTM models."""
|
|
2
|
+
|
|
3
|
+
from functools import lru_cache
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import pooch
|
|
7
|
+
from tesserocr import PSM, PyTessBaseAPI
|
|
8
|
+
|
|
9
|
+
_BASE_URL = (
|
|
10
|
+
"https://raw.githubusercontent.com/tesseract-ocr/tessdata_best/"
|
|
11
|
+
"e12c65a915945e4c28e237a9b52bc4a8f39a0cec/"
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
_MODELS = {
|
|
15
|
+
"pl": (
|
|
16
|
+
"pol",
|
|
17
|
+
"sha256:e80cc4cefbdface06e9223f43f089556b9dcf104020fbc0a200f6863c57d4405",
|
|
18
|
+
),
|
|
19
|
+
"en": (
|
|
20
|
+
"eng",
|
|
21
|
+
"sha256:8280aed0782fe27257a68ea10fe7ef324ca0f8d85bd2fd145d1c2b560bcb66ba",
|
|
22
|
+
),
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@lru_cache(maxsize=2)
|
|
27
|
+
def _model_directory(lang: str = "pl") -> tuple[Path, str]:
|
|
28
|
+
tess_lang, known_hash = _MODELS[lang]
|
|
29
|
+
filename = f"{tess_lang}.traineddata"
|
|
30
|
+
model = pooch.retrieve(
|
|
31
|
+
url=_BASE_URL + filename,
|
|
32
|
+
known_hash=known_hash,
|
|
33
|
+
path=pooch.os_cache("czytajpdf") / "tessdata_best",
|
|
34
|
+
fname=filename,
|
|
35
|
+
)
|
|
36
|
+
return Path(model).parent, tess_lang
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def open_engine(lang: str = "pl"):
|
|
40
|
+
model_dir, tess_lang = _model_directory(lang)
|
|
41
|
+
return PyTessBaseAPI(path=str(model_dir), lang=tess_lang, psm=PSM.AUTO)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def read_page(engine, image, dpi):
|
|
45
|
+
engine.ClearAdaptiveClassifier()
|
|
46
|
+
engine.SetImage(image)
|
|
47
|
+
engine.SetSourceResolution(dpi)
|
|
48
|
+
return engine.GetUTF8Text().strip()
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: czytajpdf
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Page-wise Polish and English PDF text extraction with optional OCR and spelling repair
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
Project-URL: Homepage, https://github.com/praha6248/CzytajPDF
|
|
7
|
+
Project-URL: Repository, https://github.com/praha6248/CzytajPDF
|
|
8
|
+
Project-URL: Issues, https://github.com/praha6248/CzytajPDF/issues
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: pypdf>=5
|
|
13
|
+
Provides-Extra: ocr
|
|
14
|
+
Requires-Dist: tesserocr<3,>=2.8; extra == "ocr"
|
|
15
|
+
Requires-Dist: pooch<2,>=1.8; extra == "ocr"
|
|
16
|
+
Requires-Dist: rapidocr<4,>=3.10; extra == "ocr"
|
|
17
|
+
Requires-Dist: onnxruntime>=1.20; extra == "ocr"
|
|
18
|
+
Requires-Dist: pypdfium2>=4.30; extra == "ocr"
|
|
19
|
+
Requires-Dist: numpy>=1.24; extra == "ocr"
|
|
20
|
+
Requires-Dist: pillow>=10; extra == "ocr"
|
|
21
|
+
Provides-Extra: polish
|
|
22
|
+
Requires-Dist: morfeusz2>=1.99; extra == "polish"
|
|
23
|
+
Requires-Dist: wordfreq>=3.1; extra == "polish"
|
|
24
|
+
Provides-Extra: pl
|
|
25
|
+
Requires-Dist: morfeusz2>=1.99; extra == "pl"
|
|
26
|
+
Requires-Dist: wordfreq>=3.1; extra == "pl"
|
|
27
|
+
Provides-Extra: english
|
|
28
|
+
Requires-Dist: wordfreq>=3.1; extra == "english"
|
|
29
|
+
Provides-Extra: en
|
|
30
|
+
Requires-Dist: wordfreq>=3.1; extra == "en"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# CzytajPDF
|
|
34
|
+
|
|
35
|
+
[](https://github.com/praha6248/CzytajPDF/actions/workflows/ci.yml)
|
|
36
|
+
|
|
37
|
+
**Polskie i angielskie PDF-y oraz skany w jednym wywołaniu Pythona.** CzytajPDF odczytuje dokument strona po stronie. Gdy pismo ma warstwę tekstową, a załącznik jest skanem, dostajesz jeden wynik z informacją, skąd pochodzi tekst każdej strony. Opcjonalna korekta przywraca polskie znaki lub naprawia typowe błędy OCR po angielsku, zachowując oryginalny odczyt do porównania.
|
|
38
|
+
|
|
39
|
+
> **English summary:** Page-wise Python library (Apache-2.0) for hybrid Polish and English PDFs — reads embedded text via `pypdf`, automatically falls back to local OCR (`tesserocr` / `rapidocr`) on scanned pages, and optionally applies Polish (`morfeusz2` + `wordfreq`) or English (`wordfreq`) OCR repair with dual-resolution reconciliation while preserving `source` and `raw_text`. Also packaged as `readpdf-pl` and `readpdf-en` from the same repository. See [README.en.md](README.en.md).
|
|
40
|
+
|
|
41
|
+
## Pakiety w repozytorium
|
|
42
|
+
|
|
43
|
+
Jedno repozytorium dostarcza wspólną bibliotekę bazową oraz dwie paczki językowe o prostszej konfiguracji domyślnej:
|
|
44
|
+
|
|
45
|
+
- **`czytajpdf`** — pełna biblioteka z parametrem `lang="pl"` (domyślnie) lub `lang="en"`
|
|
46
|
+
- **`readpdf-pl`** (`packages/readpdf-pl`) — paczka dla dokumentów polskich (domyślnie `lang="pl"`)
|
|
47
|
+
- **`readpdf-en`** (`packages/readpdf-en`) — paczka dla dokumentów angielskich (domyślnie `lang="en"`, bez zależności od `morfeusz2`)
|
|
48
|
+
|
|
49
|
+
## Pierwszy odczyt
|
|
50
|
+
|
|
51
|
+
Python 3.10+. Instalacja z GitHuba z obsługą OCR i korekty:
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
# Polski i angielski (pełna instalacja)
|
|
55
|
+
python -m pip install 'czytajpdf[ocr,polish] @ git+https://github.com/praha6248/CzytajPDF.git'
|
|
56
|
+
|
|
57
|
+
# Tylko angielski (bez morfeusz2)
|
|
58
|
+
python -m pip install 'czytajpdf[ocr,english] @ git+https://github.com/praha6248/CzytajPDF.git'
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from czytajpdf import extract
|
|
63
|
+
|
|
64
|
+
dokument = extract("pismo-z-zalacznikami.pdf", ocr=True, correct=True)
|
|
65
|
+
for strona in dokument.pages:
|
|
66
|
+
print(strona.number, strona.source, strona.text)
|
|
67
|
+
|
|
68
|
+
english_doc = extract("filing-with-exhibits.pdf", ocr=True, correct=True, lang="en")
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Do PDF-ów z warstwą tekstową wystarczy instalacja bez dodatków i wywołanie `extract("pismo.pdf")`. Cały tekst jest dostępny jako `dokument.text`. Numeracja stron zaczyna się od 1. Z paczek językowych można korzystać przez `from readpdf_pl import extract` oraz `from readpdf_en import extract`.
|
|
72
|
+
|
|
73
|
+
## Statystyki odczytu
|
|
74
|
+
|
|
75
|
+
| Dokument | Język | Strony | Strony z warstwy tekstowej | Strony z OCR | Znaki z OCR |
|
|
76
|
+
| --- | --- | ---: | ---: | ---: | ---: |
|
|
77
|
+
| [Dz.U. 2024 poz. 1](https://api.sejm.gov.pl/eli/acts/DU/2024/1/text.pdf) | `pl` | 8 | 8 | 0 | 0 |
|
|
78
|
+
| [Dz.U. 1945 nr 5 poz. 17](https://api.sejm.gov.pl/eli/acts/DU/1945/17/text.pdf) | `pl` | 2 | 2 | 0 | 0 |
|
|
79
|
+
| PDF mieszany z pierwszych stron obu polskich dokumentów | `pl` | 2 | 1 | 1 | 2716 |
|
|
80
|
+
| [SCOTUS No. 22-451](https://www.supremecourt.gov/opinions/23pdf/22-451_7m58.pdf) | `en` | 114 | 114 | 0 | 0 |
|
|
81
|
+
| [Brown v. Board of Education, 347 U.S. 483](https://cdn.loc.gov/service/ll/usrep/usrep347/usrep347483/usrep347483.pdf) | `en` | 14 | 14 | 0 | 0 |
|
|
82
|
+
| PDF mieszany z pierwszych stron obu angielskich dokumentów | `en` | 2 | 1 | 1 | 2204 |
|
|
83
|
+
|
|
84
|
+
## Skuteczność korekty polskiego OCR
|
|
85
|
+
|
|
86
|
+
Pomiar obejmuje nagłówki z 18 archiwalnych skanów Dziennika Ustaw. Na pierwotnej próbce 12 nagłówków **liczba błędów słów spadła z 79 do 2**. Do porównania użyto tych samych wycinków i transkrypcji.
|
|
87
|
+
|
|
88
|
+
| Próbka i wersja biblioteki | Błędy znaków (CER) po korekcie | Błędy słów (WER) po korekcie |
|
|
89
|
+
| --- | ---: | ---: |
|
|
90
|
+
| 12 pierwotnych nagłówków · 0.1.0, RapidOCR | 382 / 2129 · 17,94% | 79 / 293 · 26,96% |
|
|
91
|
+
| Te same 12 nagłówków · 0.2.0 / 0.3.0, Tesseract | 2 / 2129 · 0,09% | 2 / 293 · 0,68% |
|
|
92
|
+
| 3 dodatkowe nagłówki użyte do sprawdzenia i poprawienia obsługi dzielenia słów · 0.2.0 / 0.3.0 | 0 / 593 · 0,00% | 0 / 87 · 0,00% |
|
|
93
|
+
| 3 osobne nagłówki użyte do końcowej weryfikacji · 0.2.0 / 0.3.0 | 1 / 430 · 0,23% | 1 / 67 · 1,49% |
|
|
94
|
+
|
|
95
|
+
Na pierwotnej próbce surowy Tesseract popełnił 5 błędów słów, a drugi odczyt z korektą usunął 3. Pozostały pomyłki `r.` → `r,` oraz `i` → `1`. W końcowej weryfikacji OCR dodał kropkę po słowie `lutego`.
|
|
96
|
+
|
|
97
|
+
CER i WER uwzględniają zamiany, wstawienia i pominięcia względem transkrypcji z obrazu. Ujednolicono białe znaki, zachowując interpunkcję i wielkość liter. Wynik dotyczy wycinków z drukowanymi nagłówkami. [Pomiar 0.1.0](benchmarks/rapidocr-0.1.0.json) · [Pomiar 0.2.0/0.3.0](benchmarks/results.json) · [Próbka dodatkowa](benchmarks/holdout.json) · [Weryfikacja końcowa](benchmarks/validation.json) · [Skrypt pomiarowy](benchmarks/score.py)
|
|
98
|
+
|
|
99
|
+
## Skuteczność angielskiego OCR (`lang="en"`)
|
|
100
|
+
|
|
101
|
+
Pomiar obejmuje 9 archiwalnych skanów *United States Reports* z Library of Congress (lata 1937–1989). Na 6 dokumentach próbki głównej korekta naprawiła błędy `Amendfnent` → `Amendment` oraz `eriminal` → `criminal` (spadek WER z 17 do 15 na 897 słowach), a na 3 osobnych dokumentach walidacyjnych nie wprowadziła żadnych fałszywych poprawek.
|
|
102
|
+
|
|
103
|
+
| Próbka angielska (0.3.0, Tesseract `eng`) | Surowy CER / WER | Po korekcie CER | Po korekcie WER |
|
|
104
|
+
| --- | ---: | ---: | ---: |
|
|
105
|
+
| 6 skanów *U.S. Reports* (1937–1984) | 0,35% / 1,90% (17 / 897) | 17 / 5745 · 0,30% | 15 / 897 · 1,67% |
|
|
106
|
+
| 3 osobne skany walidacyjne (1943, 1964, 1989) | 0,32% / 1,99% (6 / 301) | 6 / 1871 · 0,32% | 6 / 301 · 1,99% |
|
|
107
|
+
|
|
108
|
+
[Pomiar angielski](benchmarks/english_results.json) · [Walidacja angielska](benchmarks/english_validation.json)
|
|
109
|
+
|
|
110
|
+
## Jak wybierany jest sposób odczytu
|
|
111
|
+
|
|
112
|
+
- Domyślnie działa pypdf. Przy `ocr=True` strony z mniej niż 40 znakami po pominięciu białych znaków trafiają do modelu Tesseract dla wybranego języka (`lang="pl"` albo `lang="en"`). Aby wybrać RapidOCR, przekaż `engine="rapidocr"`. Próg zmienisz przez `min_chars`. Długa, uszkodzona warstwa tekstowa może przejść ten próg.
|
|
113
|
+
- `source` przyjmuje wartość `pdf`, `tesseract` lub `rapidocr`. Pusty wynik OCR pozostawia tekst z warstwy PDF. Wynik zachowuje podział na strony, bez współrzędnych słów ani struktury tabel.
|
|
114
|
+
- `correct=True` włącza korektę wyników OCR: dla `lang="pl"` używa `wordfreq` i `Morfeusza 2`, a dla `lang="en"` `wordfreq` i ostrożnych zamian typowych pomyłek OCR. Tesseract może ponownie odczytać stronę w rozdzielczości 180 DPI, aby zweryfikować nierozpoznane słowa. Pierwszy odczyt pozostaje dostępny w `raw_text`. Przy nazwiskach i nazwach własnych warto porównać wynik z `raw_text`.
|
|
115
|
+
- Rozpoznawanie odbywa się lokalnie. Przy pierwszym użyciu Tesseract pobiera odpowiedni model `tessdata_best` (około 12 MB dla `pol` lub 15 MB dla `eng`), sprawdza SHA-256 i zapisuje plik w pamięci podręcznej. Kolejne odczyty działają offline. RapidOCR korzysta z osobnych modeli.
|
|
116
|
+
|
|
117
|
+
## Licencja i pochodzenie
|
|
118
|
+
|
|
119
|
+
Kod biblioteki jest dostępny na [Apache-2.0](LICENSE). Projekt wykorzystuje logikę z [odwolania2json](https://github.com/praha6248/odwolania2Json). Wagi OCR, słowniki i dane częstości są osobnymi zależnościami na własnych licencjach.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/czytajpdf/__init__.py
|
|
5
|
+
src/czytajpdf/_ocr.py
|
|
6
|
+
src/czytajpdf/_spell.py
|
|
7
|
+
src/czytajpdf/_tesseract.py
|
|
8
|
+
src/czytajpdf.egg-info/PKG-INFO
|
|
9
|
+
src/czytajpdf.egg-info/SOURCES.txt
|
|
10
|
+
src/czytajpdf.egg-info/dependency_links.txt
|
|
11
|
+
src/czytajpdf.egg-info/requires.txt
|
|
12
|
+
src/czytajpdf.egg-info/top_level.txt
|
|
13
|
+
tests/test_czytajpdf.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
pypdf>=5
|
|
2
|
+
|
|
3
|
+
[en]
|
|
4
|
+
wordfreq>=3.1
|
|
5
|
+
|
|
6
|
+
[english]
|
|
7
|
+
wordfreq>=3.1
|
|
8
|
+
|
|
9
|
+
[ocr]
|
|
10
|
+
tesserocr<3,>=2.8
|
|
11
|
+
pooch<2,>=1.8
|
|
12
|
+
rapidocr<4,>=3.10
|
|
13
|
+
onnxruntime>=1.20
|
|
14
|
+
pypdfium2>=4.30
|
|
15
|
+
numpy>=1.24
|
|
16
|
+
pillow>=10
|
|
17
|
+
|
|
18
|
+
[pl]
|
|
19
|
+
morfeusz2>=1.99
|
|
20
|
+
wordfreq>=3.1
|
|
21
|
+
|
|
22
|
+
[polish]
|
|
23
|
+
morfeusz2>=1.99
|
|
24
|
+
wordfreq>=3.1
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
czytajpdf
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""Offline unit tests for czytajpdf, readpdf-pl, and readpdf-en."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib.util
|
|
6
|
+
import sys
|
|
7
|
+
import types
|
|
8
|
+
import unittest
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from tempfile import TemporaryDirectory
|
|
11
|
+
from unittest.mock import patch
|
|
12
|
+
|
|
13
|
+
from czytajpdf import Document, Page, extract
|
|
14
|
+
|
|
15
|
+
HAS_POLISH = all(importlib.util.find_spec(pkg) is not None for pkg in ("morfeusz2", "wordfreq"))
|
|
16
|
+
HAS_ENGLISH = importlib.util.find_spec("wordfreq") is not None
|
|
17
|
+
HAS_WRAPPERS = all(
|
|
18
|
+
importlib.util.find_spec(pkg) is not None for pkg in ("readpdf_pl", "readpdf_en")
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _escape_pdf_text(text: str) -> str:
|
|
23
|
+
return text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _build_pdf(pages_text: list[str | None]) -> bytes:
|
|
27
|
+
"""Build a minimal valid PDF with one object per page and optional Helvetica text."""
|
|
28
|
+
objects: list[bytes] = [
|
|
29
|
+
b"<< /Type /Catalog /Pages 2 0 R >>",
|
|
30
|
+
b"", # Placeholder for /Pages (2 0 obj)
|
|
31
|
+
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
|
32
|
+
]
|
|
33
|
+
kids = []
|
|
34
|
+
for text in pages_text:
|
|
35
|
+
page_id = len(objects) + 1
|
|
36
|
+
kids.append(f"{page_id} 0 R")
|
|
37
|
+
if text is None:
|
|
38
|
+
objects.append(
|
|
39
|
+
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>"
|
|
40
|
+
)
|
|
41
|
+
continue
|
|
42
|
+
stream = f"BT /F1 12 Tf 20 150 Td ({_escape_pdf_text(text)}) Tj ET".encode("latin-1")
|
|
43
|
+
content_id = page_id + 1
|
|
44
|
+
objects.append(
|
|
45
|
+
f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] "
|
|
46
|
+
f"/Resources << /Font << /F1 3 0 R >> >> /Contents {content_id} 0 R >>".encode("ascii")
|
|
47
|
+
)
|
|
48
|
+
objects.append(
|
|
49
|
+
b"<< /Length " + str(len(stream)).encode("ascii") + b" >>\nstream\n" + stream + b"\nendstream"
|
|
50
|
+
)
|
|
51
|
+
objects[1] = f"<< /Type /Pages /Kids [{' '.join(kids)}] /Count {len(kids)} >>".encode("ascii")
|
|
52
|
+
|
|
53
|
+
out = bytearray(b"%PDF-1.4\n")
|
|
54
|
+
offsets = [0]
|
|
55
|
+
for idx, body in enumerate(objects, start=1):
|
|
56
|
+
offsets.append(len(out))
|
|
57
|
+
out.extend(f"{idx} 0 obj\n".encode("ascii"))
|
|
58
|
+
out.extend(body)
|
|
59
|
+
out.extend(b"\nendobj\n")
|
|
60
|
+
xref_pos = len(out)
|
|
61
|
+
out.extend(f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode("ascii"))
|
|
62
|
+
for offset in offsets[1:]:
|
|
63
|
+
out.extend(f"{offset:010d} 00000 n \n".encode("ascii"))
|
|
64
|
+
out.extend(
|
|
65
|
+
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{xref_pos}\n%%EOF\n".encode("ascii")
|
|
66
|
+
)
|
|
67
|
+
return bytes(out)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class ExtractTests(unittest.TestCase):
|
|
71
|
+
def test_argument_validation(self) -> None:
|
|
72
|
+
with self.assertRaises(ValueError):
|
|
73
|
+
extract("ignored.pdf", lang="de") # type: ignore[arg-type]
|
|
74
|
+
with self.assertRaises(ValueError):
|
|
75
|
+
extract("ignored.pdf", engine="unknown") # type: ignore[arg-type]
|
|
76
|
+
with self.assertRaises(ValueError):
|
|
77
|
+
extract("ignored.pdf", min_chars=0)
|
|
78
|
+
with self.assertRaises(ValueError):
|
|
79
|
+
extract("ignored.pdf", ocr=False, correct=True)
|
|
80
|
+
|
|
81
|
+
def test_native_extraction_preserves_page_numbers(self) -> None:
|
|
82
|
+
with TemporaryDirectory() as tmp:
|
|
83
|
+
pdf_path = Path(tmp) / "native.pdf"
|
|
84
|
+
pdf_path.write_bytes(_build_pdf([
|
|
85
|
+
"Pierwsza strona ma pelna warstwe tekstowa z kilkoma slowami.",
|
|
86
|
+
None,
|
|
87
|
+
]))
|
|
88
|
+
doc = extract(pdf_path)
|
|
89
|
+
self.assertIsInstance(doc, Document)
|
|
90
|
+
self.assertEqual(len(doc.pages), 2)
|
|
91
|
+
self.assertEqual(doc.pages[0].number, 1)
|
|
92
|
+
self.assertEqual(doc.pages[0].source, "pdf")
|
|
93
|
+
self.assertIn("Pierwsza strona", doc.pages[0].text)
|
|
94
|
+
self.assertEqual(doc.pages[0].raw_text, doc.pages[0].text)
|
|
95
|
+
self.assertEqual(doc.pages[1], Page(2, "pdf", "", ""))
|
|
96
|
+
self.assertEqual(doc.text, f"{doc.pages[0].text}\n")
|
|
97
|
+
|
|
98
|
+
def test_ocr_routing_lang_and_empty_fallback(self) -> None:
|
|
99
|
+
calls: list[tuple[Path, list[int], str, bool, str]] = []
|
|
100
|
+
|
|
101
|
+
def fake_recognize(
|
|
102
|
+
path: Path, indices: list[int], engine: str, correct: bool, lang: str = "pl"
|
|
103
|
+
) -> dict[int, tuple[str, str]]:
|
|
104
|
+
calls.append((path, indices, engine, correct, lang))
|
|
105
|
+
return {1: ("raw ocr", "corrected ocr"), 2: (" ", " ")}
|
|
106
|
+
|
|
107
|
+
fake_mod = types.ModuleType("czytajpdf._ocr")
|
|
108
|
+
fake_mod.recognize = fake_recognize # type: ignore[attr-defined]
|
|
109
|
+
|
|
110
|
+
with TemporaryDirectory() as tmp, patch.dict(sys.modules, {"czytajpdf._ocr": fake_mod}):
|
|
111
|
+
pdf_path = Path(tmp) / "mixed.pdf"
|
|
112
|
+
pdf_path.write_bytes(_build_pdf([
|
|
113
|
+
"This page has well over forty characters in its text layer.",
|
|
114
|
+
None,
|
|
115
|
+
"short",
|
|
116
|
+
]))
|
|
117
|
+
doc = extract(pdf_path, ocr=True, correct=True, engine="tesseract", lang="en")
|
|
118
|
+
self.assertEqual(calls, [(pdf_path, [1, 2], "tesseract", True, "en")])
|
|
119
|
+
self.assertEqual(doc.pages[0].source, "pdf")
|
|
120
|
+
self.assertEqual(doc.pages[1], Page(2, "tesseract", "raw ocr", "corrected ocr"))
|
|
121
|
+
self.assertEqual(doc.pages[2], Page(3, "pdf", "short", "short"))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@unittest.skipUnless(HAS_WRAPPERS, "readpdf-pl and readpdf-en are not installed")
|
|
125
|
+
class WrapperTests(unittest.TestCase):
|
|
126
|
+
def test_wrapper_default_languages(self) -> None:
|
|
127
|
+
import readpdf_en
|
|
128
|
+
import readpdf_pl
|
|
129
|
+
|
|
130
|
+
with TemporaryDirectory() as tmp:
|
|
131
|
+
pdf_path = Path(tmp) / "scan.pdf"
|
|
132
|
+
pdf_path.write_bytes(_build_pdf([None]))
|
|
133
|
+
stub = Document((Page(1, "pdf", "", ""),))
|
|
134
|
+
with patch.object(readpdf_pl, "_extract", return_value=stub) as mock_pl:
|
|
135
|
+
readpdf_pl.extract(pdf_path, ocr=True)
|
|
136
|
+
mock_pl.assert_called_once_with(
|
|
137
|
+
pdf_path, ocr=True, correct=False, min_chars=40, engine="tesseract", lang="pl"
|
|
138
|
+
)
|
|
139
|
+
with patch.object(readpdf_en, "_extract", return_value=stub) as mock_en:
|
|
140
|
+
readpdf_en.extract(pdf_path, ocr=True, correct=True)
|
|
141
|
+
mock_en.assert_called_once_with(
|
|
142
|
+
pdf_path, ocr=True, correct=True, min_chars=40, engine="tesseract", lang="en"
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@unittest.skipUnless(HAS_ENGLISH, "czytajpdf[english] is not installed")
|
|
147
|
+
class EnglishSpellingTests(unittest.TestCase):
|
|
148
|
+
def test_correct_text_repairs_english_ocr_confusions_only(self) -> None:
|
|
149
|
+
from czytajpdf._spell import correct_text, ensure_dependencies, needs_second_read, reconcile_words
|
|
150
|
+
|
|
151
|
+
ensure_dependencies("en")
|
|
152
|
+
source = (
|
|
153
|
+
"The Amendfnent bars eriminal govern-\nment bans on proseribe "
|
|
154
|
+
"statutorv rules; Hallford testified."
|
|
155
|
+
)
|
|
156
|
+
repaired = correct_text(source, lang="en")
|
|
157
|
+
self.assertIn("Amendment", repaired)
|
|
158
|
+
self.assertIn("criminal", repaired)
|
|
159
|
+
self.assertIn("govern-\nment", repaired)
|
|
160
|
+
self.assertIn("proscribe", repaired)
|
|
161
|
+
self.assertIn("statutory", repaired)
|
|
162
|
+
self.assertIn("Hallford", repaired)
|
|
163
|
+
|
|
164
|
+
first = "The Court dgnied relief."
|
|
165
|
+
second = "The Court denied relief."
|
|
166
|
+
self.assertTrue(needs_second_read(first, "en"))
|
|
167
|
+
self.assertFalse(needs_second_read(second, "en"))
|
|
168
|
+
self.assertEqual(reconcile_words(first, second, "en"), second)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@unittest.skipUnless(HAS_POLISH, "czytajpdf[polish] is not installed")
|
|
172
|
+
class PolishSpellingTests(unittest.TestCase):
|
|
173
|
+
def test_correct_text_preserves_line_break_hyphenation(self) -> None:
|
|
174
|
+
from czytajpdf._spell import correct_text
|
|
175
|
+
|
|
176
|
+
source = "Umowa z Repub-\nliką Włoską oraz zalacznik."
|
|
177
|
+
repaired = correct_text(source, lang="pl")
|
|
178
|
+
self.assertIn("Repub-\nliką", repaired)
|
|
179
|
+
self.assertIn("załącznik.", repaired)
|
|
180
|
+
|
|
181
|
+
def test_reconcile_words_and_needs_second_read(self) -> None:
|
|
182
|
+
from czytajpdf._spell import needs_second_read, reconcile_words
|
|
183
|
+
|
|
184
|
+
first = "100 KOZPORZĄDZENIE w sprawie skiadu."
|
|
185
|
+
second = "100 ROZPORZĄDZENIE w sprawie składu."
|
|
186
|
+
self.assertTrue(needs_second_read(first, "pl"))
|
|
187
|
+
self.assertFalse(needs_second_read(second, "pl"))
|
|
188
|
+
self.assertEqual(reconcile_words(first, second, "pl"), second)
|
|
189
|
+
|
|
190
|
+
unchanged = reconcile_words("100 ABZPORZĄDZENIE xyzq.", "100 ROZPORZĄDZENIE xwzp.", "pl")
|
|
191
|
+
self.assertEqual(unchanged, "100 ABZPORZĄDZENIE xyzq.")
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
if __name__ == "__main__":
|
|
195
|
+
unittest.main()
|