czytajpdf 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
czytajpdf/__init__.py ADDED
@@ -0,0 +1,67 @@
1
+ """Read Polish and English PDF text, preserving the origin and original wording of each page."""
2
+
3
+ from dataclasses import dataclass
4
+ from pathlib import Path
5
+ from typing import Literal
6
+
7
+ from pypdf import PdfReader
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class Page:
12
+ number: int
13
+ source: Literal["pdf", "rapidocr", "tesseract"]
14
+ raw_text: str
15
+ text: str
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class Document:
20
+ pages: tuple[Page, ...]
21
+
22
+ @property
23
+ def text(self) -> str:
24
+ return "\n".join(page.text for page in self.pages)
25
+
26
+
27
+ def extract(
28
+ path: str | Path,
29
+ *,
30
+ ocr: bool = False,
31
+ correct: bool = False,
32
+ min_chars: int = 40,
33
+ engine: Literal["tesseract", "rapidocr"] = "tesseract",
34
+ lang: Literal["pl", "en"] = "pl",
35
+ ) -> Document:
36
+ """Read all pages. When enabled, try OCR below min_chars non-whitespace characters."""
37
+ if lang not in ("pl", "en"):
38
+ raise ValueError("lang must be pl or en")
39
+ if engine not in ("rapidocr", "tesseract"):
40
+ raise ValueError("engine must be tesseract or rapidocr")
41
+ if min_chars < 1:
42
+ raise ValueError("min_chars must be positive")
43
+ if correct and not ocr:
44
+ raise ValueError("correct=True requires ocr=True")
45
+ path = Path(path)
46
+ with path.open("rb") as stream:
47
+ reader = PdfReader(stream)
48
+ native = [page.extract_text() or "" for page in reader.pages]
49
+
50
+ recognized = {}
51
+ if ocr:
52
+ indices = [i for i, text in enumerate(native) if len("".join(text.split())) < min_chars]
53
+ if indices:
54
+ try:
55
+ from ._ocr import recognize
56
+ except ImportError as exc:
57
+ raise ImportError("OCR requires czytajpdf[ocr]") from exc
58
+ recognized = recognize(path, indices, engine, correct, lang)
59
+
60
+ pages = []
61
+ for index, original in enumerate(native):
62
+ candidate, text = recognized.get(index, ("", ""))
63
+ if not candidate.strip():
64
+ pages.append(Page(index + 1, "pdf", original, original))
65
+ continue
66
+ pages.append(Page(index + 1, engine, candidate, text))
67
+ return Document(tuple(pages))
czytajpdf/_ocr.py ADDED
@@ -0,0 +1,72 @@
1
+ """Rasterization and Polish/English recognition adapted from odwolania2json (Apache-2.0)."""
2
+
3
+ from contextlib import ExitStack
4
+ from functools import lru_cache
5
+ from pathlib import Path
6
+ from threading import Lock
7
+
8
+ import pypdfium2 as pdfium
9
+
10
+ _lock = Lock()
11
+
12
+
13
+ @lru_cache(maxsize=2)
14
+ def _engine(lang: str = "pl"):
15
+ from rapidocr import ModelType, OCRVersion, RapidOCR
16
+
17
+ return RapidOCR(params={
18
+ "Rec.lang_type": lang,
19
+ "Rec.ocr_version": OCRVersion.PPOCRV6,
20
+ "Rec.model_type": ModelType.SMALL,
21
+ })
22
+
23
+
24
+ def _render(page, dpi):
25
+ bitmap = page.render(scale=dpi / 72)
26
+ try:
27
+ return bitmap.to_pil().convert("RGB")
28
+ finally:
29
+ bitmap.close()
30
+
31
+
32
+ def recognize(
33
+ path: Path,
34
+ indices: list[int],
35
+ engine: str,
36
+ correct: bool,
37
+ lang: str = "pl",
38
+ ) -> dict[int, tuple[str, str]]:
39
+ result = {}
40
+ if correct:
41
+ from ._spell import correct_text, ensure_dependencies, needs_second_read, reconcile_words
42
+
43
+ ensure_dependencies(lang)
44
+ # PDFium is not thread-safe. Serialize access to it and the shared OCR model.
45
+ with _lock, pdfium.PdfDocument(path) as pdf, ExitStack() as stack:
46
+ if engine == "tesseract":
47
+ try:
48
+ from ._tesseract import open_engine, read_page
49
+ except ImportError as exc:
50
+ raise ImportError("OCR requires czytajpdf[ocr]") from exc
51
+
52
+ tesseract = stack.enter_context(open_engine(lang))
53
+ else:
54
+ import numpy as np
55
+
56
+ for index in indices:
57
+ page = pdf[index]
58
+ try:
59
+ if engine == "tesseract":
60
+ raw = read_page(tesseract, _render(page, 300), 300)
61
+ text = raw
62
+ if correct and needs_second_read(raw, lang):
63
+ alternate = read_page(tesseract, _render(page, 180), 180)
64
+ text = reconcile_words(raw, alternate, lang)
65
+ else:
66
+ image = np.asarray(_render(page, 180))[:, :, ::-1].copy()
67
+ output = _engine(lang)(image)
68
+ raw = text = "\n".join(output.txts or ())
69
+ result[index] = raw, correct_text(text, lang=lang) if correct else text
70
+ finally:
71
+ page.close()
72
+ return result
czytajpdf/_spell.py ADDED
@@ -0,0 +1,241 @@
1
+ """Polish and English OCR spelling repair adapted from odwolania2json under Apache-2.0."""
2
+ from __future__ import annotations
3
+
4
+ from difflib import SequenceMatcher
5
+ from functools import lru_cache
6
+ import itertools
7
+ import re
8
+ import unicodedata
9
+
10
+
11
+ def fold(text: str) -> str:
12
+ text = text.replace("ł", "l").replace("Ł", "L")
13
+ return "".join(c for c in unicodedata.normalize("NFD", text) if not unicodedata.combining(c))
14
+
15
+
16
+ WORD = re.compile(r"[A-Za-zÀ-žĄĆĘŁŃÓŚŹŻąćęłńóśźż]{3,}")
17
+ L_LOOKALIKES = "tfh"
18
+ MAX_SUBSTITUTIONS = 3
19
+ ACCENT_MARGIN = 0.5
20
+ ACCENTS = {"a": "ą", "e": "ę", "l": "ł", "t": "ł", "o": "ó", "s": "ś", "c": "ć", "n": "ń", "z": "żź"}
21
+
22
+ # High-precision visual OCR confusions for English tokens; avoids corrupting proper nouns.
23
+ EN_OCR_PAIRS = (
24
+ ("rn", "m"),
25
+ ("fn", "m"),
26
+ ("hi", "m"),
27
+ ("in", "m"),
28
+ ("ni", "m"),
29
+ ("vv", "w"),
30
+ ("cl", "d"),
31
+ ("e", "c"),
32
+ ("c", "e"),
33
+ ("f", "t"),
34
+ ("t", "f"),
35
+ ("l", "i"),
36
+ ("i", "l"),
37
+ ("v", "y"),
38
+ )
39
+
40
+
41
+ def ensure_dependencies(lang: str = "pl") -> None:
42
+ """Verify optional spelling dependencies for the selected language."""
43
+ try:
44
+ import wordfreq # noqa: F401
45
+ except ImportError as exc:
46
+ extra = "polish" if lang == "pl" else "english"
47
+ label = "Polish" if lang == "pl" else "English"
48
+ raise ImportError(f"{label} spelling repair requires czytajpdf[{extra}]") from exc
49
+ if lang == "pl":
50
+ try:
51
+ import morfeusz2 # noqa: F401
52
+ except ImportError as exc:
53
+ raise ImportError("Polish spelling repair requires czytajpdf[polish]") from exc
54
+
55
+
56
+ @lru_cache(maxsize=2)
57
+ def lexicon(lang: str = "pl", size: int = 400000) -> tuple[frozenset[str], dict[str, str], dict[str, str]]:
58
+ """Return known words and (for Polish) accent-folded lookup tables."""
59
+ import wordfreq
60
+
61
+ words = wordfreq.top_n_list(lang, size)
62
+ if lang != "pl":
63
+ return frozenset(words), {}, {}
64
+ index: dict[str, str] = {}
65
+ accented: dict[str, str] = {}
66
+ for word in words:
67
+ key = fold(word)
68
+ index.setdefault(key, word)
69
+ if word != key:
70
+ accented.setdefault(key, word)
71
+ return frozenset(words), index, accented
72
+
73
+
74
+ @lru_cache(maxsize=1)
75
+ def _morfeusz():
76
+ import morfeusz2
77
+
78
+ return morfeusz2.Morfeusz(generate=False)
79
+
80
+
81
+ @lru_cache(maxsize=65536)
82
+ def is_word(word: str, lang: str = "pl") -> bool:
83
+ """Tell whether a form exists in the frequency lexicon (or Morfeusz2 for Polish)."""
84
+ known = lexicon(lang)[0]
85
+ if word.lower() in known:
86
+ return True
87
+ if lang != "pl":
88
+ return False
89
+ analyser = _morfeusz()
90
+ return any(item[2][2] != "ign" for item in analyser.analyse(word))
91
+
92
+
93
+ def _frequency(word: str, lang: str = "pl") -> float:
94
+ """Zipf frequency of a word (0 when unknown)."""
95
+ import wordfreq
96
+
97
+ return wordfreq.zipf_frequency(word, lang)
98
+
99
+
100
+ def _variants(key: str) -> list[str]:
101
+ """Folded variants where up to MAX_SUBSTITUTIONS look-alike letters are read as 'l' (folded 'ł')."""
102
+ positions = [i for i, c in enumerate(key) if c in L_LOOKALIKES]
103
+ out = []
104
+ for count in range(1, min(MAX_SUBSTITUTIONS, len(positions)) + 1):
105
+ for chosen in itertools.combinations(positions, count):
106
+ chars = list(key)
107
+ for i in chosen:
108
+ chars[i] = "l"
109
+ out.append("".join(chars))
110
+ return out
111
+
112
+
113
+ def _restore_with_morfeusz(lower: str, max_changes: int = 2) -> str | None:
114
+ """Find a Polish dictionary form differing by at most two diacritic or 'ł' restorations."""
115
+ if len(lower) > 24:
116
+ return None
117
+ positions = [i for i, c in enumerate(lower) if c in ACCENTS]
118
+ if len(positions) > 8:
119
+ max_changes = 1
120
+ for count in range(1, max_changes + 1):
121
+ for chosen in itertools.combinations(positions, count):
122
+ for letters in itertools.product(*(ACCENTS[lower[i]] for i in chosen)):
123
+ chars = list(lower)
124
+ for i, letter in zip(chosen, letters):
125
+ chars[i] = letter
126
+ candidate = "".join(chars)
127
+ if is_word(candidate, "pl"):
128
+ return candidate
129
+ return None
130
+
131
+
132
+ def _match_case(original: str, word: str) -> str:
133
+ """Apply the capitalization pattern of the original token to the replacement."""
134
+ if original.isupper() and len(original) > 1:
135
+ return word.upper()
136
+ if original[0].isupper():
137
+ return word[0].upper() + word[1:]
138
+ return word
139
+
140
+
141
+ def _correct_english_word(token: str, known: frozenset[str], min_zipf: float) -> str:
142
+ lower = token.lower()
143
+ if len(lower) < 5 or lower in known:
144
+ return token
145
+ candidates = set()
146
+ for old, new in EN_OCR_PAIRS:
147
+ start = 0
148
+ while (idx := lower.find(old, start)) != -1:
149
+ candidate = lower[:idx] + new + lower[idx + len(old):]
150
+ if candidate in known:
151
+ candidates.add(candidate)
152
+ start = idx + 1
153
+ if not candidates:
154
+ return token
155
+ best = max(candidates, key=lambda word: _frequency(word, "en"))
156
+ # Require stronger evidence for title-cased words so proper nouns stay intact.
157
+ threshold = max(min_zipf, 3.5) if token[0].isupper() and not token.isupper() else min_zipf
158
+ if _frequency(best, "en") < threshold:
159
+ return token
160
+ return _match_case(token, best)
161
+
162
+
163
+ def _correct_polish_word(token: str, known: frozenset[str], index: dict[str, str],
164
+ accented: dict[str, str], min_zipf: float) -> str:
165
+ lower = token.lower()
166
+ key = fold(lower)
167
+ if lower in known:
168
+ better = accented.get(key)
169
+ if better and lower == key and _frequency(better, "pl") - _frequency(lower, "pl") >= ACCENT_MARGIN:
170
+ return _match_case(token, better)
171
+ return token
172
+ if key in index and index[key] != lower:
173
+ return _match_case(token, index[key])
174
+ if is_word(token, "pl"):
175
+ return token
176
+ candidates = [index[variant] for variant in _variants(key) if variant in index]
177
+ if not candidates:
178
+ restored = _restore_with_morfeusz(lower)
179
+ return _match_case(token, restored) if restored else token
180
+ best = max(candidates, key=lambda word: _frequency(word, "pl"))
181
+ if _frequency(best, "pl") < min_zipf:
182
+ return token
183
+ return _match_case(token, best)
184
+
185
+
186
+ def correct_word(token: str, min_zipf: float | None = None, lang: str = "pl") -> str:
187
+ """Return the most likely intended word for an OCR token, or the token itself."""
188
+ if len(token) > 48:
189
+ return token
190
+ known, index, accented = lexicon(lang)
191
+ if not known:
192
+ return token
193
+ if lang == "en":
194
+ return _correct_english_word(token, known, 1.5 if min_zipf is None else min_zipf)
195
+ return _correct_polish_word(token, known, index, accented, 2.0 if min_zipf is None else min_zipf)
196
+
197
+
198
+ # A line-ending hyphen splits one lexical item into two apparent words.
199
+ # Keep both fragments intact, including their original line separator.
200
+ SPELL_TOKEN = re.compile(r"(?P<split>[^\W\d_]+[-‐]\s*\n[ \t]*[^\W\d_]+)|" + WORD.pattern)
201
+
202
+
203
+ def correct_text(text: str, min_zipf: float | None = None, lang: str = "pl") -> str:
204
+ """Repair every word of an OCR text that is not in the lexicon."""
205
+ return SPELL_TOKEN.sub(
206
+ lambda m: m[0] if m["split"] else correct_word(m[0], min_zipf, lang=lang),
207
+ text,
208
+ )
209
+
210
+
211
+ def needs_second_read(text: str, lang: str = "pl") -> bool:
212
+ return any(not is_word(word, lang) for word in WORD.findall(text))
213
+
214
+
215
+ def reconcile_words(text: str, second_read: str, lang: str = "pl") -> str:
216
+ """Repair unknown words only when a second reading supplies a similar known form."""
217
+ tokens = list(re.finditer(r"\S+", text))
218
+ first = [token.group() for token in tokens]
219
+ second = second_read.split()
220
+ pieces = []
221
+ last = 0
222
+ for tag, a, b, c, d in SequenceMatcher(None, first, second, autojunk=False).get_opcodes():
223
+ if tag != "replace" or b - a != d - c:
224
+ continue
225
+ for token, candidate in zip(tokens[a:b], second[c:d]):
226
+ original = token.group()
227
+ match = WORD.search(original)
228
+ other = WORD.search(candidate)
229
+ if not match or not other:
230
+ continue
231
+ before, after = match.group(), other.group()
232
+ if (original[:match.start()] != candidate[:other.start()]
233
+ or original[match.end():] != candidate[other.end():]):
234
+ continue
235
+ if len(before) != len(after) or sum(x != y for x, y in zip(before, after)) > 1:
236
+ continue
237
+ if not is_word(before, lang) and is_word(after, lang):
238
+ pieces.extend((text[last:token.start()], candidate))
239
+ last = token.end()
240
+ pieces.append(text[last:])
241
+ return "".join(pieces)
@@ -0,0 +1,48 @@
1
+ """Offline recognition after a one-time download of tessdata_best LSTM models."""
2
+
3
+ from functools import lru_cache
4
+ from pathlib import Path
5
+
6
+ import pooch
7
+ from tesserocr import PSM, PyTessBaseAPI
8
+
9
+ _BASE_URL = (
10
+ "https://raw.githubusercontent.com/tesseract-ocr/tessdata_best/"
11
+ "e12c65a915945e4c28e237a9b52bc4a8f39a0cec/"
12
+ )
13
+
14
+ _MODELS = {
15
+ "pl": (
16
+ "pol",
17
+ "sha256:e80cc4cefbdface06e9223f43f089556b9dcf104020fbc0a200f6863c57d4405",
18
+ ),
19
+ "en": (
20
+ "eng",
21
+ "sha256:8280aed0782fe27257a68ea10fe7ef324ca0f8d85bd2fd145d1c2b560bcb66ba",
22
+ ),
23
+ }
24
+
25
+
26
+ @lru_cache(maxsize=2)
27
+ def _model_directory(lang: str = "pl") -> tuple[Path, str]:
28
+ tess_lang, known_hash = _MODELS[lang]
29
+ filename = f"{tess_lang}.traineddata"
30
+ model = pooch.retrieve(
31
+ url=_BASE_URL + filename,
32
+ known_hash=known_hash,
33
+ path=pooch.os_cache("czytajpdf") / "tessdata_best",
34
+ fname=filename,
35
+ )
36
+ return Path(model).parent, tess_lang
37
+
38
+
39
+ def open_engine(lang: str = "pl"):
40
+ model_dir, tess_lang = _model_directory(lang)
41
+ return PyTessBaseAPI(path=str(model_dir), lang=tess_lang, psm=PSM.AUTO)
42
+
43
+
44
+ def read_page(engine, image, dpi):
45
+ engine.ClearAdaptiveClassifier()
46
+ engine.SetImage(image)
47
+ engine.SetSourceResolution(dpi)
48
+ return engine.GetUTF8Text().strip()
@@ -0,0 +1,119 @@
1
+ Metadata-Version: 2.4
2
+ Name: czytajpdf
3
+ Version: 0.3.0
4
+ Summary: Page-wise Polish and English PDF text extraction with optional OCR and spelling repair
5
+ License-Expression: Apache-2.0
6
+ Project-URL: Homepage, https://github.com/praha6248/CzytajPDF
7
+ Project-URL: Repository, https://github.com/praha6248/CzytajPDF
8
+ Project-URL: Issues, https://github.com/praha6248/CzytajPDF/issues
9
+ Requires-Python: >=3.10
10
+ Description-Content-Type: text/markdown
11
+ License-File: LICENSE
12
+ Requires-Dist: pypdf>=5
13
+ Provides-Extra: ocr
14
+ Requires-Dist: tesserocr<3,>=2.8; extra == "ocr"
15
+ Requires-Dist: pooch<2,>=1.8; extra == "ocr"
16
+ Requires-Dist: rapidocr<4,>=3.10; extra == "ocr"
17
+ Requires-Dist: onnxruntime>=1.20; extra == "ocr"
18
+ Requires-Dist: pypdfium2>=4.30; extra == "ocr"
19
+ Requires-Dist: numpy>=1.24; extra == "ocr"
20
+ Requires-Dist: pillow>=10; extra == "ocr"
21
+ Provides-Extra: polish
22
+ Requires-Dist: morfeusz2>=1.99; extra == "polish"
23
+ Requires-Dist: wordfreq>=3.1; extra == "polish"
24
+ Provides-Extra: pl
25
+ Requires-Dist: morfeusz2>=1.99; extra == "pl"
26
+ Requires-Dist: wordfreq>=3.1; extra == "pl"
27
+ Provides-Extra: english
28
+ Requires-Dist: wordfreq>=3.1; extra == "english"
29
+ Provides-Extra: en
30
+ Requires-Dist: wordfreq>=3.1; extra == "en"
31
+ Dynamic: license-file
32
+
33
+ # CzytajPDF
34
+
35
+ [![CI](https://github.com/praha6248/CzytajPDF/actions/workflows/ci.yml/badge.svg)](https://github.com/praha6248/CzytajPDF/actions/workflows/ci.yml)
36
+
37
+ **Polskie i angielskie PDF-y oraz skany w jednym wywołaniu Pythona.** CzytajPDF odczytuje dokument strona po stronie. Gdy pismo ma warstwę tekstową, a załącznik jest skanem, dostajesz jeden wynik z informacją, skąd pochodzi tekst każdej strony. Opcjonalna korekta przywraca polskie znaki lub naprawia typowe błędy OCR po angielsku, zachowując oryginalny odczyt do porównania.
38
+
39
+ > **English summary:** Page-wise Python library (Apache-2.0) for hybrid Polish and English PDFs — reads embedded text via `pypdf`, automatically falls back to local OCR (`tesserocr` / `rapidocr`) on scanned pages, and optionally applies Polish (`morfeusz2` + `wordfreq`) or English (`wordfreq`) OCR repair with dual-resolution reconciliation while preserving `source` and `raw_text`. Also packaged as `readpdf-pl` and `readpdf-en` from the same repository. See [README.en.md](README.en.md).
40
+
41
+ ## Pakiety w repozytorium
42
+
43
+ Jedno repozytorium dostarcza wspólną bibliotekę bazową oraz dwie paczki językowe o prostszej konfiguracji domyślnej:
44
+
45
+ - **`czytajpdf`** — pełna biblioteka z parametrem `lang="pl"` (domyślnie) lub `lang="en"`
46
+ - **`readpdf-pl`** (`packages/readpdf-pl`) — paczka dla dokumentów polskich (domyślnie `lang="pl"`)
47
+ - **`readpdf-en`** (`packages/readpdf-en`) — paczka dla dokumentów angielskich (domyślnie `lang="en"`, bez zależności od `morfeusz2`)
48
+
49
+ ## Pierwszy odczyt
50
+
51
+ Python 3.10+. Instalacja z GitHuba z obsługą OCR i korekty:
52
+
53
+ ```sh
54
+ # Polski i angielski (pełna instalacja)
55
+ python -m pip install 'czytajpdf[ocr,polish] @ git+https://github.com/praha6248/CzytajPDF.git'
56
+
57
+ # Tylko angielski (bez morfeusz2)
58
+ python -m pip install 'czytajpdf[ocr,english] @ git+https://github.com/praha6248/CzytajPDF.git'
59
+ ```
60
+
61
+ ```python
62
+ from czytajpdf import extract
63
+
64
+ dokument = extract("pismo-z-zalacznikami.pdf", ocr=True, correct=True)
65
+ for strona in dokument.pages:
66
+ print(strona.number, strona.source, strona.text)
67
+
68
+ english_doc = extract("filing-with-exhibits.pdf", ocr=True, correct=True, lang="en")
69
+ ```
70
+
71
+ Do PDF-ów z warstwą tekstową wystarczy instalacja bez dodatków i wywołanie `extract("pismo.pdf")`. Cały tekst jest dostępny jako `dokument.text`. Numeracja stron zaczyna się od 1. Z paczek językowych można korzystać przez `from readpdf_pl import extract` oraz `from readpdf_en import extract`.
72
+
73
+ ## Statystyki odczytu
74
+
75
+ | Dokument | Język | Strony | Strony z warstwy tekstowej | Strony z OCR | Znaki z OCR |
76
+ | --- | --- | ---: | ---: | ---: | ---: |
77
+ | [Dz.U. 2024 poz. 1](https://api.sejm.gov.pl/eli/acts/DU/2024/1/text.pdf) | `pl` | 8 | 8 | 0 | 0 |
78
+ | [Dz.U. 1945 nr 5 poz. 17](https://api.sejm.gov.pl/eli/acts/DU/1945/17/text.pdf) | `pl` | 2 | 2 | 0 | 0 |
79
+ | PDF mieszany z pierwszych stron obu polskich dokumentów | `pl` | 2 | 1 | 1 | 2716 |
80
+ | [SCOTUS No. 22-451](https://www.supremecourt.gov/opinions/23pdf/22-451_7m58.pdf) | `en` | 114 | 114 | 0 | 0 |
81
+ | [Brown v. Board of Education, 347 U.S. 483](https://cdn.loc.gov/service/ll/usrep/usrep347/usrep347483/usrep347483.pdf) | `en` | 14 | 14 | 0 | 0 |
82
+ | PDF mieszany z pierwszych stron obu angielskich dokumentów | `en` | 2 | 1 | 1 | 2204 |
83
+
84
+ ## Skuteczność korekty polskiego OCR
85
+
86
+ Pomiar obejmuje nagłówki z 18 archiwalnych skanów Dziennika Ustaw. Na pierwotnej próbce 12 nagłówków **liczba błędów słów spadła z 79 do 2**. Do porównania użyto tych samych wycinków i transkrypcji.
87
+
88
+ | Próbka i wersja biblioteki | Błędy znaków (CER) po korekcie | Błędy słów (WER) po korekcie |
89
+ | --- | ---: | ---: |
90
+ | 12 pierwotnych nagłówków · 0.1.0, RapidOCR | 382 / 2129 · 17,94% | 79 / 293 · 26,96% |
91
+ | Te same 12 nagłówków · 0.2.0 / 0.3.0, Tesseract | 2 / 2129 · 0,09% | 2 / 293 · 0,68% |
92
+ | 3 dodatkowe nagłówki użyte do sprawdzenia i poprawienia obsługi dzielenia słów · 0.2.0 / 0.3.0 | 0 / 593 · 0,00% | 0 / 87 · 0,00% |
93
+ | 3 osobne nagłówki użyte do końcowej weryfikacji · 0.2.0 / 0.3.0 | 1 / 430 · 0,23% | 1 / 67 · 1,49% |
94
+
95
+ Na pierwotnej próbce surowy Tesseract popełnił 5 błędów słów, a drugi odczyt z korektą usunął 3. Pozostały pomyłki `r.` → `r,` oraz `i` → `1`. W końcowej weryfikacji OCR dodał kropkę po słowie `lutego`.
96
+
97
+ CER i WER uwzględniają zamiany, wstawienia i pominięcia względem transkrypcji z obrazu. Ujednolicono białe znaki, zachowując interpunkcję i wielkość liter. Wynik dotyczy wycinków z drukowanymi nagłówkami. [Pomiar 0.1.0](benchmarks/rapidocr-0.1.0.json) · [Pomiar 0.2.0/0.3.0](benchmarks/results.json) · [Próbka dodatkowa](benchmarks/holdout.json) · [Weryfikacja końcowa](benchmarks/validation.json) · [Skrypt pomiarowy](benchmarks/score.py)
98
+
99
+ ## Skuteczność angielskiego OCR (`lang="en"`)
100
+
101
+ Pomiar obejmuje 9 archiwalnych skanów *United States Reports* z Library of Congress (lata 1937–1989). Na 6 dokumentach próbki głównej korekta naprawiła błędy `Amendfnent` → `Amendment` oraz `eriminal` → `criminal` (spadek WER z 17 do 15 na 897 słowach), a na 3 osobnych dokumentach walidacyjnych nie wprowadziła żadnych fałszywych poprawek.
102
+
103
+ | Próbka angielska (0.3.0, Tesseract `eng`) | Surowy CER / WER | Po korekcie CER | Po korekcie WER |
104
+ | --- | ---: | ---: | ---: |
105
+ | 6 skanów *U.S. Reports* (1937–1984) | 0,35% / 1,90% (17 / 897) | 17 / 5745 · 0,30% | 15 / 897 · 1,67% |
106
+ | 3 osobne skany walidacyjne (1943, 1964, 1989) | 0,32% / 1,99% (6 / 301) | 6 / 1871 · 0,32% | 6 / 301 · 1,99% |
107
+
108
+ [Pomiar angielski](benchmarks/english_results.json) · [Walidacja angielska](benchmarks/english_validation.json)
109
+
110
+ ## Jak wybierany jest sposób odczytu
111
+
112
+ - Domyślnie działa pypdf. Przy `ocr=True` strony z mniej niż 40 znakami po pominięciu białych znaków trafiają do modelu Tesseract dla wybranego języka (`lang="pl"` albo `lang="en"`). Aby wybrać RapidOCR, przekaż `engine="rapidocr"`. Próg zmienisz przez `min_chars`. Długa, uszkodzona warstwa tekstowa może przejść ten próg.
113
+ - `source` przyjmuje wartość `pdf`, `tesseract` lub `rapidocr`. Pusty wynik OCR pozostawia tekst z warstwy PDF. Wynik zachowuje podział na strony, bez współrzędnych słów ani struktury tabel.
114
+ - `correct=True` włącza korektę wyników OCR: dla `lang="pl"` używa `wordfreq` i `Morfeusza 2`, a dla `lang="en"` `wordfreq` i ostrożnych zamian typowych pomyłek OCR. Tesseract może ponownie odczytać stronę w rozdzielczości 180 DPI, aby zweryfikować nierozpoznane słowa. Pierwszy odczyt pozostaje dostępny w `raw_text`. Przy nazwiskach i nazwach własnych warto porównać wynik z `raw_text`.
115
+ - Rozpoznawanie odbywa się lokalnie. Przy pierwszym użyciu Tesseract pobiera odpowiedni model `tessdata_best` (około 12 MB dla `pol` lub 15 MB dla `eng`), sprawdza SHA-256 i zapisuje plik w pamięci podręcznej. Kolejne odczyty działają offline. RapidOCR korzysta z osobnych modeli.
116
+
117
+ ## Licencja i pochodzenie
118
+
119
+ Kod biblioteki jest dostępny na [Apache-2.0](LICENSE). Projekt wykorzystuje logikę z [odwolania2json](https://github.com/praha6248/odwolania2Json). Wagi OCR, słowniki i dane częstości są osobnymi zależnościami na własnych licencjach.
@@ -0,0 +1,9 @@
1
+ czytajpdf/__init__.py,sha256=HTxeUXLpoL1LqVw0B92cCe_joBaSCS8Q4g9VHkJaHRI,2106
2
+ czytajpdf/_ocr.py,sha256=CzETc2gEvDVd-gQOj9Uc4CEr5zrIMotBtNi_PTuc7sg,2292
3
+ czytajpdf/_spell.py,sha256=uf9n0SuVWB9nGhGhC-HN_AOCcZcUFCa7rxJG-O3BP10,8820
4
+ czytajpdf/_tesseract.py,sha256=JI6m7izCKe8u9-_KdPXxGa74nddusQNbxmA4DAaUOIQ,1310
5
+ czytajpdf-0.3.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
6
+ czytajpdf-0.3.0.dist-info/METADATA,sha256=xDlzB1wJR7AaqsW_L2mxcP9ms5CJP_994e_ZPmG3qgE,8435
7
+ czytajpdf-0.3.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
8
+ czytajpdf-0.3.0.dist-info/top_level.txt,sha256=oeg1nW2c1gQGI_dv0vfr7YrOhzTvOmnxJ_A97WK6PMw,10
9
+ czytajpdf-0.3.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,202 @@
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright [yyyy] [name of copyright owner]
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
@@ -0,0 +1 @@
1
+ czytajpdf