polis-recognizer 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polis_recognizer-0.1.0/.gitignore +38 -0
- polis_recognizer-0.1.0/CHANGELOG.md +44 -0
- polis_recognizer-0.1.0/CONTRIBUTING.md +68 -0
- polis_recognizer-0.1.0/LICENSE +21 -0
- polis_recognizer-0.1.0/PKG-INFO +192 -0
- polis_recognizer-0.1.0/README.md +151 -0
- polis_recognizer-0.1.0/examples/basic_usage.py +43 -0
- polis_recognizer-0.1.0/polis_recognizer/__init__.py +98 -0
- polis_recognizer-0.1.0/polis_recognizer/contract_field_extractor.py +564 -0
- polis_recognizer-0.1.0/polis_recognizer/exceptions.py +33 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/__init__.py +22 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/candidates.py +89 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/layout.py +163 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/negation.py +78 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/normalizer.py +133 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/numeric.py +103 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/parsers/__init__.py +42 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/parsers/base.py +56 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/parsers/franchise.py +311 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/parsers/limit.py +256 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/parsers/policy_number.py +223 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/parsers/policy_period.py +256 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/parsers/premium.py +215 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/parsers/repair_mode.py +214 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/parsers/sum_type.py +168 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/pipeline.py +130 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/ranker.py +57 -0
- polis_recognizer-0.1.0/polis_recognizer/extraction/tables.py +122 -0
- polis_recognizer-0.1.0/polis_recognizer/extractor.py +242 -0
- polis_recognizer-0.1.0/polis_recognizer/hybrid_ingestion.py +72 -0
- polis_recognizer-0.1.0/polis_recognizer/image_preprocessing.py +223 -0
- polis_recognizer-0.1.0/polis_recognizer/ocr_config.py +246 -0
- polis_recognizer-0.1.0/polis_recognizer/ocr_service.py +838 -0
- polis_recognizer-0.1.0/polis_recognizer/pdf_extraction_router.py +320 -0
- polis_recognizer-0.1.0/polis_recognizer/pdfplumber_ingestion.py +98 -0
- polis_recognizer-0.1.0/polis_recognizer/policy_ingestion.py +145 -0
- polis_recognizer-0.1.0/pyproject.toml +96 -0
- polis_recognizer-0.1.0/tests/test_components.py +188 -0
- polis_recognizer-0.1.0/tests/test_hybrid_ingestion.py +90 -0
- polis_recognizer-0.1.0/tests/test_pipeline.py +339 -0
- polis_recognizer-0.1.0/tests/test_policy_number.py +110 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
.Python
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
*.egg
|
|
11
|
+
MANIFEST
|
|
12
|
+
|
|
13
|
+
# Virtual envs
|
|
14
|
+
venv/
|
|
15
|
+
.venv/
|
|
16
|
+
env/
|
|
17
|
+
|
|
18
|
+
# Testing
|
|
19
|
+
.pytest_cache/
|
|
20
|
+
.coverage
|
|
21
|
+
htmlcov/
|
|
22
|
+
.ruff_cache/
|
|
23
|
+
|
|
24
|
+
# IDE
|
|
25
|
+
.vscode/
|
|
26
|
+
.idea/
|
|
27
|
+
*.swp
|
|
28
|
+
.DS_Store
|
|
29
|
+
|
|
30
|
+
# OS
|
|
31
|
+
Thumbs.db
|
|
32
|
+
|
|
33
|
+
# Test fixtures generated at runtime
|
|
34
|
+
tests/fixtures/_generated/
|
|
35
|
+
|
|
36
|
+
# Local-only PDFs (don't commit any real policy)
|
|
37
|
+
*.pdf
|
|
38
|
+
!tests/fixtures/**/*.pdf
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [0.1.0] — 2026-05-03
|
|
9
|
+
|
|
10
|
+
Initial public release.
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- `PolicyExtractor` facade for end-to-end PDF → 7 structured fields.
|
|
15
|
+
- `extract_from_pdf` / `extract_from_bytes` / `extract_from_text` entry points.
|
|
16
|
+
- 7 deterministic field parsers:
|
|
17
|
+
- `policy_period`
|
|
18
|
+
- `franchise`
|
|
19
|
+
- `limit`
|
|
20
|
+
- `repair_mode`
|
|
21
|
+
- `premium`
|
|
22
|
+
- `sum_type`
|
|
23
|
+
- `policy_number`
|
|
24
|
+
- Three PDF extractor backends: `pypdf`, `pdfplumber`, `hybrid` (default).
|
|
25
|
+
- Hybrid mode reuses pypdf text and pdfplumber tables in one pass for the
|
|
26
|
+
best recall/latency trade-off on KASKO templates.
|
|
27
|
+
- Tesseract OCR fallback for scanned PDFs.
|
|
28
|
+
- OpenCV-based image preprocessing (`fallback` / `always` / `never` modes).
|
|
29
|
+
- Pre-built parser patterns for major Russian KASKO insurers:
|
|
30
|
+
АльфаСтрахование XLS forms (5/3/5/2 and 5/3/7/2 numbers, branch-letter
|
|
31
|
+
variants), СОГАЗ-АВТО (`SGZA…` policy numbers), Чулпан (OCR pipe
|
|
32
|
+
tolerance in policy_number), Ингосстрах (legacy `RUR` currency code,
|
|
33
|
+
prose-spaced premium label), ВСК (two-row КАСКО layout),
|
|
34
|
+
АбсолютСтрахование (glued text-layer detection), Diadoc/Kontur EDI
|
|
35
|
+
envelope detection.
|
|
36
|
+
- Lower-level `run_extraction(text, *, tables=None)` for use without a PDF.
|
|
37
|
+
|
|
38
|
+
### Notes
|
|
39
|
+
|
|
40
|
+
- API is pre-stable. Public dataclass shapes
|
|
41
|
+
(`ExtractedPolicy`, `MonetaryField`, etc.) may change before 1.0.
|
|
42
|
+
- KASKO-only for now; ОСАГО support is on the roadmap.
|
|
43
|
+
|
|
44
|
+
[0.1.0]: https://github.com/grigra27/polis-recognizer/releases/tag/v0.1.0
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Thanks for your interest in `polis-recognizer`. The project is
|
|
4
|
+
maintained as a side-project, so reviews aren't instant — but PRs are
|
|
5
|
+
welcome.
|
|
6
|
+
|
|
7
|
+
## What contributions are most useful
|
|
8
|
+
|
|
9
|
+
The single most valuable contribution is a **parser pattern for an
|
|
10
|
+
insurer format we don't recognize yet**. The recognizer ships with
|
|
11
|
+
patterns for the major Russian KASKO insurers (АльфаСтрахование, СОГАЗ,
|
|
12
|
+
Чулпан, Ингосстрах, ВСК, АбсолютСтрахование, Росгосстрах). Other
|
|
13
|
+
insurers, or unusual templates from these, often don't extract well.
|
|
14
|
+
|
|
15
|
+
## How to report a new format
|
|
16
|
+
|
|
17
|
+
**Don't paste real policy PDFs in the issue tracker** — they're personal
|
|
18
|
+
data. Instead:
|
|
19
|
+
|
|
20
|
+
1. Open an issue titled `New format: <Insurer name>`.
|
|
21
|
+
2. Paste the **plain-text** extraction of the relevant rows
|
|
22
|
+
(you can run it yourself with
|
|
23
|
+
`from polis_recognizer import PolicyExtractor; print(PolicyExtractor().extract_from_pdf("polis.pdf"))`
|
|
24
|
+
and copy the relevant text from the diagnostics, OR run
|
|
25
|
+
`pdftotext` on the file).
|
|
26
|
+
3. List which fields are missing or wrong, with the expected values.
|
|
27
|
+
4. Anonymize anything that identifies a real client (insured name,
|
|
28
|
+
address, VIN, policy number — replace with placeholders).
|
|
29
|
+
|
|
30
|
+
That's enough to write a regex/table pattern. We don't need the
|
|
31
|
+
original PDF.
|
|
32
|
+
|
|
33
|
+
## How to add a parser pattern
|
|
34
|
+
|
|
35
|
+
Each field has its own parser at
|
|
36
|
+
`polis_recognizer/extraction/parsers/<field>.py`. The structure is a
|
|
37
|
+
list of `(pattern_id, regex, pattern_strength, context_strength)`
|
|
38
|
+
tuples. Add yours, run the test suite, and open a PR.
|
|
39
|
+
|
|
40
|
+
Patterns should be **specific** — a pattern that matches "anything that
|
|
41
|
+
looks like a policy number" will produce false positives across the
|
|
42
|
+
corpus. Anchor on insurer-specific labels or layout artifacts when
|
|
43
|
+
possible.
|
|
44
|
+
|
|
45
|
+
## Testing
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
# Install dev dependencies
|
|
49
|
+
pip install -e ".[test]"
|
|
50
|
+
|
|
51
|
+
# Run tests
|
|
52
|
+
pytest
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The test suite uses synthetic PDFs generated via `reportlab` — no real
|
|
56
|
+
policies are committed to the repo.
|
|
57
|
+
|
|
58
|
+
## Code style
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
ruff check polis_recognizer tests
|
|
62
|
+
ruff format polis_recognizer tests
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## License
|
|
66
|
+
|
|
67
|
+
By contributing, you agree your contribution is licensed under the
|
|
68
|
+
project's [MIT License](LICENSE).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Grigorii Grachev
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: polis-recognizer
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Russian KASKO insurance policy field extractor — pulls 7 structured fields from PDF policies (text-layer + tables + OCR fallback).
|
|
5
|
+
Project-URL: Homepage, https://github.com/grigra27/polis-recognizer
|
|
6
|
+
Project-URL: Repository, https://github.com/grigra27/polis-recognizer
|
|
7
|
+
Project-URL: Issues, https://github.com/grigra27/polis-recognizer/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/grigra27/polis-recognizer/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: Grigorii Grachev <grigorii@example.com>
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: extraction,insurance,kasko,ocr,pdf,policy,russian
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Natural Language :: Russian
|
|
17
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
18
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Topic :: Office/Business
|
|
23
|
+
Classifier: Topic :: Text Processing
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Requires-Dist: opencv-python-headless<5,>=4.8
|
|
26
|
+
Requires-Dist: pdf2image<2,>=1.16
|
|
27
|
+
Requires-Dist: pdfplumber<1,>=0.11.0
|
|
28
|
+
Requires-Dist: pillow<12,>=10.0
|
|
29
|
+
Requires-Dist: pypdf<6,>=3.17.4
|
|
30
|
+
Requires-Dist: pytesseract<1,>=0.3.10
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
34
|
+
Requires-Dist: reportlab>=4.0; extra == 'dev'
|
|
35
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
36
|
+
Provides-Extra: test
|
|
37
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'test'
|
|
38
|
+
Requires-Dist: pytest>=8.0; extra == 'test'
|
|
39
|
+
Requires-Dist: reportlab>=4.0; extra == 'test'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# polis-recognizer
|
|
43
|
+
|
|
44
|
+
A deterministic field extractor for Russian KASKO insurance policy PDFs.
|
|
45
|
+
Pulls 7 structured fields without LLMs — text-layer extraction
|
|
46
|
+
(`pypdf`) plus optional table-aware reading (`pdfplumber`), with
|
|
47
|
+
Tesseract OCR fallback for scanned policies.
|
|
48
|
+
|
|
49
|
+
> **Status: pre-stable (0.x).** API may change before 1.0.
|
|
50
|
+
|
|
51
|
+
## What it extracts
|
|
52
|
+
|
|
53
|
+
| Field | Type | Example |
|
|
54
|
+
|---|---|---|
|
|
55
|
+
| `policy_number` | `str` | `"AC524160804"` |
|
|
56
|
+
| `policy_period` | `{start, end}` (`date`) | `{"start": date(2025, 2, 27), "end": date(2026, 2, 26)}` |
|
|
57
|
+
| `franchise` | `{value, currency, absent}` | `{"value": 30000, "currency": "RUB", "absent": False}` |
|
|
58
|
+
| `limit` | `{value, currency}` | `{"value": 5525000, "currency": "RUB"}` |
|
|
59
|
+
| `premium` | `{value, currency}` | `{"value": 220000, "currency": "RUB"}` |
|
|
60
|
+
| `sum_type` | `"aggregate"` / `"non_aggregate"` | `"non_aggregate"` |
|
|
61
|
+
| `repair_mode` | `"dealer"` / `"service"` / `"cash"` | `"dealer"` |
|
|
62
|
+
|
|
63
|
+
## Quick start
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from polis_recognizer import PolicyExtractor
|
|
67
|
+
|
|
68
|
+
extractor = PolicyExtractor()
|
|
69
|
+
result = extractor.extract_from_pdf("/path/to/polis.pdf")
|
|
70
|
+
|
|
71
|
+
print(result.policy_number)
|
|
72
|
+
# → "AC524160804"
|
|
73
|
+
print(result.policy_period)
|
|
74
|
+
# → {"start": date(2025, 2, 27), "end": date(2026, 2, 26)}
|
|
75
|
+
print(result.franchise)
|
|
76
|
+
# → {"value": 30000.0, "currency": "RUB", "absent": False}
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Input methods:
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
extractor.extract_from_pdf("polis.pdf")
|
|
83
|
+
extractor.extract_from_bytes(pdf_bytes, filename="polis.pdf")
|
|
84
|
+
extractor.extract_from_text("сырой текст полиса") # bypass PDF/OCR
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Installation
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
pip install polis-recognizer
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### System dependencies
|
|
94
|
+
|
|
95
|
+
`polis-recognizer` shells out to Tesseract for OCR and to `poppler`
|
|
96
|
+
for PDF→image conversion. These are NOT pip-installable; install them
|
|
97
|
+
through your OS package manager.
|
|
98
|
+
|
|
99
|
+
**Linux (Debian/Ubuntu):**
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
sudo apt-get install -y tesseract-ocr tesseract-ocr-rus poppler-utils libgl1
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
**macOS (Homebrew):**
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
brew install tesseract tesseract-lang poppler
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
**Windows:** best-effort. Install
|
|
112
|
+
[Tesseract for Windows](https://github.com/UB-Mannheim/tesseract/wiki)
|
|
113
|
+
and add it to PATH; install Poppler binaries for PDF support. We don't
|
|
114
|
+
test on Windows in CI.
|
|
115
|
+
|
|
116
|
+
The Russian language pack (`tesseract-ocr-rus` / `tesseract-lang`) is
|
|
117
|
+
required — without it, OCR silently falls back to English and Cyrillic
|
|
118
|
+
documents come back as garbage. The library logs a CRITICAL warning at
|
|
119
|
+
import time if the pack is missing.
|
|
120
|
+
|
|
121
|
+
## How it works
|
|
122
|
+
|
|
123
|
+
The extractor runs three stages:
|
|
124
|
+
|
|
125
|
+
1. **PDF ingestion** — `PdfExtractionRouter` tries text-layer extraction
|
|
126
|
+
first (pypdf for text, pdfplumber for tables on the same page). If
|
|
127
|
+
the result is too short (fewer than 100 chars by default) or detected
|
|
128
|
+
as glued/EDI-envelope text, it falls back to Tesseract OCR.
|
|
129
|
+
2. **Text normalization** — Unicode NFKC, NBSP stripping, hyphenated
|
|
130
|
+
line-break healing, runs of multiple spaces collapsed.
|
|
131
|
+
3. **Field extraction** — 7 deterministic parsers (one per field) run
|
|
132
|
+
regex + table-aware patterns and emit ``Candidate``s with confidence
|
|
133
|
+
scores. A ranker picks the winner per field.
|
|
134
|
+
|
|
135
|
+
There's no LLM and no cloud dependency. Everything runs locally.
|
|
136
|
+
|
|
137
|
+
## PDF extractor choice
|
|
138
|
+
|
|
139
|
+
The default is ``"hybrid"`` — pypdf text plus pdfplumber tables in one
|
|
140
|
+
pass. Two alternatives:
|
|
141
|
+
|
|
142
|
+
| Option | When to use |
|
|
143
|
+
|---|---|
|
|
144
|
+
| `"hybrid"` (default) | Best for KASKO. pypdf preserves date/period text quality, pdfplumber's tables fix the column layout for limit/franchise/premium. |
|
|
145
|
+
| `"pypdf"` | Faster, no tables. Use when document quality is uniform and tables aren't needed. |
|
|
146
|
+
| `"pdfplumber"` | Fully layout-aware. Slower; on KASKO it slightly regresses date parsing. Use for table-heavy non-KASKO formats. |
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
extractor = PolicyExtractor(pdf_extractor="pypdf")
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## Supported insurer formats
|
|
153
|
+
|
|
154
|
+
The parser ships with patterns for these Russian insurers' KASKO
|
|
155
|
+
templates: АльфаСтрахование (XLS form-mask), СОГАЗ-АВТО, Чулпан,
|
|
156
|
+
Ингосстрах, ВСК, АбсолютСтрахование, Росгосстрах, СОГАЗ Diadoc-wrapped
|
|
157
|
+
PDFs. Recall on real-world KASKO corpora is ~50-65% per field; pulling
|
|
158
|
+
above that requires per-format parser additions.
|
|
159
|
+
|
|
160
|
+
If you have a policy from an insurer not on this list — see
|
|
161
|
+
[CONTRIBUTING.md](CONTRIBUTING.md) for how to add a parser pattern.
|
|
162
|
+
|
|
163
|
+
## Configuration
|
|
164
|
+
|
|
165
|
+
Constructor arguments:
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
extractor = PolicyExtractor(
|
|
169
|
+
ocr_language="rus+eng", # Tesseract language string
|
|
170
|
+
ocr_timeout_seconds=300,
|
|
171
|
+
ocr_page_limit=50,
|
|
172
|
+
ocr_max_text_size=500_000,
|
|
173
|
+
pdf_extractor="hybrid", # "pypdf" | "pdfplumber" | "hybrid"
|
|
174
|
+
image_preprocessing="fallback", # "never" | "fallback" | "always"
|
|
175
|
+
psm=None, # Tesseract --psm (None = auto)
|
|
176
|
+
oem=None, # Tesseract --oem (None = auto)
|
|
177
|
+
max_image_size_bytes=None, # reject images larger than this
|
|
178
|
+
)
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
## License
|
|
182
|
+
|
|
183
|
+
[MIT](LICENSE) © Grigorii Grachev. Free for any use, including
|
|
184
|
+
commercial.
|
|
185
|
+
|
|
186
|
+
## Roadmap
|
|
187
|
+
|
|
188
|
+
This 0.1.0 release focuses on KASKO. ОСАГО support is on the roadmap —
|
|
189
|
+
the underlying field model already accommodates it; only parser
|
|
190
|
+
patterns need to be added.
|
|
191
|
+
|
|
192
|
+
See [CHANGELOG.md](CHANGELOG.md) for release history.
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# polis-recognizer
|
|
2
|
+
|
|
3
|
+
A deterministic field extractor for Russian KASKO insurance policy PDFs.
|
|
4
|
+
Pulls 7 structured fields without LLMs — text-layer extraction
|
|
5
|
+
(`pypdf`) plus optional table-aware reading (`pdfplumber`), with
|
|
6
|
+
Tesseract OCR fallback for scanned policies.
|
|
7
|
+
|
|
8
|
+
> **Status: pre-stable (0.x).** API may change before 1.0.
|
|
9
|
+
|
|
10
|
+
## What it extracts
|
|
11
|
+
|
|
12
|
+
| Field | Type | Example |
|
|
13
|
+
|---|---|---|
|
|
14
|
+
| `policy_number` | `str` | `"AC524160804"` |
|
|
15
|
+
| `policy_period` | `{start, end}` (`date`) | `{"start": date(2025, 2, 27), "end": date(2026, 2, 26)}` |
|
|
16
|
+
| `franchise` | `{value, currency, absent}` | `{"value": 30000, "currency": "RUB", "absent": False}` |
|
|
17
|
+
| `limit` | `{value, currency}` | `{"value": 5525000, "currency": "RUB"}` |
|
|
18
|
+
| `premium` | `{value, currency}` | `{"value": 220000, "currency": "RUB"}` |
|
|
19
|
+
| `sum_type` | `"aggregate"` / `"non_aggregate"` | `"non_aggregate"` |
|
|
20
|
+
| `repair_mode` | `"dealer"` / `"service"` / `"cash"` | `"dealer"` |
|
|
21
|
+
|
|
22
|
+
## Quick start
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
from polis_recognizer import PolicyExtractor
|
|
26
|
+
|
|
27
|
+
extractor = PolicyExtractor()
|
|
28
|
+
result = extractor.extract_from_pdf("/path/to/polis.pdf")
|
|
29
|
+
|
|
30
|
+
print(result.policy_number)
|
|
31
|
+
# → "AC524160804"
|
|
32
|
+
print(result.policy_period)
|
|
33
|
+
# → {"start": date(2025, 2, 27), "end": date(2026, 2, 26)}
|
|
34
|
+
print(result.franchise)
|
|
35
|
+
# → {"value": 30000.0, "currency": "RUB", "absent": False}
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Input methods:
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
extractor.extract_from_pdf("polis.pdf")
|
|
42
|
+
extractor.extract_from_bytes(pdf_bytes, filename="polis.pdf")
|
|
43
|
+
extractor.extract_from_text("сырой текст полиса") # bypass PDF/OCR
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Installation
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install polis-recognizer
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
### System dependencies
|
|
53
|
+
|
|
54
|
+
`polis-recognizer` shells out to Tesseract for OCR and to `poppler`
|
|
55
|
+
for PDF→image conversion. These are NOT pip-installable; install them
|
|
56
|
+
through your OS package manager.
|
|
57
|
+
|
|
58
|
+
**Linux (Debian/Ubuntu):**
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
sudo apt-get install -y tesseract-ocr tesseract-ocr-rus poppler-utils libgl1
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
**macOS (Homebrew):**
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
brew install tesseract tesseract-lang poppler
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
**Windows:** best-effort. Install
|
|
71
|
+
[Tesseract for Windows](https://github.com/UB-Mannheim/tesseract/wiki)
|
|
72
|
+
and add it to PATH; install Poppler binaries for PDF support. We don't
|
|
73
|
+
test on Windows in CI.
|
|
74
|
+
|
|
75
|
+
The Russian language pack (`tesseract-ocr-rus` / `tesseract-lang`) is
|
|
76
|
+
required — without it, OCR silently falls back to English and Cyrillic
|
|
77
|
+
documents come back as garbage. The library logs a CRITICAL warning at
|
|
78
|
+
import time if the pack is missing.
|
|
79
|
+
|
|
80
|
+
## How it works
|
|
81
|
+
|
|
82
|
+
The extractor runs three stages:
|
|
83
|
+
|
|
84
|
+
1. **PDF ingestion** — `PdfExtractionRouter` tries text-layer extraction
|
|
85
|
+
first (pypdf for text, pdfplumber for tables on the same page). If
|
|
86
|
+
the result is too short (fewer than 100 chars by default) or detected
|
|
87
|
+
as glued/EDI-envelope text, it falls back to Tesseract OCR.
|
|
88
|
+
2. **Text normalization** — Unicode NFKC, NBSP stripping, hyphenated
|
|
89
|
+
line-break healing, runs of multiple spaces collapsed.
|
|
90
|
+
3. **Field extraction** — 7 deterministic parsers (one per field) run
|
|
91
|
+
regex + table-aware patterns and emit ``Candidate``s with confidence
|
|
92
|
+
scores. A ranker picks the winner per field.
|
|
93
|
+
|
|
94
|
+
There's no LLM and no cloud dependency. Everything runs locally.
|
|
95
|
+
|
|
96
|
+
## PDF extractor choice
|
|
97
|
+
|
|
98
|
+
The default is ``"hybrid"`` — pypdf text plus pdfplumber tables in one
|
|
99
|
+
pass. Two alternatives:
|
|
100
|
+
|
|
101
|
+
| Option | When to use |
|
|
102
|
+
|---|---|
|
|
103
|
+
| `"hybrid"` (default) | Best for KASKO. pypdf preserves date/period text quality, pdfplumber's tables fix the column layout for limit/franchise/premium. |
|
|
104
|
+
| `"pypdf"` | Faster, no tables. Use when document quality is uniform and tables aren't needed. |
|
|
105
|
+
| `"pdfplumber"` | Fully layout-aware. Slower; on KASKO it slightly regresses date parsing. Use for table-heavy non-KASKO formats. |
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
extractor = PolicyExtractor(pdf_extractor="pypdf")
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## Supported insurer formats
|
|
112
|
+
|
|
113
|
+
The parser ships with patterns for these Russian insurers' KASKO
|
|
114
|
+
templates: АльфаСтрахование (XLS form-mask), СОГАЗ-АВТО, Чулпан,
|
|
115
|
+
Ингосстрах, ВСК, АбсолютСтрахование, Росгосстрах, СОГАЗ Diadoc-wrapped
|
|
116
|
+
PDFs. Recall on real-world KASKO corpora is ~50-65% per field; pulling
|
|
117
|
+
above that requires per-format parser additions.
|
|
118
|
+
|
|
119
|
+
If you have a policy from an insurer not on this list — see
|
|
120
|
+
[CONTRIBUTING.md](CONTRIBUTING.md) for how to add a parser pattern.
|
|
121
|
+
|
|
122
|
+
## Configuration
|
|
123
|
+
|
|
124
|
+
Constructor arguments:
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
extractor = PolicyExtractor(
|
|
128
|
+
ocr_language="rus+eng", # Tesseract language string
|
|
129
|
+
ocr_timeout_seconds=300,
|
|
130
|
+
ocr_page_limit=50,
|
|
131
|
+
ocr_max_text_size=500_000,
|
|
132
|
+
pdf_extractor="hybrid", # "pypdf" | "pdfplumber" | "hybrid"
|
|
133
|
+
image_preprocessing="fallback", # "never" | "fallback" | "always"
|
|
134
|
+
psm=None, # Tesseract --psm (None = auto)
|
|
135
|
+
oem=None, # Tesseract --oem (None = auto)
|
|
136
|
+
max_image_size_bytes=None, # reject images larger than this
|
|
137
|
+
)
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## License
|
|
141
|
+
|
|
142
|
+
[MIT](LICENSE) © Grigorii Grachev. Free for any use, including
|
|
143
|
+
commercial.
|
|
144
|
+
|
|
145
|
+
## Roadmap
|
|
146
|
+
|
|
147
|
+
This 0.1.0 release focuses on KASKO. ОСАГО support is on the roadmap —
|
|
148
|
+
the underlying field model already accommodates it; only parser
|
|
149
|
+
patterns need to be added.
|
|
150
|
+
|
|
151
|
+
See [CHANGELOG.md](CHANGELOG.md) for release history.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Minimal example — extract structured fields from a single policy PDF."""
|
|
2
|
+
|
|
3
|
+
from polis_recognizer import PolicyExtractor
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def main(pdf_path: str) -> None:
|
|
7
|
+
extractor = PolicyExtractor()
|
|
8
|
+
result = extractor.extract_from_pdf(pdf_path)
|
|
9
|
+
|
|
10
|
+
print(f"Policy number: {result.policy_number or '—'}")
|
|
11
|
+
if result.policy_period:
|
|
12
|
+
print(
|
|
13
|
+
f"Policy period: {result.policy_period['start']} → "
|
|
14
|
+
f"{result.policy_period['end']}"
|
|
15
|
+
)
|
|
16
|
+
if result.franchise:
|
|
17
|
+
if result.franchise.get("absent"):
|
|
18
|
+
print(f"Franchise: absent (no deductible)")
|
|
19
|
+
else:
|
|
20
|
+
print(
|
|
21
|
+
f"Franchise: {result.franchise['value']} "
|
|
22
|
+
f"{result.franchise['currency']}"
|
|
23
|
+
)
|
|
24
|
+
if result.limit:
|
|
25
|
+
print(f"Sum insured: {result.limit['value']} {result.limit['currency']}")
|
|
26
|
+
if result.premium:
|
|
27
|
+
print(
|
|
28
|
+
f"Premium: {result.premium['value']} {result.premium['currency']}"
|
|
29
|
+
)
|
|
30
|
+
print(f"Sum type: {result.sum_type or '—'}")
|
|
31
|
+
print(f"Repair mode: {result.repair_mode or '—'}")
|
|
32
|
+
print()
|
|
33
|
+
print(f"Extraction method: {result.extraction_method}")
|
|
34
|
+
print(f"Status: {result.extraction_status}")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
if __name__ == "__main__":
|
|
38
|
+
import sys
|
|
39
|
+
|
|
40
|
+
if len(sys.argv) != 2:
|
|
41
|
+
print("Usage: python examples/basic_usage.py /path/to/policy.pdf")
|
|
42
|
+
sys.exit(1)
|
|
43
|
+
main(sys.argv[1])
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""polis-recognizer — Russian insurance policy field extractor.
|
|
2
|
+
|
|
3
|
+
Extracts 7 structured fields from KASKO/insurance policy PDFs:
|
|
4
|
+
``policy_period``, ``franchise``, ``limit``, ``repair_mode``,
|
|
5
|
+
``premium``, ``sum_type``, ``policy_number``.
|
|
6
|
+
|
|
7
|
+
Quick start::
|
|
8
|
+
|
|
9
|
+
from polis_recognizer import PolicyExtractor
|
|
10
|
+
|
|
11
|
+
extractor = PolicyExtractor()
|
|
12
|
+
result = extractor.extract_from_pdf("/path/to/polis.pdf")
|
|
13
|
+
|
|
14
|
+
print(result.policy_number)
|
|
15
|
+
print(result.policy_period.start, result.policy_period.end)
|
|
16
|
+
print(result.franchise.value, result.franchise.currency)
|
|
17
|
+
|
|
18
|
+
The extractor combines a text-layer reader (pypdf) with optional
|
|
19
|
+
table-aware extraction (pdfplumber) and an OCR fallback (Tesseract).
|
|
20
|
+
See README.md for the full API and supported insurer formats.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from .contract_field_extractor import (
|
|
24
|
+
ContractFieldExtractor,
|
|
25
|
+
ContractFieldsResult,
|
|
26
|
+
FieldDiagnostic,
|
|
27
|
+
MonetaryField,
|
|
28
|
+
PolicyPeriodField,
|
|
29
|
+
TextField,
|
|
30
|
+
)
|
|
31
|
+
from .exceptions import (
|
|
32
|
+
OCRProcessingError,
|
|
33
|
+
OCRTimeoutError,
|
|
34
|
+
UnsupportedFileTypeError,
|
|
35
|
+
)
|
|
36
|
+
from .extraction import (
|
|
37
|
+
Candidate,
|
|
38
|
+
ExtractionV2Result,
|
|
39
|
+
run_extraction,
|
|
40
|
+
)
|
|
41
|
+
from .extractor import ExtractedPolicy, PolicyExtractor
|
|
42
|
+
from .hybrid_ingestion import HybridIngestionService
|
|
43
|
+
from .ocr_config import (
|
|
44
|
+
OCRConfig,
|
|
45
|
+
OCRResult,
|
|
46
|
+
get_ocr_config,
|
|
47
|
+
reset_ocr_config,
|
|
48
|
+
validate_language_pack,
|
|
49
|
+
)
|
|
50
|
+
from .ocr_service import OCRService
|
|
51
|
+
from .pdf_extraction_router import (
|
|
52
|
+
PdfExtractionOutcome,
|
|
53
|
+
PdfExtractionRouter,
|
|
54
|
+
build_text_service,
|
|
55
|
+
)
|
|
56
|
+
from .pdfplumber_ingestion import PdfPlumberIngestionService
|
|
57
|
+
from .policy_ingestion import ExtractedTextResult, PolicyIngestionService
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
__version__ = "0.1.0"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
__all__ = [
|
|
64
|
+
"__version__",
|
|
65
|
+
# Top-level facade (recommended entry point)
|
|
66
|
+
"PolicyExtractor",
|
|
67
|
+
"ExtractedPolicy",
|
|
68
|
+
# Field result dataclasses
|
|
69
|
+
"ContractFieldsResult",
|
|
70
|
+
"FieldDiagnostic",
|
|
71
|
+
"MonetaryField",
|
|
72
|
+
"PolicyPeriodField",
|
|
73
|
+
"TextField",
|
|
74
|
+
# Lower-level pipeline (for advanced use)
|
|
75
|
+
"ContractFieldExtractor",
|
|
76
|
+
"Candidate",
|
|
77
|
+
"ExtractionV2Result",
|
|
78
|
+
"run_extraction",
|
|
79
|
+
# OCR service
|
|
80
|
+
"OCRService",
|
|
81
|
+
"OCRConfig",
|
|
82
|
+
"OCRResult",
|
|
83
|
+
"get_ocr_config",
|
|
84
|
+
"reset_ocr_config",
|
|
85
|
+
"validate_language_pack",
|
|
86
|
+
# PDF ingestion
|
|
87
|
+
"PdfExtractionRouter",
|
|
88
|
+
"PdfExtractionOutcome",
|
|
89
|
+
"ExtractedTextResult",
|
|
90
|
+
"PolicyIngestionService",
|
|
91
|
+
"PdfPlumberIngestionService",
|
|
92
|
+
"HybridIngestionService",
|
|
93
|
+
"build_text_service",
|
|
94
|
+
# Exceptions
|
|
95
|
+
"OCRProcessingError",
|
|
96
|
+
"OCRTimeoutError",
|
|
97
|
+
"UnsupportedFileTypeError",
|
|
98
|
+
]
|