bankstatementparser 0.0.25__tar.gz → 0.0.27__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/PKG-INFO +5 -5
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/README.md +4 -4
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/__init__.py +15 -2
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/additional_parsers.py +16 -1
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/api.py +14 -2
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/api_worker.py +77 -0
- bankstatementparser-0.0.27/bankstatementparser/camt_reports.py +429 -0
- bankstatementparser-0.0.27/bankstatementparser/export/parquet.py +251 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/input_validator.py +46 -171
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parsers/__init__.py +4 -0
- bankstatementparser-0.0.27/bankstatementparser/parsers/bai2.py +293 -0
- bankstatementparser-0.0.27/bankstatementparser/parsers/mt942.py +275 -0
- bankstatementparser-0.0.27/bankstatementparser/xml_validator.py +280 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/pyproject.toml +1 -1
- bankstatementparser-0.0.25/bankstatementparser/export/parquet.py +0 -164
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/LICENSE +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/LICENSE-APACHE +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/LICENSE-MIT +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/_amounts.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/_llm_common.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/analytics/__init__.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/analytics/cash_flow.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/analytics/common.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/analytics/models.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/analytics/patterns.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/api_limits.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/bank_statement_parsers.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/base_parser.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/camt_parser.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/cli.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/__init__.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/account_mapper.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/categorizer.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/exceptions.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/export/__init__.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/export/ledger.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/field86_parser.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/forensics.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/__init__.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/evaluation.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/llm_extractor.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/ollama_direct.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/orchestrator.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/pdf_text.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/prompts.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/scanner.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/verification.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/vision.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/pain001_parser.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parallel.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parsers/common.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parsers/csv_parser.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parsers/mt940.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parsers/ofx.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/plugins.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/privacy.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/py.typed +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/reconciliation.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/record_types.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/telemetry.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/transaction_deduplicator.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/transaction_models.py +0 -0
- {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/zip_security.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bankstatementparser
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.27
|
|
4
4
|
Summary: Parse bank statements (CAMT.053, PAIN.001, MT940, OFX, BAI2, CSV, and PDF) into unified Transactions with treasury analytics, 2-way reconciliation, PDF forensics, and ledger export.
|
|
5
5
|
License: Apache-2.0 OR MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -64,7 +64,7 @@ Description-Content-Type: text/markdown
|
|
|
64
64
|
|
|
65
65
|
<p align="center">
|
|
66
66
|
<a href="https://github.com/sebastienrousseau/bankstatementparser/actions"><img src="https://img.shields.io/github/actions/workflow/status/sebastienrousseau/bankstatementparser/quality-gates.yml?style=for-the-badge&logo=github" alt="Build" /></a>
|
|
67
|
-
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.
|
|
67
|
+
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.27" alt="PyPI" /></a>
|
|
68
68
|
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/dm/bankstatementparser.svg?style=for-the-badge" alt="PyPI Downloads" /></a>
|
|
69
69
|
<a href="https://codecov.io/github/sebastienrousseau/bankstatementparser?branch=main"><img src="https://img.shields.io/codecov/c/github/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="Codecov" /></a>
|
|
70
70
|
<a href="LICENSE"><img src="https://img.shields.io/github/license/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="License" /></a>
|
|
@@ -390,7 +390,7 @@ for the full surface.
|
|
|
390
390
|
|---|---|
|
|
391
391
|
| **PII redaction** | Names, IBANs, and addresses masked by default — opt in with `--show-pii` |
|
|
392
392
|
| **Secure ZIP** | `iter_secure_xml_entries()` rejects zip bombs, encrypted entries, and suspicious compression ratios |
|
|
393
|
-
| **Tested** |
|
|
393
|
+
| **Tested** | 1058 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
|
|
394
394
|
|
|
395
395
|
---
|
|
396
396
|
|
|
@@ -750,7 +750,7 @@ cleanly — see each companion's README for runnable examples.
|
|
|
750
750
|
## Project Layout
|
|
751
751
|
|
|
752
752
|
```text
|
|
753
|
-
bankstatementparser/ Source code (
|
|
753
|
+
bankstatementparser/ Source code (55 modules)
|
|
754
754
|
bankstatementparser/analytics/ Financial metrics, cash flow, cadence & anomaly detection
|
|
755
755
|
bankstatementparser/parsers/ Modular statement parsers (CSV, MT940, OFX/QFX)
|
|
756
756
|
bankstatementparser/hybrid/ PDF pipeline: orchestrator, llm_extractor, vision, scanner, ollama_direct, verification
|
|
@@ -760,7 +760,7 @@ bankstatementparser/api.py REST API microservice (FastAPI)
|
|
|
760
760
|
docs/compliance/ ISO 13485 validation, risk register, traceability matrix
|
|
761
761
|
examples/ 14 deterministic + 9 hybrid runnable example scripts
|
|
762
762
|
scripts/ SBOM generation, checksums, signature verification
|
|
763
|
-
tests/
|
|
763
|
+
tests/ 1058 tests (unit, integration, property-based, security, hybrid mocks)
|
|
764
764
|
```
|
|
765
765
|
|
|
766
766
|
---
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
|
|
20
20
|
<p align="center">
|
|
21
21
|
<a href="https://github.com/sebastienrousseau/bankstatementparser/actions"><img src="https://img.shields.io/github/actions/workflow/status/sebastienrousseau/bankstatementparser/quality-gates.yml?style=for-the-badge&logo=github" alt="Build" /></a>
|
|
22
|
-
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.
|
|
22
|
+
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.27" alt="PyPI" /></a>
|
|
23
23
|
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/dm/bankstatementparser.svg?style=for-the-badge" alt="PyPI Downloads" /></a>
|
|
24
24
|
<a href="https://codecov.io/github/sebastienrousseau/bankstatementparser?branch=main"><img src="https://img.shields.io/codecov/c/github/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="Codecov" /></a>
|
|
25
25
|
<a href="LICENSE"><img src="https://img.shields.io/github/license/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="License" /></a>
|
|
@@ -345,7 +345,7 @@ for the full surface.
|
|
|
345
345
|
|---|---|
|
|
346
346
|
| **PII redaction** | Names, IBANs, and addresses masked by default — opt in with `--show-pii` |
|
|
347
347
|
| **Secure ZIP** | `iter_secure_xml_entries()` rejects zip bombs, encrypted entries, and suspicious compression ratios |
|
|
348
|
-
| **Tested** |
|
|
348
|
+
| **Tested** | 1058 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
|
|
349
349
|
|
|
350
350
|
---
|
|
351
351
|
|
|
@@ -705,7 +705,7 @@ cleanly — see each companion's README for runnable examples.
|
|
|
705
705
|
## Project Layout
|
|
706
706
|
|
|
707
707
|
```text
|
|
708
|
-
bankstatementparser/ Source code (
|
|
708
|
+
bankstatementparser/ Source code (55 modules)
|
|
709
709
|
bankstatementparser/analytics/ Financial metrics, cash flow, cadence & anomaly detection
|
|
710
710
|
bankstatementparser/parsers/ Modular statement parsers (CSV, MT940, OFX/QFX)
|
|
711
711
|
bankstatementparser/hybrid/ PDF pipeline: orchestrator, llm_extractor, vision, scanner, ollama_direct, verification
|
|
@@ -715,7 +715,7 @@ bankstatementparser/api.py REST API microservice (FastAPI)
|
|
|
715
715
|
docs/compliance/ ISO 13485 validation, risk register, traceability matrix
|
|
716
716
|
examples/ 14 deterministic + 9 hybrid runnable example scripts
|
|
717
717
|
scripts/ SBOM generation, checksums, signature verification
|
|
718
|
-
tests/
|
|
718
|
+
tests/ 1058 tests (unit, integration, property-based, security, hybrid mocks)
|
|
719
719
|
```
|
|
720
720
|
|
|
721
721
|
---
|
|
@@ -20,8 +20,12 @@ operations.
|
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
22
|
from .additional_parsers import (
|
|
23
|
+
Bai2Parser,
|
|
24
|
+
Camt052Parser,
|
|
25
|
+
Camt054Parser,
|
|
23
26
|
CsvStatementParser,
|
|
24
27
|
Mt940Parser,
|
|
28
|
+
Mt942Parser,
|
|
25
29
|
OfxParser,
|
|
26
30
|
QfxParser,
|
|
27
31
|
create_parser,
|
|
@@ -48,7 +52,11 @@ from .exceptions import (
|
|
|
48
52
|
Pain001ParseError,
|
|
49
53
|
ParserError,
|
|
50
54
|
)
|
|
51
|
-
from .export.parquet import
|
|
55
|
+
from .export.parquet import (
|
|
56
|
+
ParquetStreamWriter,
|
|
57
|
+
export_parquet,
|
|
58
|
+
export_parquet_stream,
|
|
59
|
+
)
|
|
52
60
|
from .field86_parser import Field86Structure, parse_field_86
|
|
53
61
|
from .forensics import (
|
|
54
62
|
ForensicFinding,
|
|
@@ -99,14 +107,17 @@ from .zip_security import (
|
|
|
99
107
|
#: so the conformance gate can compare it statically against
|
|
100
108
|
#: pyproject.toml -- that comparison is the thing that catches drift, and
|
|
101
109
|
#: a runtime lookup would make the two trivially equal and check nothing.
|
|
102
|
-
__version__ = "0.0.
|
|
110
|
+
__version__ = "0.0.27"
|
|
103
111
|
|
|
104
112
|
__all__ = [
|
|
105
113
|
"AnalyticsReport",
|
|
106
114
|
"AnomalyFinding",
|
|
115
|
+
"Bai2Parser",
|
|
107
116
|
"BankStatementParser",
|
|
108
117
|
"BankStatementParserError",
|
|
109
118
|
"BoundingBox",
|
|
119
|
+
"Camt052Parser",
|
|
120
|
+
"Camt054Parser",
|
|
110
121
|
"CamtParser",
|
|
111
122
|
"CashFlowMetrics",
|
|
112
123
|
"CsvStatementParser",
|
|
@@ -123,10 +134,12 @@ __all__ = [
|
|
|
123
134
|
"InputValidator",
|
|
124
135
|
"MatchGroup",
|
|
125
136
|
"Mt940Parser",
|
|
137
|
+
"Mt942Parser",
|
|
126
138
|
"NullSpan",
|
|
127
139
|
"OfxParser",
|
|
128
140
|
"Pain001ParseError",
|
|
129
141
|
"Pain001Parser",
|
|
142
|
+
"ParquetStreamWriter",
|
|
130
143
|
"ParserError",
|
|
131
144
|
"QfxParser",
|
|
132
145
|
"ReconciliationMatch",
|
{bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/additional_parsers.py
RENAMED
|
@@ -21,12 +21,15 @@ from pathlib import Path
|
|
|
21
21
|
|
|
22
22
|
from .base_parser import BankStatementParser
|
|
23
23
|
from .camt_parser import CamtParser
|
|
24
|
+
from .camt_reports import Camt052Parser, Camt054Parser
|
|
24
25
|
from .input_validator import ValidationError
|
|
25
26
|
from .pain001_parser import Pain001Parser
|
|
26
27
|
from .parsers import (
|
|
27
28
|
CSV_COLUMN_GROUPS,
|
|
29
|
+
Bai2Parser,
|
|
28
30
|
CsvStatementParser,
|
|
29
31
|
Mt940Parser,
|
|
32
|
+
Mt942Parser,
|
|
30
33
|
OfxParser,
|
|
31
34
|
QfxParser,
|
|
32
35
|
_amount_or_zero,
|
|
@@ -42,8 +45,12 @@ from .plugins import get_registered_loaders
|
|
|
42
45
|
|
|
43
46
|
__all__ = [
|
|
44
47
|
"CSV_COLUMN_GROUPS",
|
|
48
|
+
"Bai2Parser",
|
|
49
|
+
"Camt052Parser",
|
|
50
|
+
"Camt054Parser",
|
|
45
51
|
"CsvStatementParser",
|
|
46
52
|
"Mt940Parser",
|
|
53
|
+
"Mt942Parser",
|
|
47
54
|
"OfxParser",
|
|
48
55
|
"QfxParser",
|
|
49
56
|
"_amount_or_zero",
|
|
@@ -67,7 +74,7 @@ def _detect_by_suffix(suffix: str) -> str | None:
|
|
|
67
74
|
return "ofx"
|
|
68
75
|
if suffix in {".mt940", ".sta"}:
|
|
69
76
|
return "mt940"
|
|
70
|
-
if suffix
|
|
77
|
+
if suffix in {".bai2", ".bai"}:
|
|
71
78
|
return "bai2"
|
|
72
79
|
if suffix == ".mt942":
|
|
73
80
|
return "mt942"
|
|
@@ -78,6 +85,10 @@ def _detect_xml_format(lowered: str) -> str | None:
|
|
|
78
85
|
"""Detect statement format from XML content tags."""
|
|
79
86
|
if "cstmrcdttrfinitn" in lowered or "pain.001" in lowered:
|
|
80
87
|
return "pain001"
|
|
88
|
+
if "bktocstmrdbtcdtntfctn" in lowered or "camt.054" in lowered:
|
|
89
|
+
return "camt054"
|
|
90
|
+
if "bktocstmracctrpt" in lowered or "camt.052" in lowered:
|
|
91
|
+
return "camt052"
|
|
81
92
|
if "bktocstmrstmt" in lowered or "camt." in lowered:
|
|
82
93
|
return "camt"
|
|
83
94
|
return None
|
|
@@ -135,11 +146,15 @@ def create_parser(
|
|
|
135
146
|
selected = (format_name or detect_statement_format(file_name)).lower()
|
|
136
147
|
parser_map: dict[str, type[BankStatementParser]] = {
|
|
137
148
|
"camt": CamtParser,
|
|
149
|
+
"camt052": Camt052Parser,
|
|
150
|
+
"camt054": Camt054Parser,
|
|
138
151
|
"pain001": Pain001Parser,
|
|
139
152
|
"csv": CsvStatementParser,
|
|
140
153
|
"ofx": OfxParser,
|
|
141
154
|
"qfx": QfxParser,
|
|
142
155
|
"mt940": Mt940Parser,
|
|
156
|
+
"mt942": Mt942Parser,
|
|
157
|
+
"bai2": Bai2Parser,
|
|
143
158
|
}
|
|
144
159
|
# Augment with dynamic plugins
|
|
145
160
|
parser_map.update(get_registered_loaders())
|
|
@@ -53,11 +53,13 @@ from .api_limits import (
|
|
|
53
53
|
_safe_basename,
|
|
54
54
|
)
|
|
55
55
|
from .api_worker import (
|
|
56
|
+
PersistentWorkerPool,
|
|
56
57
|
_ingest_in_process,
|
|
57
58
|
_ingest_in_thread,
|
|
58
59
|
_reap_worker,
|
|
59
60
|
_result_to_dict,
|
|
60
61
|
_run_ingest_worker,
|
|
62
|
+
_run_worker_job,
|
|
61
63
|
_verification_dict,
|
|
62
64
|
)
|
|
63
65
|
from .input_validator import InputValidator
|
|
@@ -70,6 +72,7 @@ __all__ = [
|
|
|
70
72
|
"ENV_MAX_UPLOAD_BYTES",
|
|
71
73
|
"_UPLOAD_CHUNK_BYTES",
|
|
72
74
|
"APIError",
|
|
75
|
+
"PersistentWorkerPool",
|
|
73
76
|
"_IngestLimits",
|
|
74
77
|
"_allowed_suffix",
|
|
75
78
|
"_ingest_in_process",
|
|
@@ -78,6 +81,7 @@ __all__ = [
|
|
|
78
81
|
"_resolve_max_upload_bytes",
|
|
79
82
|
"_result_to_dict",
|
|
80
83
|
"_run_ingest_worker",
|
|
84
|
+
"_run_worker_job",
|
|
81
85
|
"_safe_basename",
|
|
82
86
|
"_verification_dict",
|
|
83
87
|
"create_app",
|
|
@@ -101,6 +105,7 @@ def _build_ingest_handler(
|
|
|
101
105
|
app: Any,
|
|
102
106
|
max_upload: int,
|
|
103
107
|
ingest_timeout: float | None,
|
|
108
|
+
worker_pool: Optional[PersistentWorkerPool] = None,
|
|
104
109
|
) -> Any:
|
|
105
110
|
"""Register the POST /ingest endpoint on the FastAPI application."""
|
|
106
111
|
from fastapi import File, UploadFile
|
|
@@ -151,7 +156,11 @@ def _build_ingest_handler(
|
|
|
151
156
|
},
|
|
152
157
|
status_code=413,
|
|
153
158
|
)
|
|
154
|
-
if ingest_timeout is None:
|
|
159
|
+
if worker_pool is not None and ingest_timeout is not None:
|
|
160
|
+
payload = await worker_pool.run_ingest(
|
|
161
|
+
tmp_path, ingest_timeout
|
|
162
|
+
)
|
|
163
|
+
elif ingest_timeout is None:
|
|
155
164
|
result = await _ingest_in_thread(smart_ingest, tmp_path)
|
|
156
165
|
payload = _result_to_dict(result)
|
|
157
166
|
else:
|
|
@@ -187,6 +196,7 @@ def create_app(
|
|
|
187
196
|
max_concurrent_ingests: int = 4,
|
|
188
197
|
upload_timeout: float = 60.0,
|
|
189
198
|
ingest_timeout: float | None = 120.0,
|
|
199
|
+
worker_pool: Optional[PersistentWorkerPool] = None,
|
|
190
200
|
) -> Any:
|
|
191
201
|
"""Create a FastAPI application wrapping :func:`smart_ingest`."""
|
|
192
202
|
try:
|
|
@@ -226,7 +236,9 @@ def create_app(
|
|
|
226
236
|
upload_timeout=upload_timeout,
|
|
227
237
|
)
|
|
228
238
|
|
|
229
|
-
_build_ingest_handler(
|
|
239
|
+
_build_ingest_handler(
|
|
240
|
+
app, upload_cap, ingest_timeout, worker_pool=worker_pool
|
|
241
|
+
)
|
|
230
242
|
|
|
231
243
|
@app.get("/health") # type: ignore[untyped-decorator]
|
|
232
244
|
async def health() -> dict[str, str]:
|
|
@@ -21,12 +21,24 @@ import asyncio
|
|
|
21
21
|
import json
|
|
22
22
|
import sys
|
|
23
23
|
import tempfile
|
|
24
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
24
25
|
from contextlib import suppress
|
|
25
26
|
from pathlib import Path
|
|
26
27
|
from typing import Any, Optional, cast
|
|
27
28
|
|
|
28
29
|
from .api_limits import APIError
|
|
29
30
|
|
|
31
|
+
__all__ = [
|
|
32
|
+
"PersistentWorkerPool",
|
|
33
|
+
"_ingest_in_process",
|
|
34
|
+
"_ingest_in_thread",
|
|
35
|
+
"_reap_worker",
|
|
36
|
+
"_result_to_dict",
|
|
37
|
+
"_run_ingest_worker",
|
|
38
|
+
"_run_worker_job",
|
|
39
|
+
"_verification_dict",
|
|
40
|
+
]
|
|
41
|
+
|
|
30
42
|
|
|
31
43
|
def _verification_dict(v: Any) -> Optional[dict[str, Any]]:
|
|
32
44
|
"""Serialize a verification result to a JSON-safe dict, or None."""
|
|
@@ -76,6 +88,17 @@ def _run_ingest_worker(input_path: str, output_path: str) -> None:
|
|
|
76
88
|
json.dump(payload, output)
|
|
77
89
|
|
|
78
90
|
|
|
91
|
+
def _run_worker_job(input_path: str) -> dict[str, Any]:
|
|
92
|
+
"""Execute ingestion in a warm process worker and serialize to dict."""
|
|
93
|
+
from .hybrid import smart_ingest
|
|
94
|
+
from .telemetry import trace_span
|
|
95
|
+
|
|
96
|
+
with trace_span(
|
|
97
|
+
"bankstatementparser.api.worker_pool", {"input_path": input_path}
|
|
98
|
+
):
|
|
99
|
+
return _result_to_dict(smart_ingest(input_path))
|
|
100
|
+
|
|
101
|
+
|
|
79
102
|
async def _reap_worker(process: asyncio.subprocess.Process) -> None:
|
|
80
103
|
"""Wait for process exit even if cleanup receives repeated cancellation."""
|
|
81
104
|
waiter = asyncio.create_task(process.wait())
|
|
@@ -148,3 +171,57 @@ async def _ingest_in_process(path: str, timeout: float) -> dict[str, Any]:
|
|
|
148
171
|
with suppress(ProcessLookupError):
|
|
149
172
|
process.kill()
|
|
150
173
|
await _reap_worker(process)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
class PersistentWorkerPool:
|
|
177
|
+
"""Persistent warm process pool with memory recycling and execution deadlines."""
|
|
178
|
+
|
|
179
|
+
def __init__(
|
|
180
|
+
self,
|
|
181
|
+
max_workers: int = 4,
|
|
182
|
+
max_tasks_per_child: int = 50,
|
|
183
|
+
) -> None:
|
|
184
|
+
"""Initialize worker pool with concurrency and memory recycling bounds."""
|
|
185
|
+
self._max_workers = max_workers
|
|
186
|
+
self._max_tasks_per_child = max_tasks_per_child
|
|
187
|
+
self._executor: ProcessPoolExecutor | None = None
|
|
188
|
+
|
|
189
|
+
@property
|
|
190
|
+
def is_running(self) -> bool:
|
|
191
|
+
"""Check whether the process pool is currently initialized and active."""
|
|
192
|
+
return self._executor is not None
|
|
193
|
+
|
|
194
|
+
def start(self) -> None:
|
|
195
|
+
"""Initialize the persistent worker pool if not already running."""
|
|
196
|
+
if self._executor is None:
|
|
197
|
+
kwargs: dict[str, Any] = {}
|
|
198
|
+
if (
|
|
199
|
+
sys.version_info >= (3, 11) and self._max_tasks_per_child
|
|
200
|
+
): # pragma: no cover - version dependent
|
|
201
|
+
kwargs["max_tasks_per_child"] = self._max_tasks_per_child
|
|
202
|
+
self._executor = ProcessPoolExecutor(
|
|
203
|
+
max_workers=self._max_workers,
|
|
204
|
+
**kwargs,
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
def shutdown(self, wait: bool = True) -> None:
|
|
208
|
+
"""Shut down the pool and release worker processes."""
|
|
209
|
+
if self._executor is not None:
|
|
210
|
+
self._executor.shutdown(wait=wait, cancel_futures=True)
|
|
211
|
+
self._executor = None
|
|
212
|
+
|
|
213
|
+
async def run_ingest(self, path: str, timeout: float) -> dict[str, Any]:
|
|
214
|
+
"""Execute statement ingestion in a warm worker under an execution deadline."""
|
|
215
|
+
if self._executor is None:
|
|
216
|
+
self.start()
|
|
217
|
+
executor = self._executor
|
|
218
|
+
if executor is None:
|
|
219
|
+
raise APIError("Worker pool failed to initialize")
|
|
220
|
+
loop = asyncio.get_running_loop()
|
|
221
|
+
future = loop.run_in_executor(executor, _run_worker_job, path)
|
|
222
|
+
try:
|
|
223
|
+
return await asyncio.wait_for(future, timeout=timeout)
|
|
224
|
+
except asyncio.TimeoutError:
|
|
225
|
+
raise
|
|
226
|
+
except Exception as exc:
|
|
227
|
+
raise APIError(f"Worker ingestion failed: {exc}") from exc
|