bankstatementparser 0.0.25__tar.gz → 0.0.27__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/PKG-INFO +5 -5
  2. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/README.md +4 -4
  3. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/__init__.py +15 -2
  4. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/additional_parsers.py +16 -1
  5. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/api.py +14 -2
  6. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/api_worker.py +77 -0
  7. bankstatementparser-0.0.27/bankstatementparser/camt_reports.py +429 -0
  8. bankstatementparser-0.0.27/bankstatementparser/export/parquet.py +251 -0
  9. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/input_validator.py +46 -171
  10. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parsers/__init__.py +4 -0
  11. bankstatementparser-0.0.27/bankstatementparser/parsers/bai2.py +293 -0
  12. bankstatementparser-0.0.27/bankstatementparser/parsers/mt942.py +275 -0
  13. bankstatementparser-0.0.27/bankstatementparser/xml_validator.py +280 -0
  14. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/pyproject.toml +1 -1
  15. bankstatementparser-0.0.25/bankstatementparser/export/parquet.py +0 -164
  16. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/LICENSE +0 -0
  17. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/LICENSE-APACHE +0 -0
  18. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/LICENSE-MIT +0 -0
  19. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/_amounts.py +0 -0
  20. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/_llm_common.py +0 -0
  21. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/analytics/__init__.py +0 -0
  22. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/analytics/cash_flow.py +0 -0
  23. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/analytics/common.py +0 -0
  24. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/analytics/models.py +0 -0
  25. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/analytics/patterns.py +0 -0
  26. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/api_limits.py +0 -0
  27. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/bank_statement_parsers.py +0 -0
  28. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/base_parser.py +0 -0
  29. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/camt_parser.py +0 -0
  30. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/cli.py +0 -0
  31. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/__init__.py +0 -0
  32. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/account_mapper.py +0 -0
  33. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/categorizer.py +0 -0
  34. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/exceptions.py +0 -0
  35. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/export/__init__.py +0 -0
  36. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/export/ledger.py +0 -0
  37. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/field86_parser.py +0 -0
  38. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/forensics.py +0 -0
  39. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/__init__.py +0 -0
  40. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/evaluation.py +0 -0
  41. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/llm_extractor.py +0 -0
  42. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/ollama_direct.py +0 -0
  43. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/orchestrator.py +0 -0
  44. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/pdf_text.py +0 -0
  45. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/prompts.py +0 -0
  46. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/scanner.py +0 -0
  47. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/verification.py +0 -0
  48. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/vision.py +0 -0
  49. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/pain001_parser.py +0 -0
  50. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parallel.py +0 -0
  51. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parsers/common.py +0 -0
  52. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parsers/csv_parser.py +0 -0
  53. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parsers/mt940.py +0 -0
  54. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/parsers/ofx.py +0 -0
  55. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/plugins.py +0 -0
  56. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/privacy.py +0 -0
  57. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/py.typed +0 -0
  58. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/reconciliation.py +0 -0
  59. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/record_types.py +0 -0
  60. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/telemetry.py +0 -0
  61. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/transaction_deduplicator.py +0 -0
  62. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/transaction_models.py +0 -0
  63. {bankstatementparser-0.0.25 → bankstatementparser-0.0.27}/bankstatementparser/zip_security.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: bankstatementparser
3
- Version: 0.0.25
3
+ Version: 0.0.27
4
4
  Summary: Parse bank statements (CAMT.053, PAIN.001, MT940, OFX, BAI2, CSV, and PDF) into unified Transactions with treasury analytics, 2-way reconciliation, PDF forensics, and ledger export.
5
5
  License: Apache-2.0 OR MIT
6
6
  License-File: LICENSE
@@ -64,7 +64,7 @@ Description-Content-Type: text/markdown
64
64
 
65
65
  <p align="center">
66
66
  <a href="https://github.com/sebastienrousseau/bankstatementparser/actions"><img src="https://img.shields.io/github/actions/workflow/status/sebastienrousseau/bankstatementparser/quality-gates.yml?style=for-the-badge&logo=github" alt="Build" /></a>
67
- <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.25" alt="PyPI" /></a>
67
+ <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.27" alt="PyPI" /></a>
68
68
  <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/dm/bankstatementparser.svg?style=for-the-badge" alt="PyPI Downloads" /></a>
69
69
  <a href="https://codecov.io/github/sebastienrousseau/bankstatementparser?branch=main"><img src="https://img.shields.io/codecov/c/github/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="Codecov" /></a>
70
70
  <a href="LICENSE"><img src="https://img.shields.io/github/license/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="License" /></a>
@@ -390,7 +390,7 @@ for the full surface.
390
390
  |---|---|
391
391
  | **PII redaction** | Names, IBANs, and addresses masked by default — opt in with `--show-pii` |
392
392
  | **Secure ZIP** | `iter_secure_xml_entries()` rejects zip bombs, encrypted entries, and suspicious compression ratios |
393
- | **Tested** | 1023 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
393
+ | **Tested** | 1058 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
394
394
 
395
395
  ---
396
396
 
@@ -750,7 +750,7 @@ cleanly — see each companion's README for runnable examples.
750
750
  ## Project Layout
751
751
 
752
752
  ```text
753
- bankstatementparser/ Source code (51 modules)
753
+ bankstatementparser/ Source code (55 modules)
754
754
  bankstatementparser/analytics/ Financial metrics, cash flow, cadence & anomaly detection
755
755
  bankstatementparser/parsers/ Modular statement parsers (CSV, MT940, OFX/QFX)
756
756
  bankstatementparser/hybrid/ PDF pipeline: orchestrator, llm_extractor, vision, scanner, ollama_direct, verification
@@ -760,7 +760,7 @@ bankstatementparser/api.py REST API microservice (FastAPI)
760
760
  docs/compliance/ ISO 13485 validation, risk register, traceability matrix
761
761
  examples/ 14 deterministic + 9 hybrid runnable example scripts
762
762
  scripts/ SBOM generation, checksums, signature verification
763
- tests/ 1023 tests (unit, integration, property-based, security, hybrid mocks)
763
+ tests/ 1058 tests (unit, integration, property-based, security, hybrid mocks)
764
764
  ```
765
765
 
766
766
  ---
@@ -19,7 +19,7 @@
19
19
 
20
20
  <p align="center">
21
21
  <a href="https://github.com/sebastienrousseau/bankstatementparser/actions"><img src="https://img.shields.io/github/actions/workflow/status/sebastienrousseau/bankstatementparser/quality-gates.yml?style=for-the-badge&logo=github" alt="Build" /></a>
22
- <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.25" alt="PyPI" /></a>
22
+ <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.27" alt="PyPI" /></a>
23
23
  <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/dm/bankstatementparser.svg?style=for-the-badge" alt="PyPI Downloads" /></a>
24
24
  <a href="https://codecov.io/github/sebastienrousseau/bankstatementparser?branch=main"><img src="https://img.shields.io/codecov/c/github/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="Codecov" /></a>
25
25
  <a href="LICENSE"><img src="https://img.shields.io/github/license/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="License" /></a>
@@ -345,7 +345,7 @@ for the full surface.
345
345
  |---|---|
346
346
  | **PII redaction** | Names, IBANs, and addresses masked by default — opt in with `--show-pii` |
347
347
  | **Secure ZIP** | `iter_secure_xml_entries()` rejects zip bombs, encrypted entries, and suspicious compression ratios |
348
- | **Tested** | 1023 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
348
+ | **Tested** | 1058 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
349
349
 
350
350
  ---
351
351
 
@@ -705,7 +705,7 @@ cleanly — see each companion's README for runnable examples.
705
705
  ## Project Layout
706
706
 
707
707
  ```text
708
- bankstatementparser/ Source code (51 modules)
708
+ bankstatementparser/ Source code (55 modules)
709
709
  bankstatementparser/analytics/ Financial metrics, cash flow, cadence & anomaly detection
710
710
  bankstatementparser/parsers/ Modular statement parsers (CSV, MT940, OFX/QFX)
711
711
  bankstatementparser/hybrid/ PDF pipeline: orchestrator, llm_extractor, vision, scanner, ollama_direct, verification
@@ -715,7 +715,7 @@ bankstatementparser/api.py REST API microservice (FastAPI)
715
715
  docs/compliance/ ISO 13485 validation, risk register, traceability matrix
716
716
  examples/ 14 deterministic + 9 hybrid runnable example scripts
717
717
  scripts/ SBOM generation, checksums, signature verification
718
- tests/ 1023 tests (unit, integration, property-based, security, hybrid mocks)
718
+ tests/ 1058 tests (unit, integration, property-based, security, hybrid mocks)
719
719
  ```
720
720
 
721
721
  ---
@@ -20,8 +20,12 @@ operations.
20
20
  """
21
21
 
22
22
  from .additional_parsers import (
23
+ Bai2Parser,
24
+ Camt052Parser,
25
+ Camt054Parser,
23
26
  CsvStatementParser,
24
27
  Mt940Parser,
28
+ Mt942Parser,
25
29
  OfxParser,
26
30
  QfxParser,
27
31
  create_parser,
@@ -48,7 +52,11 @@ from .exceptions import (
48
52
  Pain001ParseError,
49
53
  ParserError,
50
54
  )
51
- from .export.parquet import export_parquet, export_parquet_stream
55
+ from .export.parquet import (
56
+ ParquetStreamWriter,
57
+ export_parquet,
58
+ export_parquet_stream,
59
+ )
52
60
  from .field86_parser import Field86Structure, parse_field_86
53
61
  from .forensics import (
54
62
  ForensicFinding,
@@ -99,14 +107,17 @@ from .zip_security import (
99
107
  #: so the conformance gate can compare it statically against
100
108
  #: pyproject.toml -- that comparison is the thing that catches drift, and
101
109
  #: a runtime lookup would make the two trivially equal and check nothing.
102
- __version__ = "0.0.25"
110
+ __version__ = "0.0.27"
103
111
 
104
112
  __all__ = [
105
113
  "AnalyticsReport",
106
114
  "AnomalyFinding",
115
+ "Bai2Parser",
107
116
  "BankStatementParser",
108
117
  "BankStatementParserError",
109
118
  "BoundingBox",
119
+ "Camt052Parser",
120
+ "Camt054Parser",
110
121
  "CamtParser",
111
122
  "CashFlowMetrics",
112
123
  "CsvStatementParser",
@@ -123,10 +134,12 @@ __all__ = [
123
134
  "InputValidator",
124
135
  "MatchGroup",
125
136
  "Mt940Parser",
137
+ "Mt942Parser",
126
138
  "NullSpan",
127
139
  "OfxParser",
128
140
  "Pain001ParseError",
129
141
  "Pain001Parser",
142
+ "ParquetStreamWriter",
130
143
  "ParserError",
131
144
  "QfxParser",
132
145
  "ReconciliationMatch",
@@ -21,12 +21,15 @@ from pathlib import Path
21
21
 
22
22
  from .base_parser import BankStatementParser
23
23
  from .camt_parser import CamtParser
24
+ from .camt_reports import Camt052Parser, Camt054Parser
24
25
  from .input_validator import ValidationError
25
26
  from .pain001_parser import Pain001Parser
26
27
  from .parsers import (
27
28
  CSV_COLUMN_GROUPS,
29
+ Bai2Parser,
28
30
  CsvStatementParser,
29
31
  Mt940Parser,
32
+ Mt942Parser,
30
33
  OfxParser,
31
34
  QfxParser,
32
35
  _amount_or_zero,
@@ -42,8 +45,12 @@ from .plugins import get_registered_loaders
42
45
 
43
46
  __all__ = [
44
47
  "CSV_COLUMN_GROUPS",
48
+ "Bai2Parser",
49
+ "Camt052Parser",
50
+ "Camt054Parser",
45
51
  "CsvStatementParser",
46
52
  "Mt940Parser",
53
+ "Mt942Parser",
47
54
  "OfxParser",
48
55
  "QfxParser",
49
56
  "_amount_or_zero",
@@ -67,7 +74,7 @@ def _detect_by_suffix(suffix: str) -> str | None:
67
74
  return "ofx"
68
75
  if suffix in {".mt940", ".sta"}:
69
76
  return "mt940"
70
- if suffix == ".bai2":
77
+ if suffix in {".bai2", ".bai"}:
71
78
  return "bai2"
72
79
  if suffix == ".mt942":
73
80
  return "mt942"
@@ -78,6 +85,10 @@ def _detect_xml_format(lowered: str) -> str | None:
78
85
  """Detect statement format from XML content tags."""
79
86
  if "cstmrcdttrfinitn" in lowered or "pain.001" in lowered:
80
87
  return "pain001"
88
+ if "bktocstmrdbtcdtntfctn" in lowered or "camt.054" in lowered:
89
+ return "camt054"
90
+ if "bktocstmracctrpt" in lowered or "camt.052" in lowered:
91
+ return "camt052"
81
92
  if "bktocstmrstmt" in lowered or "camt." in lowered:
82
93
  return "camt"
83
94
  return None
@@ -135,11 +146,15 @@ def create_parser(
135
146
  selected = (format_name or detect_statement_format(file_name)).lower()
136
147
  parser_map: dict[str, type[BankStatementParser]] = {
137
148
  "camt": CamtParser,
149
+ "camt052": Camt052Parser,
150
+ "camt054": Camt054Parser,
138
151
  "pain001": Pain001Parser,
139
152
  "csv": CsvStatementParser,
140
153
  "ofx": OfxParser,
141
154
  "qfx": QfxParser,
142
155
  "mt940": Mt940Parser,
156
+ "mt942": Mt942Parser,
157
+ "bai2": Bai2Parser,
143
158
  }
144
159
  # Augment with dynamic plugins
145
160
  parser_map.update(get_registered_loaders())
@@ -53,11 +53,13 @@ from .api_limits import (
53
53
  _safe_basename,
54
54
  )
55
55
  from .api_worker import (
56
+ PersistentWorkerPool,
56
57
  _ingest_in_process,
57
58
  _ingest_in_thread,
58
59
  _reap_worker,
59
60
  _result_to_dict,
60
61
  _run_ingest_worker,
62
+ _run_worker_job,
61
63
  _verification_dict,
62
64
  )
63
65
  from .input_validator import InputValidator
@@ -70,6 +72,7 @@ __all__ = [
70
72
  "ENV_MAX_UPLOAD_BYTES",
71
73
  "_UPLOAD_CHUNK_BYTES",
72
74
  "APIError",
75
+ "PersistentWorkerPool",
73
76
  "_IngestLimits",
74
77
  "_allowed_suffix",
75
78
  "_ingest_in_process",
@@ -78,6 +81,7 @@ __all__ = [
78
81
  "_resolve_max_upload_bytes",
79
82
  "_result_to_dict",
80
83
  "_run_ingest_worker",
84
+ "_run_worker_job",
81
85
  "_safe_basename",
82
86
  "_verification_dict",
83
87
  "create_app",
@@ -101,6 +105,7 @@ def _build_ingest_handler(
101
105
  app: Any,
102
106
  max_upload: int,
103
107
  ingest_timeout: float | None,
108
+ worker_pool: Optional[PersistentWorkerPool] = None,
104
109
  ) -> Any:
105
110
  """Register the POST /ingest endpoint on the FastAPI application."""
106
111
  from fastapi import File, UploadFile
@@ -151,7 +156,11 @@ def _build_ingest_handler(
151
156
  },
152
157
  status_code=413,
153
158
  )
154
- if ingest_timeout is None:
159
+ if worker_pool is not None and ingest_timeout is not None:
160
+ payload = await worker_pool.run_ingest(
161
+ tmp_path, ingest_timeout
162
+ )
163
+ elif ingest_timeout is None:
155
164
  result = await _ingest_in_thread(smart_ingest, tmp_path)
156
165
  payload = _result_to_dict(result)
157
166
  else:
@@ -187,6 +196,7 @@ def create_app(
187
196
  max_concurrent_ingests: int = 4,
188
197
  upload_timeout: float = 60.0,
189
198
  ingest_timeout: float | None = 120.0,
199
+ worker_pool: Optional[PersistentWorkerPool] = None,
190
200
  ) -> Any:
191
201
  """Create a FastAPI application wrapping :func:`smart_ingest`."""
192
202
  try:
@@ -226,7 +236,9 @@ def create_app(
226
236
  upload_timeout=upload_timeout,
227
237
  )
228
238
 
229
- _build_ingest_handler(app, upload_cap, ingest_timeout)
239
+ _build_ingest_handler(
240
+ app, upload_cap, ingest_timeout, worker_pool=worker_pool
241
+ )
230
242
 
231
243
  @app.get("/health") # type: ignore[untyped-decorator]
232
244
  async def health() -> dict[str, str]:
@@ -21,12 +21,24 @@ import asyncio
21
21
  import json
22
22
  import sys
23
23
  import tempfile
24
+ from concurrent.futures import ProcessPoolExecutor
24
25
  from contextlib import suppress
25
26
  from pathlib import Path
26
27
  from typing import Any, Optional, cast
27
28
 
28
29
  from .api_limits import APIError
29
30
 
31
+ __all__ = [
32
+ "PersistentWorkerPool",
33
+ "_ingest_in_process",
34
+ "_ingest_in_thread",
35
+ "_reap_worker",
36
+ "_result_to_dict",
37
+ "_run_ingest_worker",
38
+ "_run_worker_job",
39
+ "_verification_dict",
40
+ ]
41
+
30
42
 
31
43
  def _verification_dict(v: Any) -> Optional[dict[str, Any]]:
32
44
  """Serialize a verification result to a JSON-safe dict, or None."""
@@ -76,6 +88,17 @@ def _run_ingest_worker(input_path: str, output_path: str) -> None:
76
88
  json.dump(payload, output)
77
89
 
78
90
 
91
+ def _run_worker_job(input_path: str) -> dict[str, Any]:
92
+ """Execute ingestion in a warm process worker and serialize to dict."""
93
+ from .hybrid import smart_ingest
94
+ from .telemetry import trace_span
95
+
96
+ with trace_span(
97
+ "bankstatementparser.api.worker_pool", {"input_path": input_path}
98
+ ):
99
+ return _result_to_dict(smart_ingest(input_path))
100
+
101
+
79
102
  async def _reap_worker(process: asyncio.subprocess.Process) -> None:
80
103
  """Wait for process exit even if cleanup receives repeated cancellation."""
81
104
  waiter = asyncio.create_task(process.wait())
@@ -148,3 +171,57 @@ async def _ingest_in_process(path: str, timeout: float) -> dict[str, Any]:
148
171
  with suppress(ProcessLookupError):
149
172
  process.kill()
150
173
  await _reap_worker(process)
174
+
175
+
176
+ class PersistentWorkerPool:
177
+ """Persistent warm process pool with memory recycling and execution deadlines."""
178
+
179
+ def __init__(
180
+ self,
181
+ max_workers: int = 4,
182
+ max_tasks_per_child: int = 50,
183
+ ) -> None:
184
+ """Initialize worker pool with concurrency and memory recycling bounds."""
185
+ self._max_workers = max_workers
186
+ self._max_tasks_per_child = max_tasks_per_child
187
+ self._executor: ProcessPoolExecutor | None = None
188
+
189
+ @property
190
+ def is_running(self) -> bool:
191
+ """Check whether the process pool is currently initialized and active."""
192
+ return self._executor is not None
193
+
194
+ def start(self) -> None:
195
+ """Initialize the persistent worker pool if not already running."""
196
+ if self._executor is None:
197
+ kwargs: dict[str, Any] = {}
198
+ if (
199
+ sys.version_info >= (3, 11) and self._max_tasks_per_child
200
+ ): # pragma: no cover - version dependent
201
+ kwargs["max_tasks_per_child"] = self._max_tasks_per_child
202
+ self._executor = ProcessPoolExecutor(
203
+ max_workers=self._max_workers,
204
+ **kwargs,
205
+ )
206
+
207
+ def shutdown(self, wait: bool = True) -> None:
208
+ """Shut down the pool and release worker processes."""
209
+ if self._executor is not None:
210
+ self._executor.shutdown(wait=wait, cancel_futures=True)
211
+ self._executor = None
212
+
213
+ async def run_ingest(self, path: str, timeout: float) -> dict[str, Any]:
214
+ """Execute statement ingestion in a warm worker under an execution deadline."""
215
+ if self._executor is None:
216
+ self.start()
217
+ executor = self._executor
218
+ if executor is None:
219
+ raise APIError("Worker pool failed to initialize")
220
+ loop = asyncio.get_running_loop()
221
+ future = loop.run_in_executor(executor, _run_worker_job, path)
222
+ try:
223
+ return await asyncio.wait_for(future, timeout=timeout)
224
+ except asyncio.TimeoutError:
225
+ raise
226
+ except Exception as exc:
227
+ raise APIError(f"Worker ingestion failed: {exc}") from exc