bankstatementparser 0.0.26__tar.gz → 0.0.27__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/PKG-INFO +5 -5
  2. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/README.md +4 -4
  3. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/__init__.py +9 -2
  4. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/additional_parsers.py +3 -0
  5. bankstatementparser-0.0.27/bankstatementparser/export/parquet.py +251 -0
  6. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/input_validator.py +44 -171
  7. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/__init__.py +2 -0
  8. bankstatementparser-0.0.27/bankstatementparser/parsers/mt942.py +275 -0
  9. bankstatementparser-0.0.27/bankstatementparser/xml_validator.py +280 -0
  10. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/pyproject.toml +1 -1
  11. bankstatementparser-0.0.26/bankstatementparser/export/parquet.py +0 -164
  12. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/LICENSE +0 -0
  13. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/LICENSE-APACHE +0 -0
  14. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/LICENSE-MIT +0 -0
  15. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/_amounts.py +0 -0
  16. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/_llm_common.py +0 -0
  17. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/analytics/__init__.py +0 -0
  18. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/analytics/cash_flow.py +0 -0
  19. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/analytics/common.py +0 -0
  20. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/analytics/models.py +0 -0
  21. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/analytics/patterns.py +0 -0
  22. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/api.py +0 -0
  23. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/api_limits.py +0 -0
  24. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/api_worker.py +0 -0
  25. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/bank_statement_parsers.py +0 -0
  26. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/base_parser.py +0 -0
  27. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/camt_parser.py +0 -0
  28. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/camt_reports.py +0 -0
  29. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/cli.py +0 -0
  30. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/__init__.py +0 -0
  31. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/account_mapper.py +0 -0
  32. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/categorizer.py +0 -0
  33. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/exceptions.py +0 -0
  34. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/export/__init__.py +0 -0
  35. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/export/ledger.py +0 -0
  36. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/field86_parser.py +0 -0
  37. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/forensics.py +0 -0
  38. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/__init__.py +0 -0
  39. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/evaluation.py +0 -0
  40. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/llm_extractor.py +0 -0
  41. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/ollama_direct.py +0 -0
  42. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/orchestrator.py +0 -0
  43. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/pdf_text.py +0 -0
  44. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/prompts.py +0 -0
  45. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/scanner.py +0 -0
  46. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/verification.py +0 -0
  47. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/vision.py +0 -0
  48. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/pain001_parser.py +0 -0
  49. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parallel.py +0 -0
  50. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/bai2.py +0 -0
  51. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/common.py +0 -0
  52. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/csv_parser.py +0 -0
  53. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/mt940.py +0 -0
  54. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/ofx.py +0 -0
  55. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/plugins.py +0 -0
  56. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/privacy.py +0 -0
  57. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/py.typed +0 -0
  58. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/reconciliation.py +0 -0
  59. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/record_types.py +0 -0
  60. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/telemetry.py +0 -0
  61. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/transaction_deduplicator.py +0 -0
  62. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/transaction_models.py +0 -0
  63. {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/zip_security.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: bankstatementparser
3
- Version: 0.0.26
3
+ Version: 0.0.27
4
4
  Summary: Parse bank statements (CAMT.053, PAIN.001, MT940, OFX, BAI2, CSV, and PDF) into unified Transactions with treasury analytics, 2-way reconciliation, PDF forensics, and ledger export.
5
5
  License: Apache-2.0 OR MIT
6
6
  License-File: LICENSE
@@ -64,7 +64,7 @@ Description-Content-Type: text/markdown
64
64
 
65
65
  <p align="center">
66
66
  <a href="https://github.com/sebastienrousseau/bankstatementparser/actions"><img src="https://img.shields.io/github/actions/workflow/status/sebastienrousseau/bankstatementparser/quality-gates.yml?style=for-the-badge&logo=github" alt="Build" /></a>
67
- <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.26" alt="PyPI" /></a>
67
+ <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.27" alt="PyPI" /></a>
68
68
  <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/dm/bankstatementparser.svg?style=for-the-badge" alt="PyPI Downloads" /></a>
69
69
  <a href="https://codecov.io/github/sebastienrousseau/bankstatementparser?branch=main"><img src="https://img.shields.io/codecov/c/github/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="Codecov" /></a>
70
70
  <a href="LICENSE"><img src="https://img.shields.io/github/license/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="License" /></a>
@@ -390,7 +390,7 @@ for the full surface.
390
390
  |---|---|
391
391
  | **PII redaction** | Names, IBANs, and addresses masked by default — opt in with `--show-pii` |
392
392
  | **Secure ZIP** | `iter_secure_xml_entries()` rejects zip bombs, encrypted entries, and suspicious compression ratios |
393
- | **Tested** | 1045 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
393
+ | **Tested** | 1058 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
394
394
 
395
395
  ---
396
396
 
@@ -750,7 +750,7 @@ cleanly — see each companion's README for runnable examples.
750
750
  ## Project Layout
751
751
 
752
752
  ```text
753
- bankstatementparser/ Source code (53 modules)
753
+ bankstatementparser/ Source code (55 modules)
754
754
  bankstatementparser/analytics/ Financial metrics, cash flow, cadence & anomaly detection
755
755
  bankstatementparser/parsers/ Modular statement parsers (CSV, MT940, OFX/QFX)
756
756
  bankstatementparser/hybrid/ PDF pipeline: orchestrator, llm_extractor, vision, scanner, ollama_direct, verification
@@ -760,7 +760,7 @@ bankstatementparser/api.py REST API microservice (FastAPI)
760
760
  docs/compliance/ ISO 13485 validation, risk register, traceability matrix
761
761
  examples/ 14 deterministic + 9 hybrid runnable example scripts
762
762
  scripts/ SBOM generation, checksums, signature verification
763
- tests/ 1045 tests (unit, integration, property-based, security, hybrid mocks)
763
+ tests/ 1058 tests (unit, integration, property-based, security, hybrid mocks)
764
764
  ```
765
765
 
766
766
  ---
@@ -19,7 +19,7 @@
19
19
 
20
20
  <p align="center">
21
21
  <a href="https://github.com/sebastienrousseau/bankstatementparser/actions"><img src="https://img.shields.io/github/actions/workflow/status/sebastienrousseau/bankstatementparser/quality-gates.yml?style=for-the-badge&logo=github" alt="Build" /></a>
22
- <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.26" alt="PyPI" /></a>
22
+ <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.27" alt="PyPI" /></a>
23
23
  <a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/dm/bankstatementparser.svg?style=for-the-badge" alt="PyPI Downloads" /></a>
24
24
  <a href="https://codecov.io/github/sebastienrousseau/bankstatementparser?branch=main"><img src="https://img.shields.io/codecov/c/github/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="Codecov" /></a>
25
25
  <a href="LICENSE"><img src="https://img.shields.io/github/license/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="License" /></a>
@@ -345,7 +345,7 @@ for the full surface.
345
345
  |---|---|
346
346
  | **PII redaction** | Names, IBANs, and addresses masked by default — opt in with `--show-pii` |
347
347
  | **Secure ZIP** | `iter_secure_xml_entries()` rejects zip bombs, encrypted entries, and suspicious compression ratios |
348
- | **Tested** | 1045 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
348
+ | **Tested** | 1058 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
349
349
 
350
350
  ---
351
351
 
@@ -705,7 +705,7 @@ cleanly — see each companion's README for runnable examples.
705
705
  ## Project Layout
706
706
 
707
707
  ```text
708
- bankstatementparser/ Source code (53 modules)
708
+ bankstatementparser/ Source code (55 modules)
709
709
  bankstatementparser/analytics/ Financial metrics, cash flow, cadence & anomaly detection
710
710
  bankstatementparser/parsers/ Modular statement parsers (CSV, MT940, OFX/QFX)
711
711
  bankstatementparser/hybrid/ PDF pipeline: orchestrator, llm_extractor, vision, scanner, ollama_direct, verification
@@ -715,7 +715,7 @@ bankstatementparser/api.py REST API microservice (FastAPI)
715
715
  docs/compliance/ ISO 13485 validation, risk register, traceability matrix
716
716
  examples/ 14 deterministic + 9 hybrid runnable example scripts
717
717
  scripts/ SBOM generation, checksums, signature verification
718
- tests/ 1045 tests (unit, integration, property-based, security, hybrid mocks)
718
+ tests/ 1058 tests (unit, integration, property-based, security, hybrid mocks)
719
719
  ```
720
720
 
721
721
  ---
@@ -25,6 +25,7 @@ from .additional_parsers import (
25
25
  Camt054Parser,
26
26
  CsvStatementParser,
27
27
  Mt940Parser,
28
+ Mt942Parser,
28
29
  OfxParser,
29
30
  QfxParser,
30
31
  create_parser,
@@ -51,7 +52,11 @@ from .exceptions import (
51
52
  Pain001ParseError,
52
53
  ParserError,
53
54
  )
54
- from .export.parquet import export_parquet, export_parquet_stream
55
+ from .export.parquet import (
56
+ ParquetStreamWriter,
57
+ export_parquet,
58
+ export_parquet_stream,
59
+ )
55
60
  from .field86_parser import Field86Structure, parse_field_86
56
61
  from .forensics import (
57
62
  ForensicFinding,
@@ -102,7 +107,7 @@ from .zip_security import (
102
107
  #: so the conformance gate can compare it statically against
103
108
  #: pyproject.toml -- that comparison is the thing that catches drift, and
104
109
  #: a runtime lookup would make the two trivially equal and check nothing.
105
- __version__ = "0.0.26"
110
+ __version__ = "0.0.27"
106
111
 
107
112
  __all__ = [
108
113
  "AnalyticsReport",
@@ -129,10 +134,12 @@ __all__ = [
129
134
  "InputValidator",
130
135
  "MatchGroup",
131
136
  "Mt940Parser",
137
+ "Mt942Parser",
132
138
  "NullSpan",
133
139
  "OfxParser",
134
140
  "Pain001ParseError",
135
141
  "Pain001Parser",
142
+ "ParquetStreamWriter",
136
143
  "ParserError",
137
144
  "QfxParser",
138
145
  "ReconciliationMatch",
@@ -29,6 +29,7 @@ from .parsers import (
29
29
  Bai2Parser,
30
30
  CsvStatementParser,
31
31
  Mt940Parser,
32
+ Mt942Parser,
32
33
  OfxParser,
33
34
  QfxParser,
34
35
  _amount_or_zero,
@@ -49,6 +50,7 @@ __all__ = [
49
50
  "Camt054Parser",
50
51
  "CsvStatementParser",
51
52
  "Mt940Parser",
53
+ "Mt942Parser",
52
54
  "OfxParser",
53
55
  "QfxParser",
54
56
  "_amount_or_zero",
@@ -151,6 +153,7 @@ def create_parser(
151
153
  "ofx": OfxParser,
152
154
  "qfx": QfxParser,
153
155
  "mt940": Mt940Parser,
156
+ "mt942": Mt942Parser,
154
157
  "bai2": Bai2Parser,
155
158
  }
156
159
  # Augment with dynamic plugins
@@ -0,0 +1,251 @@
1
+ # SPDX-License-Identifier: Apache-2.0 OR MIT
2
+ # Copyright (C) 2023-2026 Bank Statement Parser. All rights reserved.
3
+
4
+ """Apache Parquet columnar export for financial statements."""
5
+
6
+ from __future__ import annotations
7
+
8
+ import importlib
9
+ import io
10
+ import os
11
+ import tempfile
12
+ from collections.abc import Iterable, Mapping
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ import pandas as pd
17
+
18
+ from ..input_validator import InputValidator
19
+ from ..privacy import redact_record
20
+
21
+ __all__ = [
22
+ "ParquetStreamWriter",
23
+ "export_parquet",
24
+ "export_parquet_stream",
25
+ ]
26
+
27
+
28
+ class ParquetStreamWriter:
29
+ """Stream records in chunked batches directly to an Apache Parquet destination."""
30
+
31
+ def __init__(
32
+ self,
33
+ output_path: str | Path,
34
+ *,
35
+ schema: Any,
36
+ batch_size: int = 1000,
37
+ compression: str = "snappy",
38
+ redact_pii: bool = False,
39
+ ) -> None:
40
+ """Initialize the Parquet streaming writer with validated schema and destination."""
41
+ if (
42
+ isinstance(batch_size, bool)
43
+ or not isinstance(batch_size, int)
44
+ or batch_size <= 0
45
+ ):
46
+ raise ValueError("batch_size must be a positive integer")
47
+ try:
48
+ self._pa = importlib.import_module("pyarrow")
49
+ self._pq = importlib.import_module("pyarrow.parquet")
50
+ except ImportError as exc:
51
+ raise ImportError(
52
+ "Install bankstatementparser[parquet] for streaming Parquet export"
53
+ ) from exc
54
+
55
+ names = set(schema.names)
56
+ if len(names) != len(schema.names):
57
+ raise ValueError("Parquet schema field names must be unique")
58
+ self._schema = schema
59
+ self._names = names
60
+ self._required = [field.name for field in schema if not field.nullable]
61
+ self._batch_size = batch_size
62
+ self._compression = compression
63
+ self._redact_pii = redact_pii
64
+
65
+ self._destination = InputValidator().validate_output_file_path(
66
+ str(output_path)
67
+ )
68
+ with tempfile.NamedTemporaryFile(
69
+ dir=self._destination.parent,
70
+ prefix=".bsp_",
71
+ suffix=".parquet.tmp",
72
+ delete=False,
73
+ ) as temporary:
74
+ self._temp_path = Path(temporary.name)
75
+
76
+ self._writer = self._pq.ParquetWriter(
77
+ self._temp_path, self._schema, compression=self._compression
78
+ )
79
+ self._writer.__enter__()
80
+ self._batch: list[dict[str, Any]] = []
81
+ self._rows_written = 0
82
+ self._closed = False
83
+
84
+ @property
85
+ def rows_written(self) -> int:
86
+ """Return the total number of records successfully written."""
87
+ return self._rows_written
88
+
89
+ def write_record(self, record: Mapping[str, Any]) -> None:
90
+ """Validate, redact, buffer and write a single statement record."""
91
+ if self._closed:
92
+ raise ValueError("Cannot write to a closed ParquetStreamWriter")
93
+ row = redact_record(record) if self._redact_pii else dict(record)
94
+ if set(row) - self._names:
95
+ raise ValueError(
96
+ "Record contains fields absent from the Parquet schema"
97
+ )
98
+ if any(row.get(name) is None for name in self._required):
99
+ raise ValueError("Record is missing a required Parquet field")
100
+ self._batch.append(row)
101
+ self._rows_written += 1
102
+ if len(self._batch) >= self._batch_size:
103
+ self.flush()
104
+
105
+ def write_batch(self, records: Iterable[Mapping[str, Any]]) -> None:
106
+ """Write an iterable sequence of records to the stream."""
107
+ for record in records:
108
+ self.write_record(record)
109
+
110
+ def flush(self) -> None:
111
+ """Flush the current buffered batch of records to disk."""
112
+ if self._batch:
113
+ self._writer.write_table(
114
+ self._pa.Table.from_pylist(self._batch, schema=self._schema)
115
+ )
116
+ self._batch = []
117
+
118
+ def close(self) -> int:
119
+ """Flush pending batches, finalize file and atomically commit to destination."""
120
+ if self._closed:
121
+ return self._rows_written
122
+ try:
123
+ self.flush()
124
+ self._writer.__exit__(None, None, None)
125
+ os.replace(self._temp_path, self._destination)
126
+ self._closed = True
127
+ finally:
128
+ self._temp_path.unlink(missing_ok=True)
129
+ return self._rows_written
130
+
131
+ def __enter__(self) -> ParquetStreamWriter:
132
+ """Enter context manager scope."""
133
+ return self
134
+
135
+ def __exit__(
136
+ self,
137
+ exc_type: type[BaseException] | None,
138
+ exc_val: BaseException | None,
139
+ exc_tb: Any,
140
+ ) -> None:
141
+ """Exit context manager, committing on success or removing temp file on error."""
142
+ try:
143
+ if exc_type is None:
144
+ self.close()
145
+ else:
146
+ self._writer.__exit__(exc_type, exc_val, exc_tb)
147
+ finally:
148
+ self._temp_path.unlink(missing_ok=True)
149
+
150
+
151
+ def _convert_records(transactions: Iterable[Any]) -> list[dict[str, Any]]:
152
+ """Convert an iterable of objects or dicts into a list of row dicts."""
153
+ records: list[dict[str, Any]] = []
154
+ for item in transactions:
155
+ if isinstance(item, dict):
156
+ records.append(item)
157
+ elif hasattr(item, "model_dump"):
158
+ records.append(item.model_dump())
159
+ elif hasattr(item, "to_dict"):
160
+ records.append(item.to_dict())
161
+ elif hasattr(item, "__dict__"):
162
+ records.append(
163
+ {
164
+ k: v
165
+ for k, v in item.__dict__.items()
166
+ if not k.startswith("_")
167
+ }
168
+ )
169
+ else:
170
+ records.append({"value": str(item)})
171
+ return records
172
+
173
+
174
+ def export_parquet(
175
+ transactions: Iterable[Any] | pd.DataFrame,
176
+ output_path: str | Path | None = None,
177
+ compression: str = "snappy",
178
+ *,
179
+ redact_pii: bool = False,
180
+ ) -> bytes:
181
+ """Export statement transactions to Apache Parquet format.
182
+
183
+ Args:
184
+ transactions: Sequence of transactions, dictionaries, or a pandas DataFrame.
185
+ output_path: Optional file path to write the Parquet file to.
186
+ compression: Compression codec ('snappy', 'gzip', 'zstd', None).
187
+ redact_pii: Mask identities, narratives and provenance fields.
188
+
189
+ Returns:
190
+ Parquet file contents as bytes.
191
+ """
192
+ validated_destination = (
193
+ InputValidator().validate_output_file_path(str(output_path))
194
+ if output_path is not None
195
+ else None
196
+ )
197
+ if isinstance(transactions, pd.DataFrame):
198
+ df = transactions.copy()
199
+ else:
200
+ records = _convert_records(transactions)
201
+ df = pd.DataFrame(records)
202
+
203
+ if redact_pii:
204
+ df = pd.DataFrame(
205
+ [redact_record(row) for row in df.to_dict("records")],
206
+ columns=df.columns,
207
+ )
208
+
209
+ buf = io.BytesIO()
210
+ try:
211
+ df.to_parquet(buf, compression=compression, engine="pyarrow")
212
+ except ImportError as exc:
213
+ raise ImportError(
214
+ "Apache Parquet export requires 'pyarrow' or 'fastparquet'. "
215
+ "Install with: pip install 'bankstatementparser[parquet]' or pip install pyarrow"
216
+ ) from exc
217
+ parquet_bytes = buf.getvalue()
218
+
219
+ if validated_destination is not None:
220
+ validated_destination.write_bytes(parquet_bytes)
221
+
222
+ return parquet_bytes
223
+
224
+
225
+ def export_parquet_stream(
226
+ records: Iterable[Mapping[str, Any]],
227
+ output_path: str | Path,
228
+ *,
229
+ schema: Any,
230
+ batch_size: int = 10000,
231
+ compression: str = "snappy",
232
+ redact_pii: bool = False,
233
+ ) -> int:
234
+ """Write bounded Parquet batches atomically using an explicit Arrow schema.
235
+
236
+ Supply a ``pyarrow.Schema`` covering all input keys, with adequate Decimal
237
+ precision/scale and string types for fields being redacted. Missing nullable
238
+ fields become null. Unknown fields, missing required fields and values that
239
+ cannot be represented fail without replacing an existing destination.
240
+ Returns the written row count; unlike export_parquet, no file bytes are
241
+ retained. Memory depends on batch size and individual record sizes.
242
+ """
243
+ with ParquetStreamWriter(
244
+ output_path,
245
+ schema=schema,
246
+ batch_size=batch_size,
247
+ compression=compression,
248
+ redact_pii=redact_pii,
249
+ ) as writer:
250
+ writer.write_batch(records)
251
+ return writer.rows_written
@@ -27,10 +27,40 @@ from pathlib import Path
27
27
  from typing import ClassVar, Optional, Union
28
28
 
29
29
  from .exceptions import ValidationError
30
+ from .xml_validator import (
31
+ check_binary_signatures,
32
+ check_xml_indicators,
33
+ sanitize_filename,
34
+ sanitize_source_name,
35
+ validate_payload_size,
36
+ validate_xml_bytes_format,
37
+ validate_xml_content,
38
+ )
30
39
 
31
40
  logger = logging.getLogger(__name__)
32
41
 
33
- __all__ = ["InputValidator", "ValidationError"]
42
+ __all__ = [
43
+ "InputValidator",
44
+ "ValidationError",
45
+ "sanitize_filename",
46
+ "sanitize_source_name",
47
+ "validate_xml_content",
48
+ ]
49
+
50
+ _DANGEROUS_UNICODE: tuple[str, ...] = (
51
+ "\u0000",
52
+ "\u202e",
53
+ "\u202d",
54
+ "\u200f",
55
+ "\u200e",
56
+ "\u2066",
57
+ "\u2067",
58
+ "\u2068",
59
+ "\u2069",
60
+ "\u202a",
61
+ "\u202b",
62
+ "\u202c",
63
+ )
34
64
 
35
65
 
36
66
  class InputValidator:
@@ -275,33 +305,7 @@ class InputValidator:
275
305
  Raises:
276
306
  ValidationError: If source_name is not a string.
277
307
  """
278
- if source_name is None:
279
- return default
280
-
281
- if not isinstance(source_name, str):
282
- raise ValidationError("Source name must be a string")
283
-
284
- cleaned = []
285
- for char in source_name.strip():
286
- if ord(char) < 32 or char in {
287
- "\u202e",
288
- "\u202d",
289
- "\u200f",
290
- "\u200e",
291
- "\u2066",
292
- "\u2067",
293
- "\u2068",
294
- "\u2069",
295
- "\u202a",
296
- "\u202b",
297
- "\u202c",
298
- }:
299
- cleaned.append("?")
300
- else:
301
- cleaned.append(char)
302
-
303
- sanitized = "".join(cleaned)[:255]
304
- return sanitized or default
308
+ return sanitize_source_name(source_name, default)
305
309
 
306
310
  def validate_xml_content(
307
311
  self,
@@ -321,44 +325,17 @@ class InputValidator:
321
325
  Raises:
322
326
  ValidationError: If content is unsafe, empty, oversized, or not XML.
323
327
  """
324
- safe_source_name = self.sanitize_source_name(source_name)
325
-
326
- if isinstance(xml_content, str):
327
- if not xml_content.strip():
328
- raise ValidationError("XML content cannot be empty")
329
- xml_bytes = xml_content.encode("utf-8")
330
- elif isinstance(xml_content, bytes):
331
- if not xml_content.strip():
332
- raise ValidationError("XML content cannot be empty")
333
- xml_bytes = xml_content
334
- else:
335
- raise ValidationError(
336
- "XML content must be provided as a string or bytes"
337
- )
338
-
339
- self._validate_bytes_size(xml_bytes)
340
- self._validate_xml_bytes_format(xml_bytes, safe_source_name)
341
-
342
- return xml_bytes, safe_source_name
328
+ return validate_xml_content(
329
+ xml_content,
330
+ max_file_size=self.max_file_size,
331
+ source_name=source_name,
332
+ log=logger,
333
+ )
343
334
 
344
335
  def _check_dangerous_patterns(self, file_path: str) -> None:
345
336
  """Check for dangerous patterns in file path."""
346
337
  # Check for dangerous Unicode characters (null bytes, BiDi overrides, etc.)
347
- dangerous_unicode = [
348
- "\u0000", # Null byte
349
- "\u202e", # Right-to-left override
350
- "\u202d", # Left-to-right override
351
- "\u200f", # Right-to-left mark
352
- "\u200e", # Left-to-right mark
353
- "\u2066", # Left-to-right isolate
354
- "\u2067", # Right-to-left isolate
355
- "\u2068", # First strong isolate
356
- "\u2069", # Pop directional isolate
357
- "\u202a", # Left-to-right embedding
358
- "\u202b", # Right-to-left embedding
359
- "\u202c", # Pop directional formatting
360
- ]
361
- for char in dangerous_unicode:
338
+ for char in _DANGEROUS_UNICODE:
362
339
  if char in file_path:
363
340
  raise ValidationError(
364
341
  "Potentially dangerous path pattern detected"
@@ -431,19 +408,7 @@ class InputValidator:
431
408
 
432
409
  def _validate_bytes_size(self, data: bytes) -> None:
433
410
  """Validate in-memory payload size constraints."""
434
- payload_size = len(data)
435
-
436
- if payload_size < self.MIN_FILE_SIZE_BYTES:
437
- raise ValidationError(
438
- f"XML content is too small ({payload_size} bytes). Minimum: {self.MIN_FILE_SIZE_BYTES} bytes"
439
- )
440
-
441
- if payload_size > self.max_file_size:
442
- size_mb = payload_size / (1024 * 1024)
443
- max_mb = self.max_file_size / (1024 * 1024)
444
- raise ValidationError(
445
- f"XML content is too large ({size_mb:.1f}MB). Maximum allowed: {max_mb:.1f}MB"
446
- )
411
+ validate_payload_size(data, self.max_file_size)
447
412
 
448
413
  def _read_file_header(self, path: Path) -> bytes:
449
414
  """Read the initial bytes of a file for format detection."""
@@ -463,39 +428,11 @@ class InputValidator:
463
428
 
464
429
  def _check_binary_signatures(self, header: bytes, path: Path) -> None:
465
430
  """Check for known binary signatures that indicate invalid text formats."""
466
- binary_signatures = [
467
- b"\x89PNG",
468
- b"GIF8",
469
- b"\xff\xd8\xff",
470
- b"PK",
471
- b"\x7fELF",
472
- b"MZ",
473
- b"\x00\x00\x01\x00",
474
- b"%PDF",
475
- ]
476
- for sig in binary_signatures:
477
- if header[: len(sig)] == sig:
478
- raise ValidationError(
479
- f"File appears to contain binary data, expected XML: {path}"
480
- )
431
+ check_binary_signatures(header, path)
481
432
 
482
433
  def _check_xml_indicators(self, header: bytes, path: Path) -> None:
483
434
  """Check for XML declaration or common root namespace indicators."""
484
- header_str = header.decode("utf-8", errors="ignore").lower()
485
- xml_indicators = [
486
- "<?xml",
487
- "<document",
488
- "xmlns",
489
- "camt.053",
490
- "pain.001",
491
- "iso:std:iso:20022",
492
- ]
493
- if not any(ind in header_str for ind in xml_indicators):
494
- if any(c < 32 and c not in (9, 10, 13) for c in header[:100]):
495
- raise ValidationError(
496
- f"File appears to contain binary data, expected XML: {path}"
497
- )
498
- logger.warning(f"File may not be a valid XML document: {path}")
435
+ check_xml_indicators(header, path, logger)
499
436
 
500
437
  def _validate_input_format(self, path: Path) -> None:
501
438
  """Validate input file format by checking file content.
@@ -544,56 +481,7 @@ class InputValidator:
544
481
  Raises:
545
482
  ValidationError: If content is not plausible UTF-8 XML.
546
483
  """
547
- header = xml_bytes[:1024]
548
-
549
- binary_signatures = [
550
- b"\x89PNG",
551
- b"GIF8",
552
- b"\xff\xd8\xff",
553
- b"PK",
554
- b"\x7fELF",
555
- b"MZ",
556
- b"\x00\x00\x01\x00",
557
- b"%PDF",
558
- ]
559
- for sig in binary_signatures:
560
- if header[: len(sig)] == sig:
561
- raise ValidationError(
562
- f"XML content appears to contain binary data, expected XML: {source_name}"
563
- )
564
-
565
- try:
566
- header_str = header.decode("utf-8")
567
- except UnicodeDecodeError as exc:
568
- raise ValidationError(
569
- f"XML content encoding is not valid UTF-8: {source_name}"
570
- ) from exc
571
-
572
- header_lower = header_str.lower()
573
- xml_indicators = [
574
- "<?xml",
575
- "<document",
576
- "xmlns",
577
- "camt.",
578
- "iso:std:iso:20022",
579
- ]
580
-
581
- has_xml_indicator = any(
582
- indicator in header_lower for indicator in xml_indicators
583
- )
584
-
585
- if not has_xml_indicator:
586
- if any(
587
- byte < 32 and byte not in (9, 10, 13) for byte in header[:100]
588
- ):
589
- raise ValidationError(
590
- f"XML content appears to contain binary data, expected XML: {source_name}"
591
- )
592
-
593
- logger.warning(
594
- "XML content may not be a valid XML document: %s",
595
- source_name,
596
- )
484
+ validate_xml_bytes_format(xml_bytes, source_name, logger)
597
485
 
598
486
  def get_safe_filename(self, filename: str) -> str:
599
487
  """Generate a safe filename by removing/replacing dangerous characters.
@@ -604,19 +492,4 @@ class InputValidator:
604
492
  Returns:
605
493
  str: Safe filename.
606
494
  """
607
- # Remove or replace dangerous characters
608
- safe_chars = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "_", filename)
609
-
610
- # Remove leading/trailing dots and spaces
611
- safe_chars = safe_chars.strip(". ")
612
-
613
- # Ensure filename is not empty
614
- if not safe_chars:
615
- safe_chars = "unnamed_file"
616
-
617
- # Truncate if too long (keeping extension)
618
- if len(safe_chars) > 255:
619
- name, ext = os.path.splitext(safe_chars)
620
- safe_chars = name[: 255 - len(ext)] + ext
621
-
622
- return safe_chars
495
+ return sanitize_filename(filename)
@@ -31,6 +31,7 @@ from .common import (
31
31
  )
32
32
  from .csv_parser import CsvStatementParser
33
33
  from .mt940 import Mt940Parser
34
+ from .mt942 import Mt942Parser
34
35
  from .ofx import OfxParser, QfxParser
35
36
 
36
37
  __all__ = [
@@ -38,6 +39,7 @@ __all__ = [
38
39
  "Bai2Parser",
39
40
  "CsvStatementParser",
40
41
  "Mt940Parser",
42
+ "Mt942Parser",
41
43
  "OfxParser",
42
44
  "QfxParser",
43
45
  "_amount_or_zero",