bankstatementparser 0.0.26__tar.gz → 0.0.27__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/PKG-INFO +5 -5
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/README.md +4 -4
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/__init__.py +9 -2
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/additional_parsers.py +3 -0
- bankstatementparser-0.0.27/bankstatementparser/export/parquet.py +251 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/input_validator.py +44 -171
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/__init__.py +2 -0
- bankstatementparser-0.0.27/bankstatementparser/parsers/mt942.py +275 -0
- bankstatementparser-0.0.27/bankstatementparser/xml_validator.py +280 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/pyproject.toml +1 -1
- bankstatementparser-0.0.26/bankstatementparser/export/parquet.py +0 -164
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/LICENSE +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/LICENSE-APACHE +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/LICENSE-MIT +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/_amounts.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/_llm_common.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/analytics/__init__.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/analytics/cash_flow.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/analytics/common.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/analytics/models.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/analytics/patterns.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/api.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/api_limits.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/api_worker.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/bank_statement_parsers.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/base_parser.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/camt_parser.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/camt_reports.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/cli.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/__init__.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/account_mapper.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/enrichment/categorizer.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/exceptions.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/export/__init__.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/export/ledger.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/field86_parser.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/forensics.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/__init__.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/evaluation.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/llm_extractor.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/ollama_direct.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/orchestrator.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/pdf_text.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/prompts.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/scanner.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/verification.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/hybrid/vision.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/pain001_parser.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parallel.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/bai2.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/common.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/csv_parser.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/mt940.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/ofx.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/plugins.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/privacy.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/py.typed +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/reconciliation.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/record_types.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/telemetry.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/transaction_deduplicator.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/transaction_models.py +0 -0
- {bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/zip_security.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bankstatementparser
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.27
|
|
4
4
|
Summary: Parse bank statements (CAMT.053, PAIN.001, MT940, OFX, BAI2, CSV, and PDF) into unified Transactions with treasury analytics, 2-way reconciliation, PDF forensics, and ledger export.
|
|
5
5
|
License: Apache-2.0 OR MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -64,7 +64,7 @@ Description-Content-Type: text/markdown
|
|
|
64
64
|
|
|
65
65
|
<p align="center">
|
|
66
66
|
<a href="https://github.com/sebastienrousseau/bankstatementparser/actions"><img src="https://img.shields.io/github/actions/workflow/status/sebastienrousseau/bankstatementparser/quality-gates.yml?style=for-the-badge&logo=github" alt="Build" /></a>
|
|
67
|
-
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.
|
|
67
|
+
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.27" alt="PyPI" /></a>
|
|
68
68
|
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/dm/bankstatementparser.svg?style=for-the-badge" alt="PyPI Downloads" /></a>
|
|
69
69
|
<a href="https://codecov.io/github/sebastienrousseau/bankstatementparser?branch=main"><img src="https://img.shields.io/codecov/c/github/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="Codecov" /></a>
|
|
70
70
|
<a href="LICENSE"><img src="https://img.shields.io/github/license/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="License" /></a>
|
|
@@ -390,7 +390,7 @@ for the full surface.
|
|
|
390
390
|
|---|---|
|
|
391
391
|
| **PII redaction** | Names, IBANs, and addresses masked by default — opt in with `--show-pii` |
|
|
392
392
|
| **Secure ZIP** | `iter_secure_xml_entries()` rejects zip bombs, encrypted entries, and suspicious compression ratios |
|
|
393
|
-
| **Tested** |
|
|
393
|
+
| **Tested** | 1058 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
|
|
394
394
|
|
|
395
395
|
---
|
|
396
396
|
|
|
@@ -750,7 +750,7 @@ cleanly — see each companion's README for runnable examples.
|
|
|
750
750
|
## Project Layout
|
|
751
751
|
|
|
752
752
|
```text
|
|
753
|
-
bankstatementparser/ Source code (
|
|
753
|
+
bankstatementparser/ Source code (55 modules)
|
|
754
754
|
bankstatementparser/analytics/ Financial metrics, cash flow, cadence & anomaly detection
|
|
755
755
|
bankstatementparser/parsers/ Modular statement parsers (CSV, MT940, OFX/QFX)
|
|
756
756
|
bankstatementparser/hybrid/ PDF pipeline: orchestrator, llm_extractor, vision, scanner, ollama_direct, verification
|
|
@@ -760,7 +760,7 @@ bankstatementparser/api.py REST API microservice (FastAPI)
|
|
|
760
760
|
docs/compliance/ ISO 13485 validation, risk register, traceability matrix
|
|
761
761
|
examples/ 14 deterministic + 9 hybrid runnable example scripts
|
|
762
762
|
scripts/ SBOM generation, checksums, signature verification
|
|
763
|
-
tests/
|
|
763
|
+
tests/ 1058 tests (unit, integration, property-based, security, hybrid mocks)
|
|
764
764
|
```
|
|
765
765
|
|
|
766
766
|
---
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
|
|
20
20
|
<p align="center">
|
|
21
21
|
<a href="https://github.com/sebastienrousseau/bankstatementparser/actions"><img src="https://img.shields.io/github/actions/workflow/status/sebastienrousseau/bankstatementparser/quality-gates.yml?style=for-the-badge&logo=github" alt="Build" /></a>
|
|
22
|
-
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.
|
|
22
|
+
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/pyversions/bankstatementparser.svg?style=for-the-badge&v=0.0.27" alt="PyPI" /></a>
|
|
23
23
|
<a href="https://pypi.org/project/bankstatementparser/"><img src="https://img.shields.io/pypi/dm/bankstatementparser.svg?style=for-the-badge" alt="PyPI Downloads" /></a>
|
|
24
24
|
<a href="https://codecov.io/github/sebastienrousseau/bankstatementparser?branch=main"><img src="https://img.shields.io/codecov/c/github/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="Codecov" /></a>
|
|
25
25
|
<a href="LICENSE"><img src="https://img.shields.io/github/license/sebastienrousseau/bankstatementparser?style=for-the-badge" alt="License" /></a>
|
|
@@ -345,7 +345,7 @@ for the full surface.
|
|
|
345
345
|
|---|---|
|
|
346
346
|
| **PII redaction** | Names, IBANs, and addresses masked by default — opt in with `--show-pii` |
|
|
347
347
|
| **Secure ZIP** | `iter_secure_xml_entries()` rejects zip bombs, encrypted entries, and suspicious compression ratios |
|
|
348
|
-
| **Tested** |
|
|
348
|
+
| **Tested** | 1058 tests, coverage gated at 100% in CI, property-based fuzzing with Hypothesis |
|
|
349
349
|
|
|
350
350
|
---
|
|
351
351
|
|
|
@@ -705,7 +705,7 @@ cleanly — see each companion's README for runnable examples.
|
|
|
705
705
|
## Project Layout
|
|
706
706
|
|
|
707
707
|
```text
|
|
708
|
-
bankstatementparser/ Source code (
|
|
708
|
+
bankstatementparser/ Source code (55 modules)
|
|
709
709
|
bankstatementparser/analytics/ Financial metrics, cash flow, cadence & anomaly detection
|
|
710
710
|
bankstatementparser/parsers/ Modular statement parsers (CSV, MT940, OFX/QFX)
|
|
711
711
|
bankstatementparser/hybrid/ PDF pipeline: orchestrator, llm_extractor, vision, scanner, ollama_direct, verification
|
|
@@ -715,7 +715,7 @@ bankstatementparser/api.py REST API microservice (FastAPI)
|
|
|
715
715
|
docs/compliance/ ISO 13485 validation, risk register, traceability matrix
|
|
716
716
|
examples/ 14 deterministic + 9 hybrid runnable example scripts
|
|
717
717
|
scripts/ SBOM generation, checksums, signature verification
|
|
718
|
-
tests/
|
|
718
|
+
tests/ 1058 tests (unit, integration, property-based, security, hybrid mocks)
|
|
719
719
|
```
|
|
720
720
|
|
|
721
721
|
---
|
|
@@ -25,6 +25,7 @@ from .additional_parsers import (
|
|
|
25
25
|
Camt054Parser,
|
|
26
26
|
CsvStatementParser,
|
|
27
27
|
Mt940Parser,
|
|
28
|
+
Mt942Parser,
|
|
28
29
|
OfxParser,
|
|
29
30
|
QfxParser,
|
|
30
31
|
create_parser,
|
|
@@ -51,7 +52,11 @@ from .exceptions import (
|
|
|
51
52
|
Pain001ParseError,
|
|
52
53
|
ParserError,
|
|
53
54
|
)
|
|
54
|
-
from .export.parquet import
|
|
55
|
+
from .export.parquet import (
|
|
56
|
+
ParquetStreamWriter,
|
|
57
|
+
export_parquet,
|
|
58
|
+
export_parquet_stream,
|
|
59
|
+
)
|
|
55
60
|
from .field86_parser import Field86Structure, parse_field_86
|
|
56
61
|
from .forensics import (
|
|
57
62
|
ForensicFinding,
|
|
@@ -102,7 +107,7 @@ from .zip_security import (
|
|
|
102
107
|
#: so the conformance gate can compare it statically against
|
|
103
108
|
#: pyproject.toml -- that comparison is the thing that catches drift, and
|
|
104
109
|
#: a runtime lookup would make the two trivially equal and check nothing.
|
|
105
|
-
__version__ = "0.0.
|
|
110
|
+
__version__ = "0.0.27"
|
|
106
111
|
|
|
107
112
|
__all__ = [
|
|
108
113
|
"AnalyticsReport",
|
|
@@ -129,10 +134,12 @@ __all__ = [
|
|
|
129
134
|
"InputValidator",
|
|
130
135
|
"MatchGroup",
|
|
131
136
|
"Mt940Parser",
|
|
137
|
+
"Mt942Parser",
|
|
132
138
|
"NullSpan",
|
|
133
139
|
"OfxParser",
|
|
134
140
|
"Pain001ParseError",
|
|
135
141
|
"Pain001Parser",
|
|
142
|
+
"ParquetStreamWriter",
|
|
136
143
|
"ParserError",
|
|
137
144
|
"QfxParser",
|
|
138
145
|
"ReconciliationMatch",
|
{bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/additional_parsers.py
RENAMED
|
@@ -29,6 +29,7 @@ from .parsers import (
|
|
|
29
29
|
Bai2Parser,
|
|
30
30
|
CsvStatementParser,
|
|
31
31
|
Mt940Parser,
|
|
32
|
+
Mt942Parser,
|
|
32
33
|
OfxParser,
|
|
33
34
|
QfxParser,
|
|
34
35
|
_amount_or_zero,
|
|
@@ -49,6 +50,7 @@ __all__ = [
|
|
|
49
50
|
"Camt054Parser",
|
|
50
51
|
"CsvStatementParser",
|
|
51
52
|
"Mt940Parser",
|
|
53
|
+
"Mt942Parser",
|
|
52
54
|
"OfxParser",
|
|
53
55
|
"QfxParser",
|
|
54
56
|
"_amount_or_zero",
|
|
@@ -151,6 +153,7 @@ def create_parser(
|
|
|
151
153
|
"ofx": OfxParser,
|
|
152
154
|
"qfx": QfxParser,
|
|
153
155
|
"mt940": Mt940Parser,
|
|
156
|
+
"mt942": Mt942Parser,
|
|
154
157
|
"bai2": Bai2Parser,
|
|
155
158
|
}
|
|
156
159
|
# Augment with dynamic plugins
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0 OR MIT
|
|
2
|
+
# Copyright (C) 2023-2026 Bank Statement Parser. All rights reserved.
|
|
3
|
+
|
|
4
|
+
"""Apache Parquet columnar export for financial statements."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import importlib
|
|
9
|
+
import io
|
|
10
|
+
import os
|
|
11
|
+
import tempfile
|
|
12
|
+
from collections.abc import Iterable, Mapping
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
from ..input_validator import InputValidator
|
|
19
|
+
from ..privacy import redact_record
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"ParquetStreamWriter",
|
|
23
|
+
"export_parquet",
|
|
24
|
+
"export_parquet_stream",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ParquetStreamWriter:
|
|
29
|
+
"""Stream records in chunked batches directly to an Apache Parquet destination."""
|
|
30
|
+
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
output_path: str | Path,
|
|
34
|
+
*,
|
|
35
|
+
schema: Any,
|
|
36
|
+
batch_size: int = 1000,
|
|
37
|
+
compression: str = "snappy",
|
|
38
|
+
redact_pii: bool = False,
|
|
39
|
+
) -> None:
|
|
40
|
+
"""Initialize the Parquet streaming writer with validated schema and destination."""
|
|
41
|
+
if (
|
|
42
|
+
isinstance(batch_size, bool)
|
|
43
|
+
or not isinstance(batch_size, int)
|
|
44
|
+
or batch_size <= 0
|
|
45
|
+
):
|
|
46
|
+
raise ValueError("batch_size must be a positive integer")
|
|
47
|
+
try:
|
|
48
|
+
self._pa = importlib.import_module("pyarrow")
|
|
49
|
+
self._pq = importlib.import_module("pyarrow.parquet")
|
|
50
|
+
except ImportError as exc:
|
|
51
|
+
raise ImportError(
|
|
52
|
+
"Install bankstatementparser[parquet] for streaming Parquet export"
|
|
53
|
+
) from exc
|
|
54
|
+
|
|
55
|
+
names = set(schema.names)
|
|
56
|
+
if len(names) != len(schema.names):
|
|
57
|
+
raise ValueError("Parquet schema field names must be unique")
|
|
58
|
+
self._schema = schema
|
|
59
|
+
self._names = names
|
|
60
|
+
self._required = [field.name for field in schema if not field.nullable]
|
|
61
|
+
self._batch_size = batch_size
|
|
62
|
+
self._compression = compression
|
|
63
|
+
self._redact_pii = redact_pii
|
|
64
|
+
|
|
65
|
+
self._destination = InputValidator().validate_output_file_path(
|
|
66
|
+
str(output_path)
|
|
67
|
+
)
|
|
68
|
+
with tempfile.NamedTemporaryFile(
|
|
69
|
+
dir=self._destination.parent,
|
|
70
|
+
prefix=".bsp_",
|
|
71
|
+
suffix=".parquet.tmp",
|
|
72
|
+
delete=False,
|
|
73
|
+
) as temporary:
|
|
74
|
+
self._temp_path = Path(temporary.name)
|
|
75
|
+
|
|
76
|
+
self._writer = self._pq.ParquetWriter(
|
|
77
|
+
self._temp_path, self._schema, compression=self._compression
|
|
78
|
+
)
|
|
79
|
+
self._writer.__enter__()
|
|
80
|
+
self._batch: list[dict[str, Any]] = []
|
|
81
|
+
self._rows_written = 0
|
|
82
|
+
self._closed = False
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def rows_written(self) -> int:
|
|
86
|
+
"""Return the total number of records successfully written."""
|
|
87
|
+
return self._rows_written
|
|
88
|
+
|
|
89
|
+
def write_record(self, record: Mapping[str, Any]) -> None:
|
|
90
|
+
"""Validate, redact, buffer and write a single statement record."""
|
|
91
|
+
if self._closed:
|
|
92
|
+
raise ValueError("Cannot write to a closed ParquetStreamWriter")
|
|
93
|
+
row = redact_record(record) if self._redact_pii else dict(record)
|
|
94
|
+
if set(row) - self._names:
|
|
95
|
+
raise ValueError(
|
|
96
|
+
"Record contains fields absent from the Parquet schema"
|
|
97
|
+
)
|
|
98
|
+
if any(row.get(name) is None for name in self._required):
|
|
99
|
+
raise ValueError("Record is missing a required Parquet field")
|
|
100
|
+
self._batch.append(row)
|
|
101
|
+
self._rows_written += 1
|
|
102
|
+
if len(self._batch) >= self._batch_size:
|
|
103
|
+
self.flush()
|
|
104
|
+
|
|
105
|
+
def write_batch(self, records: Iterable[Mapping[str, Any]]) -> None:
|
|
106
|
+
"""Write an iterable sequence of records to the stream."""
|
|
107
|
+
for record in records:
|
|
108
|
+
self.write_record(record)
|
|
109
|
+
|
|
110
|
+
def flush(self) -> None:
|
|
111
|
+
"""Flush the current buffered batch of records to disk."""
|
|
112
|
+
if self._batch:
|
|
113
|
+
self._writer.write_table(
|
|
114
|
+
self._pa.Table.from_pylist(self._batch, schema=self._schema)
|
|
115
|
+
)
|
|
116
|
+
self._batch = []
|
|
117
|
+
|
|
118
|
+
def close(self) -> int:
|
|
119
|
+
"""Flush pending batches, finalize file and atomically commit to destination."""
|
|
120
|
+
if self._closed:
|
|
121
|
+
return self._rows_written
|
|
122
|
+
try:
|
|
123
|
+
self.flush()
|
|
124
|
+
self._writer.__exit__(None, None, None)
|
|
125
|
+
os.replace(self._temp_path, self._destination)
|
|
126
|
+
self._closed = True
|
|
127
|
+
finally:
|
|
128
|
+
self._temp_path.unlink(missing_ok=True)
|
|
129
|
+
return self._rows_written
|
|
130
|
+
|
|
131
|
+
def __enter__(self) -> ParquetStreamWriter:
|
|
132
|
+
"""Enter context manager scope."""
|
|
133
|
+
return self
|
|
134
|
+
|
|
135
|
+
def __exit__(
|
|
136
|
+
self,
|
|
137
|
+
exc_type: type[BaseException] | None,
|
|
138
|
+
exc_val: BaseException | None,
|
|
139
|
+
exc_tb: Any,
|
|
140
|
+
) -> None:
|
|
141
|
+
"""Exit context manager, committing on success or removing temp file on error."""
|
|
142
|
+
try:
|
|
143
|
+
if exc_type is None:
|
|
144
|
+
self.close()
|
|
145
|
+
else:
|
|
146
|
+
self._writer.__exit__(exc_type, exc_val, exc_tb)
|
|
147
|
+
finally:
|
|
148
|
+
self._temp_path.unlink(missing_ok=True)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _convert_records(transactions: Iterable[Any]) -> list[dict[str, Any]]:
|
|
152
|
+
"""Convert an iterable of objects or dicts into a list of row dicts."""
|
|
153
|
+
records: list[dict[str, Any]] = []
|
|
154
|
+
for item in transactions:
|
|
155
|
+
if isinstance(item, dict):
|
|
156
|
+
records.append(item)
|
|
157
|
+
elif hasattr(item, "model_dump"):
|
|
158
|
+
records.append(item.model_dump())
|
|
159
|
+
elif hasattr(item, "to_dict"):
|
|
160
|
+
records.append(item.to_dict())
|
|
161
|
+
elif hasattr(item, "__dict__"):
|
|
162
|
+
records.append(
|
|
163
|
+
{
|
|
164
|
+
k: v
|
|
165
|
+
for k, v in item.__dict__.items()
|
|
166
|
+
if not k.startswith("_")
|
|
167
|
+
}
|
|
168
|
+
)
|
|
169
|
+
else:
|
|
170
|
+
records.append({"value": str(item)})
|
|
171
|
+
return records
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def export_parquet(
|
|
175
|
+
transactions: Iterable[Any] | pd.DataFrame,
|
|
176
|
+
output_path: str | Path | None = None,
|
|
177
|
+
compression: str = "snappy",
|
|
178
|
+
*,
|
|
179
|
+
redact_pii: bool = False,
|
|
180
|
+
) -> bytes:
|
|
181
|
+
"""Export statement transactions to Apache Parquet format.
|
|
182
|
+
|
|
183
|
+
Args:
|
|
184
|
+
transactions: Sequence of transactions, dictionaries, or a pandas DataFrame.
|
|
185
|
+
output_path: Optional file path to write the Parquet file to.
|
|
186
|
+
compression: Compression codec ('snappy', 'gzip', 'zstd', None).
|
|
187
|
+
redact_pii: Mask identities, narratives and provenance fields.
|
|
188
|
+
|
|
189
|
+
Returns:
|
|
190
|
+
Parquet file contents as bytes.
|
|
191
|
+
"""
|
|
192
|
+
validated_destination = (
|
|
193
|
+
InputValidator().validate_output_file_path(str(output_path))
|
|
194
|
+
if output_path is not None
|
|
195
|
+
else None
|
|
196
|
+
)
|
|
197
|
+
if isinstance(transactions, pd.DataFrame):
|
|
198
|
+
df = transactions.copy()
|
|
199
|
+
else:
|
|
200
|
+
records = _convert_records(transactions)
|
|
201
|
+
df = pd.DataFrame(records)
|
|
202
|
+
|
|
203
|
+
if redact_pii:
|
|
204
|
+
df = pd.DataFrame(
|
|
205
|
+
[redact_record(row) for row in df.to_dict("records")],
|
|
206
|
+
columns=df.columns,
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
buf = io.BytesIO()
|
|
210
|
+
try:
|
|
211
|
+
df.to_parquet(buf, compression=compression, engine="pyarrow")
|
|
212
|
+
except ImportError as exc:
|
|
213
|
+
raise ImportError(
|
|
214
|
+
"Apache Parquet export requires 'pyarrow' or 'fastparquet'. "
|
|
215
|
+
"Install with: pip install 'bankstatementparser[parquet]' or pip install pyarrow"
|
|
216
|
+
) from exc
|
|
217
|
+
parquet_bytes = buf.getvalue()
|
|
218
|
+
|
|
219
|
+
if validated_destination is not None:
|
|
220
|
+
validated_destination.write_bytes(parquet_bytes)
|
|
221
|
+
|
|
222
|
+
return parquet_bytes
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def export_parquet_stream(
|
|
226
|
+
records: Iterable[Mapping[str, Any]],
|
|
227
|
+
output_path: str | Path,
|
|
228
|
+
*,
|
|
229
|
+
schema: Any,
|
|
230
|
+
batch_size: int = 10000,
|
|
231
|
+
compression: str = "snappy",
|
|
232
|
+
redact_pii: bool = False,
|
|
233
|
+
) -> int:
|
|
234
|
+
"""Write bounded Parquet batches atomically using an explicit Arrow schema.
|
|
235
|
+
|
|
236
|
+
Supply a ``pyarrow.Schema`` covering all input keys, with adequate Decimal
|
|
237
|
+
precision/scale and string types for fields being redacted. Missing nullable
|
|
238
|
+
fields become null. Unknown fields, missing required fields and values that
|
|
239
|
+
cannot be represented fail without replacing an existing destination.
|
|
240
|
+
Returns the written row count; unlike export_parquet, no file bytes are
|
|
241
|
+
retained. Memory depends on batch size and individual record sizes.
|
|
242
|
+
"""
|
|
243
|
+
with ParquetStreamWriter(
|
|
244
|
+
output_path,
|
|
245
|
+
schema=schema,
|
|
246
|
+
batch_size=batch_size,
|
|
247
|
+
compression=compression,
|
|
248
|
+
redact_pii=redact_pii,
|
|
249
|
+
) as writer:
|
|
250
|
+
writer.write_batch(records)
|
|
251
|
+
return writer.rows_written
|
{bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/input_validator.py
RENAMED
|
@@ -27,10 +27,40 @@ from pathlib import Path
|
|
|
27
27
|
from typing import ClassVar, Optional, Union
|
|
28
28
|
|
|
29
29
|
from .exceptions import ValidationError
|
|
30
|
+
from .xml_validator import (
|
|
31
|
+
check_binary_signatures,
|
|
32
|
+
check_xml_indicators,
|
|
33
|
+
sanitize_filename,
|
|
34
|
+
sanitize_source_name,
|
|
35
|
+
validate_payload_size,
|
|
36
|
+
validate_xml_bytes_format,
|
|
37
|
+
validate_xml_content,
|
|
38
|
+
)
|
|
30
39
|
|
|
31
40
|
logger = logging.getLogger(__name__)
|
|
32
41
|
|
|
33
|
-
__all__ = [
|
|
42
|
+
__all__ = [
|
|
43
|
+
"InputValidator",
|
|
44
|
+
"ValidationError",
|
|
45
|
+
"sanitize_filename",
|
|
46
|
+
"sanitize_source_name",
|
|
47
|
+
"validate_xml_content",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
_DANGEROUS_UNICODE: tuple[str, ...] = (
|
|
51
|
+
"\u0000",
|
|
52
|
+
"\u202e",
|
|
53
|
+
"\u202d",
|
|
54
|
+
"\u200f",
|
|
55
|
+
"\u200e",
|
|
56
|
+
"\u2066",
|
|
57
|
+
"\u2067",
|
|
58
|
+
"\u2068",
|
|
59
|
+
"\u2069",
|
|
60
|
+
"\u202a",
|
|
61
|
+
"\u202b",
|
|
62
|
+
"\u202c",
|
|
63
|
+
)
|
|
34
64
|
|
|
35
65
|
|
|
36
66
|
class InputValidator:
|
|
@@ -275,33 +305,7 @@ class InputValidator:
|
|
|
275
305
|
Raises:
|
|
276
306
|
ValidationError: If source_name is not a string.
|
|
277
307
|
"""
|
|
278
|
-
|
|
279
|
-
return default
|
|
280
|
-
|
|
281
|
-
if not isinstance(source_name, str):
|
|
282
|
-
raise ValidationError("Source name must be a string")
|
|
283
|
-
|
|
284
|
-
cleaned = []
|
|
285
|
-
for char in source_name.strip():
|
|
286
|
-
if ord(char) < 32 or char in {
|
|
287
|
-
"\u202e",
|
|
288
|
-
"\u202d",
|
|
289
|
-
"\u200f",
|
|
290
|
-
"\u200e",
|
|
291
|
-
"\u2066",
|
|
292
|
-
"\u2067",
|
|
293
|
-
"\u2068",
|
|
294
|
-
"\u2069",
|
|
295
|
-
"\u202a",
|
|
296
|
-
"\u202b",
|
|
297
|
-
"\u202c",
|
|
298
|
-
}:
|
|
299
|
-
cleaned.append("?")
|
|
300
|
-
else:
|
|
301
|
-
cleaned.append(char)
|
|
302
|
-
|
|
303
|
-
sanitized = "".join(cleaned)[:255]
|
|
304
|
-
return sanitized or default
|
|
308
|
+
return sanitize_source_name(source_name, default)
|
|
305
309
|
|
|
306
310
|
def validate_xml_content(
|
|
307
311
|
self,
|
|
@@ -321,44 +325,17 @@ class InputValidator:
|
|
|
321
325
|
Raises:
|
|
322
326
|
ValidationError: If content is unsafe, empty, oversized, or not XML.
|
|
323
327
|
"""
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
elif isinstance(xml_content, bytes):
|
|
331
|
-
if not xml_content.strip():
|
|
332
|
-
raise ValidationError("XML content cannot be empty")
|
|
333
|
-
xml_bytes = xml_content
|
|
334
|
-
else:
|
|
335
|
-
raise ValidationError(
|
|
336
|
-
"XML content must be provided as a string or bytes"
|
|
337
|
-
)
|
|
338
|
-
|
|
339
|
-
self._validate_bytes_size(xml_bytes)
|
|
340
|
-
self._validate_xml_bytes_format(xml_bytes, safe_source_name)
|
|
341
|
-
|
|
342
|
-
return xml_bytes, safe_source_name
|
|
328
|
+
return validate_xml_content(
|
|
329
|
+
xml_content,
|
|
330
|
+
max_file_size=self.max_file_size,
|
|
331
|
+
source_name=source_name,
|
|
332
|
+
log=logger,
|
|
333
|
+
)
|
|
343
334
|
|
|
344
335
|
def _check_dangerous_patterns(self, file_path: str) -> None:
|
|
345
336
|
"""Check for dangerous patterns in file path."""
|
|
346
337
|
# Check for dangerous Unicode characters (null bytes, BiDi overrides, etc.)
|
|
347
|
-
|
|
348
|
-
"\u0000", # Null byte
|
|
349
|
-
"\u202e", # Right-to-left override
|
|
350
|
-
"\u202d", # Left-to-right override
|
|
351
|
-
"\u200f", # Right-to-left mark
|
|
352
|
-
"\u200e", # Left-to-right mark
|
|
353
|
-
"\u2066", # Left-to-right isolate
|
|
354
|
-
"\u2067", # Right-to-left isolate
|
|
355
|
-
"\u2068", # First strong isolate
|
|
356
|
-
"\u2069", # Pop directional isolate
|
|
357
|
-
"\u202a", # Left-to-right embedding
|
|
358
|
-
"\u202b", # Right-to-left embedding
|
|
359
|
-
"\u202c", # Pop directional formatting
|
|
360
|
-
]
|
|
361
|
-
for char in dangerous_unicode:
|
|
338
|
+
for char in _DANGEROUS_UNICODE:
|
|
362
339
|
if char in file_path:
|
|
363
340
|
raise ValidationError(
|
|
364
341
|
"Potentially dangerous path pattern detected"
|
|
@@ -431,19 +408,7 @@ class InputValidator:
|
|
|
431
408
|
|
|
432
409
|
def _validate_bytes_size(self, data: bytes) -> None:
|
|
433
410
|
"""Validate in-memory payload size constraints."""
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
if payload_size < self.MIN_FILE_SIZE_BYTES:
|
|
437
|
-
raise ValidationError(
|
|
438
|
-
f"XML content is too small ({payload_size} bytes). Minimum: {self.MIN_FILE_SIZE_BYTES} bytes"
|
|
439
|
-
)
|
|
440
|
-
|
|
441
|
-
if payload_size > self.max_file_size:
|
|
442
|
-
size_mb = payload_size / (1024 * 1024)
|
|
443
|
-
max_mb = self.max_file_size / (1024 * 1024)
|
|
444
|
-
raise ValidationError(
|
|
445
|
-
f"XML content is too large ({size_mb:.1f}MB). Maximum allowed: {max_mb:.1f}MB"
|
|
446
|
-
)
|
|
411
|
+
validate_payload_size(data, self.max_file_size)
|
|
447
412
|
|
|
448
413
|
def _read_file_header(self, path: Path) -> bytes:
|
|
449
414
|
"""Read the initial bytes of a file for format detection."""
|
|
@@ -463,39 +428,11 @@ class InputValidator:
|
|
|
463
428
|
|
|
464
429
|
def _check_binary_signatures(self, header: bytes, path: Path) -> None:
|
|
465
430
|
"""Check for known binary signatures that indicate invalid text formats."""
|
|
466
|
-
|
|
467
|
-
b"\x89PNG",
|
|
468
|
-
b"GIF8",
|
|
469
|
-
b"\xff\xd8\xff",
|
|
470
|
-
b"PK",
|
|
471
|
-
b"\x7fELF",
|
|
472
|
-
b"MZ",
|
|
473
|
-
b"\x00\x00\x01\x00",
|
|
474
|
-
b"%PDF",
|
|
475
|
-
]
|
|
476
|
-
for sig in binary_signatures:
|
|
477
|
-
if header[: len(sig)] == sig:
|
|
478
|
-
raise ValidationError(
|
|
479
|
-
f"File appears to contain binary data, expected XML: {path}"
|
|
480
|
-
)
|
|
431
|
+
check_binary_signatures(header, path)
|
|
481
432
|
|
|
482
433
|
def _check_xml_indicators(self, header: bytes, path: Path) -> None:
|
|
483
434
|
"""Check for XML declaration or common root namespace indicators."""
|
|
484
|
-
|
|
485
|
-
xml_indicators = [
|
|
486
|
-
"<?xml",
|
|
487
|
-
"<document",
|
|
488
|
-
"xmlns",
|
|
489
|
-
"camt.053",
|
|
490
|
-
"pain.001",
|
|
491
|
-
"iso:std:iso:20022",
|
|
492
|
-
]
|
|
493
|
-
if not any(ind in header_str for ind in xml_indicators):
|
|
494
|
-
if any(c < 32 and c not in (9, 10, 13) for c in header[:100]):
|
|
495
|
-
raise ValidationError(
|
|
496
|
-
f"File appears to contain binary data, expected XML: {path}"
|
|
497
|
-
)
|
|
498
|
-
logger.warning(f"File may not be a valid XML document: {path}")
|
|
435
|
+
check_xml_indicators(header, path, logger)
|
|
499
436
|
|
|
500
437
|
def _validate_input_format(self, path: Path) -> None:
|
|
501
438
|
"""Validate input file format by checking file content.
|
|
@@ -544,56 +481,7 @@ class InputValidator:
|
|
|
544
481
|
Raises:
|
|
545
482
|
ValidationError: If content is not plausible UTF-8 XML.
|
|
546
483
|
"""
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
binary_signatures = [
|
|
550
|
-
b"\x89PNG",
|
|
551
|
-
b"GIF8",
|
|
552
|
-
b"\xff\xd8\xff",
|
|
553
|
-
b"PK",
|
|
554
|
-
b"\x7fELF",
|
|
555
|
-
b"MZ",
|
|
556
|
-
b"\x00\x00\x01\x00",
|
|
557
|
-
b"%PDF",
|
|
558
|
-
]
|
|
559
|
-
for sig in binary_signatures:
|
|
560
|
-
if header[: len(sig)] == sig:
|
|
561
|
-
raise ValidationError(
|
|
562
|
-
f"XML content appears to contain binary data, expected XML: {source_name}"
|
|
563
|
-
)
|
|
564
|
-
|
|
565
|
-
try:
|
|
566
|
-
header_str = header.decode("utf-8")
|
|
567
|
-
except UnicodeDecodeError as exc:
|
|
568
|
-
raise ValidationError(
|
|
569
|
-
f"XML content encoding is not valid UTF-8: {source_name}"
|
|
570
|
-
) from exc
|
|
571
|
-
|
|
572
|
-
header_lower = header_str.lower()
|
|
573
|
-
xml_indicators = [
|
|
574
|
-
"<?xml",
|
|
575
|
-
"<document",
|
|
576
|
-
"xmlns",
|
|
577
|
-
"camt.",
|
|
578
|
-
"iso:std:iso:20022",
|
|
579
|
-
]
|
|
580
|
-
|
|
581
|
-
has_xml_indicator = any(
|
|
582
|
-
indicator in header_lower for indicator in xml_indicators
|
|
583
|
-
)
|
|
584
|
-
|
|
585
|
-
if not has_xml_indicator:
|
|
586
|
-
if any(
|
|
587
|
-
byte < 32 and byte not in (9, 10, 13) for byte in header[:100]
|
|
588
|
-
):
|
|
589
|
-
raise ValidationError(
|
|
590
|
-
f"XML content appears to contain binary data, expected XML: {source_name}"
|
|
591
|
-
)
|
|
592
|
-
|
|
593
|
-
logger.warning(
|
|
594
|
-
"XML content may not be a valid XML document: %s",
|
|
595
|
-
source_name,
|
|
596
|
-
)
|
|
484
|
+
validate_xml_bytes_format(xml_bytes, source_name, logger)
|
|
597
485
|
|
|
598
486
|
def get_safe_filename(self, filename: str) -> str:
|
|
599
487
|
"""Generate a safe filename by removing/replacing dangerous characters.
|
|
@@ -604,19 +492,4 @@ class InputValidator:
|
|
|
604
492
|
Returns:
|
|
605
493
|
str: Safe filename.
|
|
606
494
|
"""
|
|
607
|
-
|
|
608
|
-
safe_chars = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "_", filename)
|
|
609
|
-
|
|
610
|
-
# Remove leading/trailing dots and spaces
|
|
611
|
-
safe_chars = safe_chars.strip(". ")
|
|
612
|
-
|
|
613
|
-
# Ensure filename is not empty
|
|
614
|
-
if not safe_chars:
|
|
615
|
-
safe_chars = "unnamed_file"
|
|
616
|
-
|
|
617
|
-
# Truncate if too long (keeping extension)
|
|
618
|
-
if len(safe_chars) > 255:
|
|
619
|
-
name, ext = os.path.splitext(safe_chars)
|
|
620
|
-
safe_chars = name[: 255 - len(ext)] + ext
|
|
621
|
-
|
|
622
|
-
return safe_chars
|
|
495
|
+
return sanitize_filename(filename)
|
{bankstatementparser-0.0.26 → bankstatementparser-0.0.27}/bankstatementparser/parsers/__init__.py
RENAMED
|
@@ -31,6 +31,7 @@ from .common import (
|
|
|
31
31
|
)
|
|
32
32
|
from .csv_parser import CsvStatementParser
|
|
33
33
|
from .mt940 import Mt940Parser
|
|
34
|
+
from .mt942 import Mt942Parser
|
|
34
35
|
from .ofx import OfxParser, QfxParser
|
|
35
36
|
|
|
36
37
|
__all__ = [
|
|
@@ -38,6 +39,7 @@ __all__ = [
|
|
|
38
39
|
"Bai2Parser",
|
|
39
40
|
"CsvStatementParser",
|
|
40
41
|
"Mt940Parser",
|
|
42
|
+
"Mt942Parser",
|
|
41
43
|
"OfxParser",
|
|
42
44
|
"QfxParser",
|
|
43
45
|
"_amount_or_zero",
|