bankstatementparser 0.0.4__py3-none-any.whl

bankstatementparser/parallel.py

@@ -0,0 +1,127 @@
+# Copyright (C) 2023 Sebastien Rousseau.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Parallel multi-file parsing for batch treasury workloads."""
+
+from __future__ import annotations
+
+import logging
+from concurrent.futures import (
+    ProcessPoolExecutor,
+    as_completed,
+)
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class FileResult:
+    """Result of parsing a single file."""
+
+    path: str
+    status: str
+    transactions: pd.DataFrame = field(
+        default_factory=pd.DataFrame
+    )
+    error: str = ""
+
+
+def _parse_single_file(
+    file_path: str,
+    format_name: str | None,
+) -> FileResult:
+    """Parse one file in a worker process."""
+    from .additional_parsers import (
+        create_parser,
+        detect_statement_format,
+    )
+
+    try:
+        fmt = format_name or detect_statement_format(file_path)
+        parser = create_parser(file_path, fmt)
+        df = parser.parse()
+        return FileResult(
+            path=file_path,
+            status="SUCCESS",
+            transactions=df,
+        )
+    except Exception as exc:
+        return FileResult(
+            path=file_path,
+            status="FAILED",
+            error=str(exc),
+        )
+
+
+def parse_files_parallel(
+    file_paths: list[str | Path],
+    *,
+    format_name: str | None = None,
+    max_workers: int | None = None,
+) -> list[FileResult]:
+    """Parse multiple statement files in parallel.
+
+    Uses process-based parallelism to bypass the GIL and
+    maximise throughput on multi-core systems. Each file is
+    parsed in its own worker process.
+
+    Args:
+        file_paths: Paths to statement files.
+        format_name: Force a specific format for all files.
+            When *None*, each file is auto-detected.
+        max_workers: Maximum worker processes. Defaults to
+            the number of CPU cores.
+
+    Returns:
+        List of ``FileResult`` in the same order as *file_paths*.
+    """
+    if not file_paths:
+        return []
+
+    str_paths = [str(p) for p in file_paths]
+
+    # Single file — skip process overhead
+    if len(str_paths) == 1:
+        return [_parse_single_file(str_paths[0], format_name)]
+
+    results: dict[str, FileResult] = {}
+
+    with ProcessPoolExecutor(
+        max_workers=max_workers
+    ) as executor:
+        future_to_path = {
+            executor.submit(
+                _parse_single_file, p, format_name
+            ): p
+            for p in str_paths
+        }
+        for future in as_completed(future_to_path):
+            path = future_to_path[future]
+            try:
+                results[path] = future.result()
+            except Exception as exc:  # pragma: no cover
+                results[path] = FileResult(
+                    path=path,
+                    status="FAILED",
+                    error=str(exc),
+                )
+
+    # Preserve original order
+    return [results[p] for p in str_paths]

bankstatementparser/record_types.py

@@ -0,0 +1,94 @@
+# Copyright (C) 2023 Sebastien Rousseau.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Typed records shared by parser implementations."""
+
+from __future__ import annotations
+
+from typing import TypedDict
+
+
+class BalanceRecord(TypedDict, total=False):
+    Amount: float
+    Currency: str | None
+    Code: str | None
+    Description: str | None
+    DrCr: str | None
+    Date: str | None
+    AccountId: str | None
+
+
+class TransactionRecord(TypedDict, total=False):
+    Amount: float
+    Currency: str | None
+    DrCr: str | None
+    Debtor: str | None
+    Creditor: str | None
+    Reference: str | None
+    ValDt: str | None
+    BookgDt: str | None
+    AccountId: str | None
+    DebtorAddress: str | None
+    CreditorAddress: str | None
+    date: str | None
+    description: str | None
+    amount: float | None
+    currency: str | None
+    balance: object
+    account_id: str | None
+    transaction_id: str | None
+    transaction_type: str | None
+
+
+class PaymentRecord(TypedDict, total=False):
+    MsgId: str | None
+    CreDtTm: str | None
+    NbOfTxs: str | None
+    InitgPty: str | None
+    PmtInfId: str | None
+    PmtMtd: str | None
+    CtrlSum: str | None
+    ReqdExctnDt: str | None
+    ChrgBr: str | None
+    DbtrNm: str | None
+    DbtrIBAN: str | None
+    DbtrBIC: str | None
+    EndToEndId: str | None
+    InstdAmt: str | None
+    Currency: str | None
+    CdtrBIC: str | None
+    CdtrNm: str | None
+    RmtInf: str | None
+
+
+class StatementStatsRecord(TypedDict, total=False):
+    StatementId: str | None
+    AccountId: str | None
+    StatementCreated: str | None
+    NumTransactions: int
+    NetAmount: float
+
+
+class SummaryRecord(TypedDict, total=False):
+    account_id: str | None
+    statement_date: str | None
+    transaction_count: int
+    total_amount: float
+    opening_balance: float | None
+    closing_balance: float | None
+    currency: str | None
+    message_id: str | None
+    initiating_party: str | None
+    error: str

bankstatementparser/transaction_deduplicator.py

@@ -0,0 +1,402 @@
+# Copyright (C) 2023 Sebastien Rousseau.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Deterministic transaction deduplication utilities."""
+
+from __future__ import annotations
+
+import hashlib
+from collections import defaultdict
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import date
+from difflib import SequenceMatcher
+
+import pandas as pd
+from pydantic import BaseModel, ConfigDict, Field
+
+from .transaction_models import Transaction
+
+
+def _days_between(left: date | None, right: date | None) -> int | None:
+    if left is None or right is None:
+        return None
+    return abs((left - right).days)
+
+
+def _description_similarity(
+    left: Transaction, right: Transaction
+) -> float:
+    if (
+        not left.normalized_description
+        or not right.normalized_description
+    ):
+        return 0.0
+    return SequenceMatcher(
+        None, left.normalized_description, right.normalized_description
+    ).ratio()
+
+
+class ExactDuplicateGroup(BaseModel):
+    """Transactions that collide on the deterministic primary hash."""
+
+    model_config = ConfigDict(frozen=True)
+
+    primary_hash: str
+    transactions: list[Transaction]
+
+
+class MatchGroup(BaseModel):
+    """A set of transactions requiring operator review."""
+
+    model_config = ConfigDict(frozen=True)
+
+    transactions: list[Transaction]
+    reason: str
+    confidence: float = Field(ge=0.0, le=1.0)
+    tier: str
+
+
+class DeduplicationResult(BaseModel):
+    """Explainable deduplication output."""
+
+    model_config = ConfigDict(frozen=True)
+
+    unique_transactions: list[Transaction]
+    exact_duplicates: list[ExactDuplicateGroup]
+    suspected_matches: list[MatchGroup]
+
+
+@dataclass(frozen=True)
+class _Candidate:
+    index: int
+    transaction: Transaction
+    primary_hash: str
+
+
+class Deduplicator:
+    """Deduplicate bank transactions with deterministic match tiers."""
+
+    def __init__(
+        self,
+        *,
+        value_date_window_days: int = 3,
+        description_similarity_threshold: float = 0.9,
+    ) -> None:
+        self.value_date_window_days = value_date_window_days
+        self.description_similarity_threshold = (
+            description_similarity_threshold
+        )
+
+    def primary_hash(self, transaction: Transaction) -> str:
+        """Return the stable primary hash for hard identity matching."""
+        material = "|".join(
+            [
+                transaction.account_id or "",
+                transaction.currency or "",
+                transaction.amount_key(),
+                transaction.booking_date.isoformat()
+                if transaction.booking_date is not None
+                else "",
+            ]
+        )
+        return hashlib.sha256(material.encode("utf-8")).hexdigest()
+
+    def normalize_transactions(
+        self,
+        transactions: Iterable[Transaction | dict[str, object]],
+        *,
+        source: str | None = None,
+    ) -> list[Transaction]:
+        """Normalize transaction inputs into deterministic models."""
+        normalized: list[Transaction] = []
+        for index, transaction in enumerate(transactions):
+            if isinstance(transaction, Transaction):
+                normalized.append(
+                    transaction.model_copy(
+                        update={
+                            "source": transaction.source or source,
+                            "source_index": (
+                                transaction.source_index
+                                if transaction.source_index is not None
+                                else index
+                            ),
+                        }
+                    )
+                )
+            else:
+                normalized.append(
+                    Transaction.from_record(
+                        dict(transaction),
+                        source=source,
+                        source_index=index,
+                    )
+                )
+        return normalized
+
+    def from_dataframe(
+        self, df: pd.DataFrame, *, source: str | None = None
+    ) -> list[Transaction]:
+        """Normalize parser DataFrame output into transaction models."""
+        return self.normalize_transactions(
+            df.to_dict("records"), source=source
+        )
+
+    def deduplicate(
+        self, transactions: Iterable[Transaction | dict[str, object]]
+    ) -> DeduplicationResult:
+        """Deduplicate transactions into unique, exact, and suspected sets."""
+        normalized = self.normalize_transactions(transactions)
+        candidates = [
+            _Candidate(
+                index=index,
+                transaction=transaction,
+                primary_hash=self.primary_hash(transaction),
+            )
+            for index, transaction in enumerate(normalized)
+        ]
+
+        exact_groups = self._find_exact_duplicates(candidates)
+        exact_indices = {
+            candidate.index
+            for bucket in self._candidate_groups_by_primary(
+                candidates
+            ).values()
+            if len(bucket) > 1
+            for candidate in bucket
+        }
+        suspected_groups = self._find_suspected_matches(
+            candidates,
+            excluded_indices=exact_indices,
+        )
+
+        suspected_indices = {
+            transaction.source_index
+            for group in suspected_groups
+            for transaction in group.transactions
+            if transaction.source_index is not None
+        }
+        unique_transactions = [
+            candidate.transaction
+            for candidate in candidates
+            if candidate.index not in exact_indices | suspected_indices
+        ]
+
+        return DeduplicationResult(
+            unique_transactions=unique_transactions,
+            exact_duplicates=exact_groups,
+            suspected_matches=suspected_groups,
+        )
+
+    def _candidate_groups_by_primary(
+        self, candidates: list[_Candidate]
+    ) -> dict[str, list[_Candidate]]:
+        groups: dict[str, list[_Candidate]] = defaultdict(list)
+        for candidate in candidates:
+            groups[candidate.primary_hash].append(candidate)
+        return groups
+
+    def _find_exact_duplicates(
+        self, candidates: list[_Candidate]
+    ) -> list[ExactDuplicateGroup]:
+        groups = self._candidate_groups_by_primary(candidates)
+        exact_groups = []
+        for primary_hash, bucket in sorted(groups.items()):
+            if len(bucket) < 2:
+                continue
+            transactions = sorted(
+                (candidate.transaction for candidate in bucket),
+                key=lambda item: (
+                    item.source_index or -1,
+                    item.reference or "",
+                ),
+            )
+            exact_groups.append(
+                ExactDuplicateGroup(
+                    primary_hash=primary_hash,
+                    transactions=transactions,
+                )
+            )
+        return exact_groups
+
+    def _find_suspected_matches(
+        self,
+        candidates: list[_Candidate],
+        *,
+        excluded_indices: set[int],
+    ) -> list[MatchGroup]:
+        probable_groups = self._find_probable_matches(candidates)
+        probable_indices = {
+            transaction.source_index
+            for group in probable_groups
+            for transaction in group.transactions
+            if transaction.source_index is not None
+        }
+        temporal_groups = self._find_temporal_matches(
+            [
+                candidate
+                for candidate in candidates
+                if candidate.index
+                not in excluded_indices | probable_indices
+            ]
+        )
+        return probable_groups + temporal_groups
+
+    def _find_probable_matches(
+        self, candidates: list[_Candidate]
+    ) -> list[MatchGroup]:
+        groups = []
+        for bucket in self._candidate_groups_by_primary(
+            candidates
+        ).values():
+            if len(bucket) < 2:
+                continue
+            similarities = []
+            for left_index, left in enumerate(bucket):
+                for right in bucket[left_index + 1 :]:
+                    similarity = _description_similarity(
+                        left.transaction, right.transaction
+                    )
+                    if (
+                        similarity
+                        >= self.description_similarity_threshold
+                        and left.transaction.normalized_description
+                        != right.transaction.normalized_description
+                    ):
+                        similarities.append(similarity)
+
+            if not similarities:
+                continue
+
+            groups.append(
+                MatchGroup(
+                    transactions=sorted(
+                        (candidate.transaction for candidate in bucket),
+                        key=lambda item: (
+                            item.source_index or -1,
+                            item.reference or "",
+                        ),
+                    ),
+                    reason=(
+                        "Primary hash collision with description similarity "
+                        f"{max(similarities):.2f}"
+                    ),
+                    confidence=min(0.99, max(similarities) + 0.05),
+                    tier="probable",
+                )
+            )
+        return groups
+
+    def _find_temporal_matches(
+        self, candidates: list[_Candidate]
+    ) -> list[MatchGroup]:
+        buckets: dict[tuple[str, str, str], list[_Candidate]] = (
+            defaultdict(list)
+        )
+        for candidate in candidates:
+            transaction = candidate.transaction
+            buckets[
+                (
+                    transaction.account_id or "",
+                    transaction.currency or "",
+                    transaction.amount_key(),
+                )
+            ].append(candidate)
+
+        groups = []
+        for bucket in buckets.values():
+            if len(bucket) < 2:
+                continue
+
+            bucket = sorted(
+                bucket,
+                key=lambda item: (
+                    item.transaction.value_date or date.min,
+                    item.index,
+                ),
+            )
+            component: list[_Candidate] = [bucket[0]]
+            component_similarities: list[float] = []
+            for candidate in bucket[1:]:
+                prev = component[-1]
+                day_delta = _days_between(
+                    prev.transaction.value_date,
+                    candidate.transaction.value_date,
+                )
+                if (
+                    day_delta is not None
+                    and day_delta <= self.value_date_window_days
+                    and prev.primary_hash != candidate.primary_hash
+                ):
+                    component.append(candidate)
+                    component_similarities.append(
+                        _description_similarity(
+                            prev.transaction, candidate.transaction
+                        )
+                    )
+                    continue
+
+                if len(component) > 1:
+                    groups.append(
+                        self._temporal_group(
+                            component, component_similarities
+                        )
+                    )
+                component = [candidate]
+                component_similarities = []
+
+            if len(component) > 1:
+                groups.append(
+                    self._temporal_group(
+                        component, component_similarities
+                    )
+                )
+
+        return groups
+
+    def _temporal_group(
+        self,
+        component: list[_Candidate],
+        similarities: list[float],
+    ) -> MatchGroup:
+        max_delta = max(
+            _days_between(
+                component[0].transaction.value_date,
+                component[-1].transaction.value_date,
+            )
+            or 0,
+            0,
+        )
+        max_similarity = max(similarities) if similarities else 0.0
+        confidence = min(0.95, 0.75 + (max_similarity * 0.2))
+        reason = (
+            f"Value date shift within {max_delta} day window"
+            if max_delta == 1
+            else f"Value date shift within {max_delta} day window"
+        )
+        if max_similarity > 0:
+            reason += f"; description similarity {max_similarity:.2f}"
+
+        return MatchGroup(
+            transactions=[
+                candidate.transaction
+                for candidate in sorted(
+                    component, key=lambda item: item.index
+                )
+            ],
+            reason=reason,
+            confidence=confidence,
+            tier="suspected",
+        )