statement-etl-ph 0.1.0__py3-none-any.whl

statement_etl_ph/__init__.py

@@ -0,0 +1,11 @@
+"""Read Philippine bank and e-wallet statements into plain transactions.
+
+    from statement_etl_ph.banks import get_bank
+
+    statement = get_bank("gcash").parser("statement.pdf", "password")
+    assert statement.reconcile().ok
+    for txn in statement.transactions:
+        print(txn.value_date, txn.signed_amount, txn.merchant)
+"""
+
+__version__ = "0.1.0"

statement_etl_ph/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

statement_etl_ph/banks/__init__.py

@@ -0,0 +1,35 @@
+"""The banks and e-wallets this package can read, one module each.
+
+The registry is a fixed list on purpose. Nothing is discovered from other
+installed packages: a statement is financial data, and which code reads it
+should be decided here, not by whatever else happens to be installed.
+"""
+
+from __future__ import annotations
+
+from .base import BankDefinition
+from .gcash import BANK as GCASH
+from .gotyme import BANK as GOTYME
+from .grab import BANK as GRAB
+
+__all__ = ["BankDefinition", "available", "get_bank", "register"]
+
+_REGISTRY: dict[str, BankDefinition] = {bank.name: bank for bank in (GCASH, GOTYME, GRAB)}
+
+
+def register(bank: BankDefinition) -> None:
+    """Add a bank from your own code, e.g. while developing a new parser."""
+    _REGISTRY[bank.name] = bank
+
+
+def available() -> list[str]:
+    return sorted(_REGISTRY)
+
+
+def get_bank(name: str) -> BankDefinition:
+    try:
+        return _REGISTRY[name.lower()]
+    except KeyError:
+        raise KeyError(
+            f"no bank called {name!r}. Available: {', '.join(available())}"
+        ) from None

statement_etl_ph/banks/base.py

@@ -0,0 +1,50 @@
+"""What a bank is to this package: how to find its mail and how to read it."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Callable
+
+from ..models import Statement
+
+# (path, password) -> Statement. A provider whose statement is the email body
+# rather than a PDF keeps the same shape and ignores the password.
+Parser = Callable[..., Statement]
+
+
+@dataclass(frozen=True)
+class BankDefinition:
+    """Everything that differs between banks, in one place.
+
+    Adding a bank means writing one module in this package that defines a
+    parser and a ``BANK`` of this type, then listing it in ``banks/__init__``.
+    """
+
+    name: str
+    # How the provider is named to a person: the Account column of a Money
+    # Manager import, a memo, a report.
+    display_name: str
+    parser: Parser
+    # Gmail search for the statement email. The processed label and date
+    # window are added at fetch time, not written here.
+    gmail_query: str
+    # The domains the provider DKIM-signs its mail with. A message no one of
+    # them signed is refused, whatever its From header says.
+    dkim_domains: tuple[str, ...]
+    # The environment variable holding the PDF password. Empty for a provider
+    # whose statement is not encrypted.
+    password_env: str = ""
+    # "pdf" for an attachment; "email_body" when the email *is* the receipt.
+    content: str = "pdf"
+    # Filename fragments that pick the right PDF when a message carries
+    # several, tried in order. A pattern that matches nothing fails the
+    # message rather than falling back to the first PDF.
+    attachment_patterns: tuple[str, ...] = field(default_factory=tuple)
+
+    @property
+    def reads_body(self) -> bool:
+        return self.content == "email_body"
+
+    @property
+    def processed_label(self) -> str:
+        return f"statement-etl-ph/{self.name}"

statement_etl_ph/banks/gcash.py

@@ -0,0 +1,234 @@
+"""GCash: decrypt the emailed transaction-history PDF and extract its table.
+
+GCash renders the statement as a borderless table, so ``extract_table()``
+guesses badly at it. What *is* reliable is geometry: the Debit, Credit and
+Balance figures are right-aligned under their headers, always at the same x
+coordinate. So we read the header row once to learn where the money columns
+live, then assign every numeric token on every row to the nearest column.
+
+That distinction matters. A text-only parse of ``59.00   3343.68`` cannot
+tell a debit from a credit, and getting it backwards would flip the sign of
+the transaction wherever it ends up.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from datetime import date, datetime
+from pathlib import Path
+from typing import Optional
+
+import pdfplumber
+
+from ..models import Statement, StatementParseError, Transaction
+from .base import BankDefinition
+from ..pdf_geometry import AMOUNT_RE, assign_money_columns, cluster_rows, find_money_columns
+
+logger = logging.getLogger(__name__)
+
+_PERIOD_RE = re.compile(r"(\d{4}-\d{2}-\d{2})\s+to\s+(\d{4}-\d{2}-\d{2})")
+# A GCash reference number: a long digit run, distinct from an amount.
+_REFERENCE_RE = re.compile(r"^\d{9,}$")
+# Same, but rescued from inside a token that picked up neighbouring glyphs.
+_DIGIT_RUN_RE = re.compile(r"\d{9,}")
+
+# Rows are single lines that never wrap, so grouping words a fraction of a
+# point apart in top position is enough to call them the same row.
+_ROW_CLUSTER_TOLERANCE = 3.0
+# How close (in points) a number's right edge must be to a header's right edge
+# before we accept the match. The three columns are ~50pt apart, so 25 is a
+# safe half-width.
+_COLUMN_TOLERANCE = 25.0
+
+_MONEY_HEADERS = ("debit", "credit", "balance")
+
+
+def _group_words_into_lines(words) -> list[list[dict]]:
+    """Cluster extracted words into visual rows, in left-to-right reading order.
+
+    No GCash row wraps, so sorting purely by ``x0`` is always reading order.
+    """
+    return cluster_rows(words, _ROW_CLUSTER_TOLERANCE, sort_key=lambda w: (w["x0"],))
+
+
+def _find_reference_zone(lines: list[list[dict]]) -> Optional[tuple[float, float]]:
+    """Locate the horizontal span of the 'Reference No.' column.
+
+    Position is a better signal than pattern matching here. A long merchant
+    description can contain digit runs of its own, and reference numbers are
+    not always the first such run on the row; the column they sit in is the
+    thing that actually identifies them.
+    """
+    for line in lines:
+        for index, word in enumerate(line):
+            if word["text"].strip().lower() != "reference":
+                continue
+            right = word["x1"]
+            if index + 1 < len(line) and line[index + 1]["text"].strip().lower() == "no.":
+                right = line[index + 1]["x1"]
+            return (word["x0"] - 30.0, right + 15.0)
+    return None
+
+
+def _split_reference_and_description(
+    words: list[dict], zone: Optional[tuple[float, float]]
+) -> tuple[str, str]:
+    """Separate the reference number from the description on one row."""
+    reference = ""
+    description_parts: list[str] = []
+
+    for word in words:
+        token = word["text"].strip()
+        if not token or AMOUNT_RE.match(token):
+            continue
+
+        in_zone = zone is not None and word["x1"] >= zone[0] and word["x0"] <= zone[1]
+        if not reference and (in_zone or zone is None):
+            match = _REFERENCE_RE.match(token) or _DIGIT_RUN_RE.search(token)
+            if match:
+                reference = match.group(0)
+                continue
+
+        # Anything to the right of the reference column is a stray artefact,
+        # not part of the merchant name.
+        if zone is not None and word["x0"] > zone[1]:
+            continue
+        if not reference:
+            description_parts.append(token)
+
+    return reference, " ".join(description_parts).strip()
+
+
+def _row_text(line: list[dict]) -> str:
+    return " ".join(w["text"] for w in line)
+
+
+def _parse_datetime(date_token: str, time_token: str) -> Optional[datetime]:
+    cleaned = time_token.upper().replace(".", "").replace(" ", "")
+    for fmt in ("%Y-%m-%d %I:%M%p", "%Y-%m-%d %H:%M"):
+        try:
+            return datetime.strptime(f"{date_token} {cleaned}", fmt)
+        except ValueError:
+            continue
+    return None
+
+
+def parse_statement(pdf_path: str | Path, password: str) -> Statement:
+    """Decrypt and parse a GCash statement PDF into a :class:`Statement`."""
+    pdf_path = Path(pdf_path)
+    transactions: list[Transaction] = []
+    starting_balance = ending_balance = total_debit = total_credit = None
+    period_start: Optional[date] = None
+    period_end: Optional[date] = None
+
+    with pdfplumber.open(str(pdf_path), password=password) as pdf:
+        if not pdf.pages:
+            raise StatementParseError("PDF has no pages")
+
+        columns: Optional[dict[str, float]] = None
+        reference_zone: Optional[tuple[float, float]] = None
+
+        for page_no, page in enumerate(pdf.pages, start=1):
+            lines = _group_words_into_lines(page.extract_words(keep_blank_chars=False))
+
+            if columns is None:
+                try:
+                    columns = find_money_columns(lines, _MONEY_HEADERS)
+                    reference_zone = _find_reference_zone(lines)
+                    logger.debug(
+                        "money columns at %s, reference zone %s", columns, reference_zone
+                    )
+                except StatementParseError:
+                    if page_no == len(pdf.pages):
+                        raise
+                    continue  # header might be on a later page
+
+            for line in lines:
+                text = _row_text(line)
+
+                if period_start is None:
+                    period_match = _PERIOD_RE.search(text)
+                    if period_match:
+                        period_start = date.fromisoformat(period_match.group(1))
+                        period_end = date.fromisoformat(period_match.group(2))
+                        continue
+
+                amounts = assign_money_columns(line, columns, _COLUMN_TOLERANCE)
+                upper = text.upper()
+
+                if "STARTING BALANCE" in upper:
+                    starting_balance = amounts.get("balance")
+                    continue
+                if "ENDING BALANCE" in upper:
+                    ending_balance = amounts.get("balance")
+                    continue
+                if "TOTAL DEBIT" in upper:
+                    total_debit = amounts.get("debit")
+                    continue
+                if "TOTAL CREDIT" in upper:
+                    total_credit = amounts.get("credit")
+                    continue
+
+                # A transaction row starts with a date and a time.
+                if len(line) < 3:
+                    continue
+                stamp = _parse_datetime(line[0]["text"], f"{line[1]['text']} {line[2]['text']}")
+                if stamp is None:
+                    continue
+
+                reference, description = _split_reference_and_description(
+                    line[3:], reference_zone
+                )
+                if not description:
+                    logger.warning("row at %s has no description; keeping it anyway", stamp)
+
+                if not amounts.get("debit") and not amounts.get("credit"):
+                    logger.warning(
+                        "skipping row with no debit or credit amount: %s", text[:90]
+                    )
+                    continue
+
+                transactions.append(
+                    Transaction(
+                        posted_at=stamp,
+                        description=description,
+                        reference_no=reference,
+                        debit=amounts.get("debit"),
+                        credit=amounts.get("credit"),
+                        balance=amounts.get("balance"),
+                    )
+                )
+
+    if not transactions:
+        raise StatementParseError("no transaction rows found in the statement")
+
+    statement = Statement(
+        period_start=period_start,
+        period_end=period_end,
+        starting_balance=starting_balance,
+        ending_balance=ending_balance,
+        total_debit=total_debit,
+        total_credit=total_credit,
+        transactions=transactions,
+    )
+    logger.info(
+        "parsed %d transactions for %s to %s",
+        len(transactions),
+        period_start,
+        period_end,
+    )
+    return statement
+
+
+BANK = BankDefinition(
+    name="gcash",
+    display_name="GCash",
+    parser=parse_statement,
+    gmail_query=(
+        'from:noreply-transhistory@gcash.com subject:"Your GCash Transaction History" '
+        "has:attachment"
+    ),
+    dkim_domains=("gcash.com",),
+    password_env="GCASH_PDF_PASSWORD",
+)

statement_etl_ph/banks/gotyme.py

@@ -0,0 +1,262 @@
+"""GoTyme Bank: decrypt a statement of account and extract its table.
+
+The technique is the one :mod:`statement_etl_ph.banks.gcash` uses, from
+:mod:`statement_etl_ph.pdf_geometry`: find the money column
+headers, then assign each numeric token to a column by its right edge. What
+does not transfer is the vocabulary, and three differences are load-bearing.
+
+**Both money columns are always printed.** GoTyme writes ``0.00`` in the
+column a row does not use, so a row reading ``0.00  174.25`` is a debit of
+174.25 and *not* a credit of zero. A zero is therefore read as "not this
+side" and stored as ``None``; storing ``Decimal("0")`` would make
+``signed_amount`` return zero for every transaction on the statement.
+
+**Descriptions wrap around the data line.** A long payee is printed on the
+line *above* the date and amounts, and continues on the line *below* them:
+
+    Transfer from J. D. SANTOS Maya Philippines, Inc (PayMaya Wallet), ...
+    08 Aug 2026                                    3,000.00   0.00  3,445.95
+    ••••••••0000
+
+Those three lines are one logical row. Geometry separates them from ordinary
+rows without guesswork: wrapped lines sit 3.9pt apart and start in the
+Details column, while consecutive rows are 15.7pt apart and start in the Date
+column. So lines are clustered vertically with a threshold between the two,
+and the description is every non-date, non-amount word in the cluster read
+top-to-bottom, left-to-right.
+
+**There are no reference numbers.** GCash prints one per row, which makes an
+ideal idempotency key. GoTyme prints none, and the usual fallback of
+``amount:date`` is not good enough here: this statement contains two GRAB
+payments of 149.00 on the same day, which would collide and silently import
+as one. The running balance disambiguates them — it is a ledger, so no two
+rows share one — so each row gets a synthetic reference derived from date,
+description, both amounts and that balance. It is stable across re-runs of
+the same statement, which is what idempotency needs.
+
+What it is *not* stable across is a **reissued** statement. A backdated
+adjustment shifts every running balance after it, so every row from that
+point onward gets a new key and re-imports alongside the originals. That is
+accepted rather than solved: dropping the balance trades visible duplicates
+for silent data loss, keying on row position breaks on any insertion, and
+hashing date and description alone collides on genuinely repeated purchases —
+one real month held eight identical ``MOVE IT`` rows. Duplicates you can see
+and delete beat a transaction that never arrives.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from datetime import date, datetime
+from decimal import Decimal
+from pathlib import Path
+from typing import Optional
+
+import pdfplumber
+
+from ..models import Statement, StatementParseError, Transaction
+from .base import BankDefinition
+from ..pdf_geometry import (
+    AMOUNT_RE,
+    assign_money_columns,
+    cluster_rows,
+    find_money_columns,
+    to_decimal,
+)
+
+logger = logging.getLogger(__name__)
+
+_DATE_RE = re.compile(r"^(\d{1,2}) ([A-Za-z]{3}) (\d{4})$")
+_PERIOD_RE = re.compile(
+    r"Period:\s*(\d{1,2} [A-Za-z]{3} \d{4})\s*[-–]\s*(\d{1,2} [A-Za-z]{3} \d{4})"
+)
+_SUMMARY_RES = {
+    "starting_balance": re.compile(r"Opening balance\s+[A-Z]{3}\s+([\d,]+\.\d{2})"),
+    "ending_balance": re.compile(r"Closing balance\s+[A-Z]{3}\s+([\d,]+\.\d{2})"),
+    "total_credit": re.compile(r"Total Credit\s+[A-Z]{3}\s+([\d,]+\.\d{2})"),
+    "total_debit": re.compile(r"Total Debit\s+[A-Z]{3}\s+([\d,]+\.\d{2})"),
+}
+
+# Wrapped continuation lines sit 3.9pt from their data line; separate rows sit
+# 15.7pt apart. Anything between the two separates rows without splitting one.
+_ROW_CLUSTER_TOLERANCE = 10.0
+# Credits and Debits are ~37pt apart, so half that is a safe column tolerance.
+_COLUMN_TOLERANCE = 15.0
+_MONEY_HEADERS = ("credits", "debits", "balance")
+
+
+def _cluster_rows(words) -> list[list[dict]]:
+    """Group words into logical rows, keeping a wrapped description together.
+
+    A row can wrap above and below its data line, so words must read back
+    top-to-bottom before left-to-right, or a wrapped line would interleave
+    with the data line by horizontal position instead of following it.
+    """
+    return cluster_rows(words, _ROW_CLUSTER_TOLERANCE, sort_key=lambda w: (w["top"], w["x0"]))
+
+
+def _synthetic_reference(
+    when: date,
+    description: str,
+    credit: Optional[Decimal],
+    debit: Optional[Decimal],
+    balance: Optional[Decimal],
+) -> str:
+    """A stable per-row key, standing in for the reference GoTyme omits.
+
+    Includes the running balance deliberately: it is what distinguishes two
+    otherwise identical payments on the same day.
+    """
+    payload = "|".join(
+        [
+            when.isoformat(),
+            " ".join(description.split()).lower(),
+            str(credit if credit is not None else ""),
+            str(debit if debit is not None else ""),
+            str(balance if balance is not None else ""),
+        ]
+    )
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def _parse_row(
+    row: list[dict], columns: dict[str, float]
+) -> Optional[Transaction]:
+    amounts = [w for w in row if AMOUNT_RE.match(w["text"])]
+    if len(amounts) < 3:
+        return None
+
+    # The date is three consecutive words ("08", "Aug", "2026") in the Date
+    # column. Find them rather than assuming they start the cluster — on a
+    # wrapped row the description is printed above them.
+    texts = [w["text"] for w in row]
+    when: Optional[date] = None
+    date_indices: set[int] = set()
+    for i in range(len(texts) - 2):
+        candidate = " ".join(texts[i : i + 3])
+        if _DATE_RE.match(candidate):
+            try:
+                when = datetime.strptime(candidate, "%d %b %Y").date()
+            except ValueError:
+                continue
+            date_indices = {i, i + 1, i + 2}
+            break
+    if when is None:
+        return None
+
+    values = assign_money_columns(row, columns, _COLUMN_TOLERANCE)
+    credit = values.get("credits")
+    debit = values.get("debits")
+    balance = values.get("balance")
+
+    # GoTyme prints 0.00 in the column a row does not use. Treating that as a
+    # real zero would make every transaction worth nothing.
+    if credit == 0:
+        credit = None
+    if debit == 0:
+        debit = None
+    if credit is None and debit is None:
+        return None
+
+    amount_ids = {id(w) for w in amounts}
+    description = " ".join(
+        w["text"]
+        for i, w in enumerate(row)
+        if i not in date_indices and id(w) not in amount_ids
+    )
+    description = " ".join(description.split())
+    if not description:
+        return None
+
+    return Transaction(
+        posted_at=datetime(when.year, when.month, when.day),
+        description=description,
+        reference_no=_synthetic_reference(when, description, credit, debit, balance),
+        debit=debit,
+        credit=credit,
+        balance=balance,
+    )
+
+
+def _parse_summary(text: str) -> dict[str, Optional[Decimal]]:
+    found: dict[str, Optional[Decimal]] = {}
+    for field, pattern in _SUMMARY_RES.items():
+        match = pattern.search(text)
+        found[field] = to_decimal(match.group(1)) if match else None
+    return found
+
+
+def _parse_period(text: str) -> tuple[Optional[date], Optional[date]]:
+    match = _PERIOD_RE.search(text)
+    if not match:
+        return None, None
+    try:
+        start = datetime.strptime(match.group(1), "%d %b %Y").date()
+        end = datetime.strptime(match.group(2), "%d %b %Y").date()
+    except ValueError:
+        return None, None
+    return start, end
+
+
+def parse_statement(pdf_path: str | Path, password: str) -> Statement:
+    """Parse a password-protected GoTyme statement into a :class:`Statement`."""
+    transactions: list[Transaction] = []
+    full_text: list[str] = []
+
+    with pdfplumber.open(str(pdf_path), password=password) as pdf:
+        all_rows: list[list[dict]] = []
+        for page in pdf.pages:
+            full_text.append(page.extract_text() or "")
+            all_rows.extend(_cluster_rows(page.extract_words()))
+
+        # GoTyme wraps the last header as "Running" over "Balance", so the
+        # three labels only land in one row because rows are grouped
+        # vertically first.
+        columns = find_money_columns(all_rows, _MONEY_HEADERS)
+        logger.debug("money columns at %s", columns)
+        for row in all_rows:
+            transaction = _parse_row(row, columns)
+            if transaction is not None:
+                transactions.append(transaction)
+
+    if not transactions:
+        raise StatementParseError(
+            "no transaction rows found; the statement may be empty or its "
+            "layout may have changed"
+        )
+
+    text = "\n".join(full_text)
+    start, end = _parse_period(text)
+    summary = _parse_summary(text)
+
+    logger.info(
+        "parsed %d transactions for %s to %s",
+        len(transactions),
+        start or "?",
+        end or "?",
+    )
+    return Statement(
+        period_start=start,
+        period_end=end,
+        starting_balance=summary["starting_balance"],
+        ending_balance=summary["ending_balance"],
+        total_debit=summary["total_debit"],
+        total_credit=summary["total_credit"],
+        transactions=transactions,
+    )
+
+
+BANK = BankDefinition(
+    name="gotyme",
+    display_name="GoTyme",
+    parser=parse_statement,
+    gmail_query='from:no-reply@gotyme.com.ph subject:"Statement of Account" has:attachment',
+    # Not amazonses.com, though GoTyme sends through it: SES signs for anyone.
+    dkim_domains=("gotyme.com.ph",),
+    password_env="GOTYME_PDF_PASSWORD",
+    # One email carries the deposit and the savings statements. The config
+    # file can pick the other one.
+    attachment_patterns=("EverydayDepositAccount",),
+)