openbooks-statements 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4 @@
1
+ """Open Books statement providers. Registered with openbooks-core through entry points; the model code is in openbooks_statements.roles."""
2
+ from ._version import __version__
3
+
4
+ __all__ = ["__version__"]
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
@@ -0,0 +1,28 @@
1
+ """Where the model assets are, with no torch import.
2
+
3
+ Order: the directory named by OPENBOOKS_MODELS_DIR, then the assets packaged inside this distribution (data/), then the
4
+ repo's models/ directory when running from a source checkout. The calibration is bound to the weights by sha256 and the
5
+ loader refuses a mismatch, so a stray or older file elsewhere can never be used silently."""
6
+ from __future__ import annotations
7
+
8
+ import os
9
+ from importlib import resources
10
+ from pathlib import Path
11
+
12
+ WEIGHTS_NAME, CALIBRATION_NAME = "roles_head.pt", "roles_calibration.json"
13
+
14
+
15
+ def models_dir() -> Path:
16
+ env = os.environ.get("OPENBOOKS_MODELS_DIR")
17
+ if env:
18
+ return Path(env)
19
+ packaged = Path(str(resources.files("openbooks_statements").joinpath("data")))
20
+ if (packaged / WEIGHTS_NAME).exists():
21
+ return packaged
22
+ for parent in Path(__file__).resolve().parents:
23
+ if (parent / "models" / WEIGHTS_NAME).exists():
24
+ return parent / "models"
25
+ return packaged # absent: load() reports that there are no weights
26
+
27
+
28
+ MODELS_DIR = models_dir()
@@ -0,0 +1,47 @@
1
+ """Calibration and abstention for the typed heads at inference: temperature-scaled probabilities and split-conformal sets.
2
+
3
+ A token is only trusted when its conformal set holds exactly one class. Anything else abstains, which the pipeline reports
4
+ as LOW_CONFIDENCE. Coverage (the true class is in the set) is guaranteed at 1 - alpha only if calibration and test data are
5
+ exchangeable; on layouts unlike the calibration set it can fail. The numbers (temperature, qhat, alpha) ship in
6
+ roles_calibration.json, bound to the exact weights by hash. Fitting them is not part of this package.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import math
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+
15
+ import numpy as np
16
+
17
+
18
+ def softmax(z: np.ndarray) -> np.ndarray:
19
+ z = z - z.max(axis=-1, keepdims=True)
20
+ e = np.exp(z)
21
+ return e / e.sum(axis=-1, keepdims=True)
22
+
23
+
24
+ def prediction_set(probs: np.ndarray, qhat: float) -> list[int]:
25
+ return [i for i, p in enumerate(probs) if 1.0 - p <= qhat]
26
+
27
+
28
+ @dataclass(frozen=True)
29
+ class Calibrator:
30
+ temperature: float = 1.0
31
+ qhat: float = float("inf")
32
+ alpha: float = 0.05
33
+
34
+ def probs(self, logits: np.ndarray) -> np.ndarray:
35
+ return softmax(np.asarray(logits, dtype=np.float64) / self.temperature)
36
+
37
+ def sets(self, logits: np.ndarray) -> list[list[int]]:
38
+ return [prediction_set(p, self.qhat) for p in self.probs(logits)]
39
+
40
+ def save(self, path: Path) -> None:
41
+ path.write_text(json.dumps({"temperature": self.temperature, "qhat": None if math.isinf(self.qhat) else self.qhat,
42
+ "alpha": self.alpha}), encoding="utf-8")
43
+
44
+ @classmethod
45
+ def load(cls, path: Path) -> "Calibrator":
46
+ d = json.loads(path.read_text(encoding="utf-8"))
47
+ return cls(d["temperature"], float("inf") if d["qhat"] is None else d["qhat"], d["alpha"])
@@ -0,0 +1 @@
1
+ {"role": {"temperature": 2.019380728929039, "qhat": 0.9976497888565063, "alpha": 0.001}, "vtype": {"temperature": 1.0769541104862101, "qhat": 0.9351925849914551, "alpha": 0.001}, "weights_sha256": "d101e39610b74fa3195399a9e4aed89dad2916c2ac874284b08a38b6a0d31056"}
Binary file
@@ -0,0 +1,13 @@
1
+ {
2
+ "model": "RoleNet (typed heads: column role + value type) over a LayoutLM-style encoder",
3
+ "parameters": 18878602,
4
+ "weights_mb_int8_at_rest": 19.04,
5
+ "weights_sha256": "d101e39610b74fa3195399a9e4aed89dad2916c2ac874284b08a38b6a0d31056",
6
+ "initialisation": "random; no pretrained weights",
7
+ "training_data": "synthetic statements only; no third-party dataset and no real client document",
8
+ "calibration": {
9
+ "method": "temperature scaling + split conformal",
10
+ "alpha": 0.001
11
+ },
12
+ "role": "cross-check only: the heads can add refusals, never cause a release"
13
+ }
@@ -0,0 +1,151 @@
1
+ """Layout encoder shared by the typed heads: hashed word and shape embeddings plus 2D box embeddings.
2
+
3
+ Release-one size: about 19M parameters (budget 25M; int8 at rest is about 1 MB per million). Trained from scratch on
4
+ synthetic statements only. Nothing here loads a pretrained checkpoint.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import re
9
+ import zlib
10
+
11
+ import torch
12
+ from torch import nn
13
+
14
+ NB, NS, MAXLEN, D, LAYERS, HEADS = 16384, 512, 512, 384, 6, 6
15
+ MONEY = re.compile(r"^\(?-?\d[\d,]*\.\d{2}\)?(?:Dr|Cr|DR|CR)?$")
16
+ NOT_MONEY, MAX_RANK, MAX_N = 5, 4, 6
17
+
18
+
19
+ def column_heads(boxes: list[list[int]], hashes: list[int], money: list[bool]) -> list[list[int]]:
20
+ """For each amount token, the hashed words (up to 3) of the nearest non-amount line above it in the same x-range.
21
+
22
+ On a statement that is the column header ('Withdrawal Amt.', 'Credit'); the model learns what those words mean. Nothing
23
+ here knows any header vocabulary, so a new header wording is just a new hash, not a rule to maintain.
24
+ """
25
+ import numpy as np
26
+
27
+ heads = [[0, 0, 0] for _ in boxes]
28
+ if not any(money):
29
+ return heads
30
+ b, mon = np.asarray(boxes), np.asarray(money)
31
+ for i in np.nonzero(mon)[0]:
32
+ ok = ~mon & (np.minimum(b[:, 2], b[i, 2]) - np.maximum(b[:, 0], b[i, 0]) > 0) & (b[:, 3] <= b[i, 1])
33
+ idx = np.nonzero(ok)[0]
34
+ if not len(idx):
35
+ continue
36
+ near = idx[np.argmax(b[idx, 3])]
37
+ h = max(4, b[near, 3] - b[near, 1])
38
+ line = idx[np.abs((b[idx, 1] + b[idx, 3]) / 2 - (b[near, 1] + b[near, 3]) / 2) <= 0.6 * h]
39
+ ids = [hashes[k] for k in line[np.argsort(b[line, 0])][:3]]
40
+ heads[i] = ids + [0] * (3 - len(ids))
41
+ return heads
42
+
43
+
44
+ def row_features(boxes: list[list[int]], money: list[bool]) -> list[list[int]]:
45
+ """Per token (rank from the right among the amount tokens of its row, number of amounts in the row).
46
+
47
+ A statement row reads description ... [debit] [credit] balance, so the balance is always rank 0. This is geometry plus
48
+ the surface money test only: no header words and no rule-based column assignment, so it stays independent of the extractor.
49
+ """
50
+ feats = [[NOT_MONEY, 0] for _ in boxes]
51
+ order = sorted((i for i, m in enumerate(money) if m), key=lambda i: boxes[i][1] + boxes[i][3])
52
+ rows: list[list[int]] = []
53
+ for i in order:
54
+ cy, h = (boxes[i][1] + boxes[i][3]) / 2, max(4, boxes[i][3] - boxes[i][1])
55
+ if rows and abs(cy - sum((boxes[k][1] + boxes[k][3]) / 2 for k in rows[-1]) / len(rows[-1])) <= 0.6 * h:
56
+ rows[-1].append(i)
57
+ else:
58
+ rows.append([i])
59
+ for row in rows:
60
+ for rank, i in enumerate(sorted(row, key=lambda k: -boxes[k][2])):
61
+ feats[i] = [min(rank, MAX_RANK), min(len(row), MAX_N)]
62
+ return feats
63
+
64
+
65
+ def h_word(w: str) -> int:
66
+ return zlib.crc32(w.lower().encode()) % (NB - 2) + 2
67
+
68
+
69
+ def h_shape(w: str) -> int:
70
+ s = "".join("d" if c.isdigit() else "A" if c.isupper() else "a" if c.isalpha() else c for c in w)
71
+ return zlib.crc32((re.sub(r"(.)\1+", r"\1", s) + str(min(len(w), 12))).encode()) % (NS - 1) + 1
72
+
73
+
74
+ class Enc(nn.Module):
75
+ """Token + shape + 2D box embeddings (LayoutLM-style), LAYERS x width D transformer."""
76
+
77
+ def __init__(self, d: int = D, layers: int = LAYERS, heads: int = HEADS):
78
+ super().__init__()
79
+ self.word, self.shape = nn.Embedding(NB, d, padding_idx=0), nn.Embedding(NS, d, padding_idx=0)
80
+ self.xe, self.ye, self.we, self.he = (nn.Embedding(1001, d) for _ in range(4))
81
+ self.pos = nn.Embedding(MAXLEN, d)
82
+ self.rank, self.nmon = nn.Embedding(NOT_MONEY + 1, d), nn.Embedding(MAX_N + 1, d)
83
+ layer = nn.TransformerEncoderLayer(d, heads, 4 * d, 0.1, activation="gelu", batch_first=True, norm_first=True)
84
+ self.tf = nn.TransformerEncoder(layer, layers, enable_nested_tensor=False)
85
+ self.norm = nn.LayerNorm(d)
86
+
87
+ def forward(self, W, S, X, pad, R, H):
88
+ pos = torch.arange(W.shape[1], device=W.device)
89
+ wd, ht = (X[..., 2] - X[..., 0]).clamp(0, 1000), (X[..., 3] - X[..., 1]).clamp(0, 1000)
90
+ x = (self.word(W) + self.shape(S) + self.xe(X[..., 0]) + self.xe(X[..., 2]) + self.ye(X[..., 1])
91
+ + self.ye(X[..., 3]) + self.we(wd) + self.he(ht) + self.pos(pos) + self.rank(R[..., 0]) + self.nmon(R[..., 1])
92
+ + self.word(H).sum(2))
93
+ return self.norm(self.tf(x, src_key_padding_mask=pad))
94
+
95
+
96
+ class RoleNet(nn.Module):
97
+ """Encoder plus two typed heads: the column role of a token and the value type it holds."""
98
+
99
+ def __init__(self, n_roles: int, n_types: int):
100
+ super().__init__()
101
+ self.enc = Enc()
102
+ self.role, self.vtype = nn.Linear(D, n_roles), nn.Linear(D, n_types)
103
+
104
+ def forward(self, W, S, X, pad, R, H):
105
+ h = self.enc(W, S, X, pad, R, H)
106
+ return self.role(h), self.vtype(h)
107
+
108
+
109
+ def windows(words) -> list[list[int]]:
110
+ """Index windows of at most MAXLEN tokens per page, in reading order (page, y, x)."""
111
+ out: list[list[int]] = []
112
+ by_page: dict[int, list[int]] = {}
113
+ for i, w in enumerate(words):
114
+ by_page.setdefault(w.page, []).append(i)
115
+ for idx in by_page.values():
116
+ idx.sort(key=lambda i: (round(words[i].cy / 4), words[i].x0))
117
+ out += [idx[k:k + MAXLEN] for k in range(0, len(idx), MAXLEN)]
118
+ return out
119
+
120
+
121
+ def tensors(words, idx: list[int], dev):
122
+ """Collate one window into (W, S, X, pad, R) batch-of-one tensors, boxes scaled to 0..1000 by page size."""
123
+ ws = [words[i] for i in idx]
124
+ clamp = lambda v: min(1000, max(0, int(v))) # noqa: E731
125
+ boxes = [[clamp(w.x0 * 1000 / w.pw), clamp(w.y0 * 1000 / w.ph), clamp(w.x1 * 1000 / w.pw), clamp(w.y1 * 1000 / w.ph)] for w in ws]
126
+ W = torch.tensor([[h_word(w.text) for w in ws]], device=dev)
127
+ S = torch.tensor([[h_shape(w.text) for w in ws]], device=dev)
128
+ money = [bool(MONEY.match(w.text)) for w in ws]
129
+ R = torch.tensor([row_features(boxes, money)], device=dev)
130
+ H = torch.tensor([column_heads(boxes, [h_word(w.text) for w in ws], money)], device=dev)
131
+ return W, S, torch.tensor([boxes], device=dev), torch.zeros(1, len(ws), dtype=torch.bool, device=dev), R, H
132
+
133
+
134
+ def n_params(net: nn.Module) -> int:
135
+ return sum(p.numel() for p in net.parameters())
136
+
137
+
138
+ def quantize_state(state: dict, min_numel: int = 4096) -> dict:
139
+ """int8 at rest: per-row symmetric scales for every large 2D weight (embeddings and linears), fp16 for the rest."""
140
+ out = {}
141
+ for k, v in state.items():
142
+ if v.ndim == 2 and v.numel() >= min_numel:
143
+ scale = (v.abs().amax(dim=1, keepdim=True) / 127).clamp(min=1e-8)
144
+ out[k] = {"q": torch.round(v / scale).clamp(-127, 127).to(torch.int8), "s": scale.squeeze(1).half()}
145
+ else:
146
+ out[k] = v.half()
147
+ return out
148
+
149
+
150
+ def dequantize_state(blob: dict) -> dict:
151
+ return {k: (v["q"].float() * v["s"].float().unsqueeze(1) if isinstance(v, dict) else v.float()) for k, v in blob.items()}
@@ -0,0 +1,17 @@
1
+ """PDF and image front end as a registry provider: extract() reads the document, identity() names the model that cross-checked it."""
2
+ from __future__ import annotations
3
+
4
+ from .pdf_front import extract_document
5
+
6
+
7
+ class _DocumentFrontend:
8
+ def extract(self, path, params):
9
+ return extract_document(path, params)
10
+
11
+ def identity(self, params):
12
+ from .roles import RoleModel
13
+
14
+ return getattr(RoleModel.load(), "identity", None)
15
+
16
+
17
+ FRONTEND = _DocumentFrontend()
@@ -0,0 +1,10 @@
1
+ """Gate for statements read by OCR. Lived in extra_gate next to the tax rules; it belongs to the statements family."""
2
+ from __future__ import annotations
3
+
4
+
5
+ def ocr_dates(b, tx):
6
+ """Dates must be in order on an OCR'd statement (guards OCR date misreads)."""
7
+ if b.get("source_method") == "ocr":
8
+ ds = [t["date"] for t in tx]
9
+ ok = sum(1 for a, c in zip(ds, ds[1:]) if a <= c)
10
+ yield f"{b['id']}: dates are in order (guards OCR date misreads)", f"{max(len(ds) - 1, 0)} steps", f"{ok} steps", False
@@ -0,0 +1,3 @@
1
+ """Label sets of the typed heads. No torch import here, so anything that needs only the names stays light."""
2
+ ROLES = ["other", "date", "desc", "debit", "credit", "balance"]
3
+ VTYPES = ["text", "date", "money", "code"]
@@ -0,0 +1,185 @@
1
+ """Bank statement parsers: bank CSV, MT940 and OFX. Moved verbatim from pipeline.py; become openbooks-statements.
2
+ """
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from datetime import datetime
7
+ from decimal import Decimal
8
+
9
+ from openbooks_core.registry import bank_cols
10
+ from openbooks_core.builders import (D, body_rows as _body, cell_getter as _cell, currency_of as _currency, empty_data as _empty_data,
11
+ finish_batch as _finish, money_fields as _money_fields, settlement_batch as _settlement,
12
+ split_total as _split_total, stated_totals as _stated_totals, statement_doc)
13
+ from openbooks_core.errors import Refusal
14
+ from openbooks_core.evidence import evidence
15
+ from openbooks_core.normalise import date_order, fmt, money, to_date, to_dec
16
+ from openbooks_core.params import Params
17
+
18
+
19
+ def parse_bank_csv(args, p: Params):
20
+ rows, hi, hs = args
21
+ idx = bank_cols(hs)
22
+ body = _body(rows, hi, hs)
23
+ cur = _currency(hs, [r for _, r in body], p)
24
+ if p.opening is None and not p.allow_derived_opening:
25
+ raise Refusal("PARSE_ERROR", "statement has no stated opening balance; supply --opening")
26
+ order = date_order([r[idx["date"]] for _, r in body], p.date_order, default="dmy", where="date column")
27
+ recs = []
28
+ for n, row in body:
29
+ def g(role: str) -> str:
30
+ return row[idx[role]].strip() if role in idx else ""
31
+
32
+ deb = to_dec(g("debit"), f"row {n} debit", blank_zero=True)
33
+ cre = to_dec(g("credit"), f"row {n} credit", blank_zero=True)
34
+ if (deb and cre) or not (deb or cre) or deb < 0 or cre < 0:
35
+ raise Refusal("PARSE_ERROR", f"row {n}: need exactly one positive debit or credit")
36
+ recs.append({
37
+ "row": n, "date": to_date(g("date"), f"row {n}", order), "dir": "credit" if cre else "debit",
38
+ "amt": cre or deb, "amt_raw": g("credit") if cre else g("debit"), "date_raw": g("date"),
39
+ "bal": to_dec(g("balance"), f"row {n} balance", dr_cr=True), "bal_raw": g("balance"),
40
+ "desc": g("desc"), "ref": g("ref") or None,
41
+ })
42
+ return statement_doc(recs, cur, p.account, p.opening)
43
+
44
+
45
+ _TAG = re.compile(r"^:(\d{2}[A-Z]?):(.*)$")
46
+ _MT_BAL = re.compile(r"([CD])(\d{6})([A-Z]{3})(\d+,\d*)")
47
+ _MT_TXN = re.compile(r"(\d{6})(\d{4})?(R?[CD])([A-Z])?(\d+,\d*)N([A-Z0-9]{3})(.*)", re.S)
48
+
49
+
50
+ def _mt_fields(text: str) -> list[list[str]]:
51
+ out: list[list[str]] = []
52
+ for line in text.replace("\r\n", "\n").replace("\r", "\n").split("\n"):
53
+ m = _TAG.match(line)
54
+ if m:
55
+ out.append([m.group(1), m.group(2)])
56
+ elif out and line.strip() not in ("", "-", "-}") and not line.startswith(("{", "}")):
57
+ out[-1][1] += "\n" + line
58
+ return out
59
+
60
+
61
+ def _mt_date(yymmdd: str, where: str) -> str:
62
+ try:
63
+ return datetime.strptime(yymmdd, "%y%m%d").date().isoformat()
64
+ except ValueError:
65
+ raise Refusal("PARSE_ERROR", f"{where}: unreadable date {yymmdd!r}") from None
66
+
67
+
68
+ def _mt_bal(v: str, where: str):
69
+ m = _MT_BAL.fullmatch(v.strip())
70
+ if not m:
71
+ raise Refusal("PARSE_ERROR", f"{where}: unreadable balance {v!r}")
72
+ sign = 1 if m.group(1) == "C" else -1
73
+ return sign * Decimal(m.group(4).replace(",", ".")), _mt_date(m.group(2), where), m.group(3)
74
+
75
+
76
+ def parse_mt940(text: str, p: Params):
77
+ stmts: list[dict] = []
78
+ for tag, val in _mt_fields(text):
79
+ if tag == "20":
80
+ stmts.append({"txns": [], "open": None, "close": None})
81
+ elif not stmts:
82
+ continue
83
+ elif tag == "25":
84
+ stmts[-1]["acct"] = val.strip()
85
+ elif tag in ("60F", "60M") and stmts[-1]["open"] is None:
86
+ stmts[-1]["open"] = (_mt_bal(val, "opening"), val.strip())
87
+ elif tag in ("62F", "62M"):
88
+ stmts[-1]["close"] = (_mt_bal(val, "closing"), val.strip())
89
+ elif tag == "61":
90
+ m = _MT_TXN.fullmatch(val.split("\n")[0].strip())
91
+ if not m:
92
+ raise Refusal("PARSE_ERROR", f":61: unreadable {val!r}")
93
+ stmts[-1]["txns"].append({"m": m, "raw": val.split("\n")[0].strip(), "info": ""})
94
+ elif tag == "86" and stmts[-1]["txns"]:
95
+ stmts[-1]["txns"][-1]["info"] = " ".join(x.strip() for x in val.split("\n"))
96
+ data, ev, accts = _empty_data(), {}, {}
97
+ for si, s in enumerate(stmts, 1):
98
+ if not (s.get("acct") and s["open"] and s["close"]):
99
+ raise Refusal("PARSE_ERROR", f"statement {si}: missing :25:, :60F: or :62F:")
100
+ (ob, od, ocur), oraw = s["open"]
101
+ (cb, cd, ccur), craw = s["close"]
102
+ if ocur != ccur:
103
+ raise Refusal("PARSE_ERROR", f"statement {si}: currency changes inside statement")
104
+ aid = accts.setdefault(s["acct"], f"acct-{len(accts) + 1}")
105
+ if not any(a["id"] == aid for a in data["accounts"]):
106
+ data["accounts"].append({"id": aid, "number": s["acct"], "currency": ocur})
107
+ bid = f"batch-{si}"
108
+ data["batches"].append({"id": bid, "kind": "statement", "account_id": aid, "currency": ocur,
109
+ "period_start": od, "period_end": cd, "opening": money(ob, ocur),
110
+ "closing": money(cb, ocur), "opening_derived": False})
111
+ for f, raw in (("opening", oraw), ("closing", craw), ("period_start", oraw), ("period_end", craw)):
112
+ evidence(ev, bid, f, None, raw)
113
+ for k, t in enumerate(s["txns"], 1):
114
+ m = t["m"]
115
+ tid = f"txn-{si}-{k:05d}"
116
+ credit = m.group(3) in ("C", "RD")
117
+ ref = m.group(7).split("//")[0].strip()
118
+ data["transactions"].append({
119
+ "id": tid, "batch_id": bid, "account_id": aid, "date": _mt_date(m.group(1), tid),
120
+ "direction": "credit" if credit else "debit",
121
+ "amount": money(Decimal(m.group(5).replace(",", ".")), ocur),
122
+ "description": t["info"], "reference": None if ref in ("", "NONREF") else ref,
123
+ })
124
+ evidence(ev, tid, "date", None, t["raw"])
125
+ evidence(ev, tid, "amount", None, t["raw"])
126
+ if not data["transactions"]:
127
+ raise Refusal("PARSE_ERROR", "no :61: transactions")
128
+ return data, ev
129
+
130
+
131
+ def _ofx_tag(block: str, name: str) -> str | None:
132
+ m = re.search(rf"<{name}>([^<\r\n]*)", block, re.I)
133
+ return m.group(1).strip() if m else None
134
+
135
+
136
+ def parse_ofx(text: str, p: Params):
137
+ chunks = [c for c in re.split(r"(?=<STMTRS>|<CCSTMTRS>)", text, flags=re.I) if re.match(r"<(CC)?STMTRS>", c, re.I)]
138
+ if not chunks:
139
+ raise Refusal("PARSE_ERROR", "no statement response in OFX")
140
+ if p.opening is None:
141
+ raise Refusal("PARSE_ERROR", "OFX carries no opening balance; supply --opening")
142
+ if len(chunks) > 1:
143
+ raise Refusal("PARSE_ERROR", "multi-statement OFX with one --opening is ambiguous")
144
+ c = chunks[0]
145
+ cur = (_ofx_tag(c, "CURDEF") or p.currency or "").upper()
146
+ acct = _ofx_tag(c, "ACCTID")
147
+ bal = _ofx_tag(c, "BALAMT")
148
+ if not (cur and bal):
149
+ raise Refusal("PARSE_ERROR", "OFX missing CURDEF or LEDGERBAL")
150
+ blocks = re.findall(r"<STMTTRN>(.*?)(?=</STMTTRN>|<STMTTRN>|</BANKTRANLIST>|</CCSTMTTRNRS>|$)", c, re.S | re.I)
151
+ data, ev = _empty_data(), {}
152
+ data["accounts"].append({"id": "acct-1", "number": acct, "currency": cur})
153
+ seen, dates = set(), []
154
+ for k, b in enumerate(blocks, 1):
155
+ fit, amt_raw, dt = _ofx_tag(b, "FITID"), _ofx_tag(b, "TRNAMT"), _ofx_tag(b, "DTPOSTED")
156
+ if not (fit and amt_raw and dt) or fit in seen:
157
+ raise Refusal("PARSE_ERROR", f"transaction {k}: missing field or duplicate FITID")
158
+ seen.add(fit)
159
+ amt = to_dec(amt_raw, f"txn {k} amount")
160
+ if amt == 0:
161
+ raise Refusal("PARSE_ERROR", f"txn {k}: zero amount")
162
+ tid = f"txn-{k:05d}"
163
+ if not re.match(r"\d{8}", dt):
164
+ raise Refusal("PARSE_ERROR", f"{tid}: unreadable DTPOSTED {dt!r}")
165
+ d = datetime.strptime(dt[:8], "%Y%m%d").date().isoformat()
166
+ dates.append(d)
167
+ data["transactions"].append({
168
+ "id": tid, "batch_id": "batch-1", "account_id": "acct-1", "date": d,
169
+ "direction": "credit" if amt > 0 else "debit", "amount": money(abs(amt), cur),
170
+ "description": _ofx_tag(b, "NAME") or _ofx_tag(b, "MEMO") or "", "reference": fit,
171
+ })
172
+ evidence(ev, tid, "date", None, dt)
173
+ evidence(ev, tid, "amount", None, amt_raw)
174
+ if not dates:
175
+ raise Refusal("PARSE_ERROR", "no transactions")
176
+ data["batches"].append({
177
+ "id": "batch-1", "kind": "statement", "account_id": "acct-1", "currency": cur,
178
+ "period_start": min(dates), "period_end": max(dates), "opening": money(p.opening, cur),
179
+ "closing": money(to_dec(bal, "LEDGERBAL"), cur), "opening_derived": False,
180
+ })
181
+ evidence(ev, "batch-1", "opening", None, "supplied by caller")
182
+ evidence(ev, "batch-1", "closing", None, bal)
183
+ evidence(ev, "batch-1", "period_start", None, min(dates))
184
+ evidence(ev, "batch-1", "period_end", None, max(dates))
185
+ return data, ev
@@ -0,0 +1,164 @@
1
+ """Reading PDFs and images with permissively licensed libraries: pypdfium2 (Apache-2.0 / BSD-3, wrapping Google's PDFium) for
2
+ PDFs and Pillow for images. Nothing here is AGPL.
3
+
4
+ The rest of the engine sees only two things per page: its words as (text, x0, y0, x1, y1) in points with the origin at the
5
+ top-left (y grows downward), and an RGB render for OCR. The word rules are the ones the engine was built and measured with:
6
+ words break at whitespace and wherever one text object ends and the next begins (a table cell is its own text object), and text
7
+ that is not horizontal (watermarks, rotated stamps) is ignored.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import ctypes
12
+ import io
13
+ import math
14
+ from collections.abc import Iterator
15
+ from pathlib import Path
16
+ from typing import Any
17
+
18
+ from openbooks_core.errors import EngineFault, Refusal
19
+
20
+ HORIZONTAL_COS, HORIZONTAL_SIN = 0.9, 0.05 # a line within ~2.9 degrees of horizontal counts as horizontal
21
+
22
+
23
+ def _pdfium():
24
+ try:
25
+ import pypdfium2 as pdfium
26
+ except ImportError as e:
27
+ raise EngineFault("PDF and image statements need pypdfium2: pip install 'openbooks-statements[pdf]'") from e
28
+ return pdfium
29
+
30
+
31
+ class PdfPage:
32
+ def __init__(self, page: Any, pdfium: Any) -> None:
33
+ self._page, self._pdfium = page, pdfium
34
+ l, b, r, t = page.get_bbox()
35
+ self._left, self._top = l, t
36
+ self.width, self.height = r - l, t - b
37
+ self.rotation = page.get_rotation()
38
+
39
+ def words(self) -> list[tuple[str, float, float, float, float]]:
40
+ if self.rotation:
41
+ raise Refusal("UNSUPPORTED_INPUT", "a page with a rotated text layer is not supported; export it upright or scan it")
42
+ raw = self._pdfium.raw
43
+ tp = self._page.get_textpage()
44
+ try:
45
+ n = tp.count_chars()
46
+ ptr = tp.raw
47
+ out: list[tuple[str, float, float, float, float]] = []
48
+ cur: list[str] = []
49
+ box: list[float] = []
50
+ obj = None
51
+
52
+ def flush() -> None:
53
+ if cur:
54
+ out.append(("".join(cur), *box))
55
+ cur.clear()
56
+ box.clear()
57
+
58
+ high = 0 # pdfium reports UTF-16 code units: a character outside the BMP (an emoji in a payee name) is two of them
59
+ for i in range(n):
60
+ code = raw.FPDFText_GetUnicode(ptr, i)
61
+ if 0xDC00 <= code <= 0xDFFF and high:
62
+ code, high = 0x10000 + ((high - 0xD800) << 10) + (code - 0xDC00), 0
63
+ if cur:
64
+ cur[-1] = chr(code) # replaces the high half appended at the previous index
65
+ l, b, r, t = tp.get_charbox(i, loose=True)
66
+ box[0], box[1] = min(box[0], l - self._left), min(box[1], self._top - t)
67
+ box[2], box[3] = max(box[2], r - self._left), max(box[3], self._top - b)
68
+ continue
69
+ high = 0
70
+ if 0xD800 <= code <= 0xDBFF:
71
+ high = code
72
+ if 0xD800 <= code <= 0xDFFF:
73
+ code = 0xFFFD # kept only if a low half follows; a lone half is never passed on (it cannot be encoded)
74
+ ch = chr(code) if code else " "
75
+ if ch.isspace() or code == 0:
76
+ flush()
77
+ continue
78
+ angle = raw.FPDFText_GetCharAngle(ptr, i)
79
+ if math.cos(angle) < HORIZONTAL_COS or abs(math.sin(angle)) > HORIZONTAL_SIN:
80
+ flush()
81
+ continue
82
+ this_obj = ctypes.cast(raw.FPDFText_GetTextObject(ptr, i), ctypes.c_void_p).value
83
+ if cur and this_obj != obj:
84
+ flush()
85
+ obj = this_obj
86
+ l, b, r, t = tp.get_charbox(i, loose=True)
87
+ x0, x1 = l - self._left, r - self._left
88
+ y0, y1 = self._top - t, self._top - b
89
+ if not cur:
90
+ box.extend([x0, y0, x1, y1])
91
+ else:
92
+ box[0], box[1], box[2], box[3] = min(box[0], x0), min(box[1], y0), max(box[2], x1), max(box[3], y1)
93
+ cur.append(ch)
94
+ flush()
95
+ return out
96
+ finally:
97
+ tp.close()
98
+
99
+ def render_rgb(self, scale: float):
100
+ bitmap = self._page.render(scale=scale, rev_byteorder=True)
101
+ return bitmap.to_numpy()[:, :, :3].copy()
102
+
103
+
104
+ class ImagePage:
105
+ """A photo or scan as a one-page document. Its size in points follows the image's own DPI (72 if it states none)."""
106
+
107
+ def __init__(self, image: Any) -> None:
108
+ dpi = image.info.get("dpi", (72, 72))[0] or 72
109
+ self._image = image.convert("RGB")
110
+ self.width, self.height = image.width * 72 / dpi, image.height * 72 / dpi
111
+
112
+ def words(self) -> list[tuple[str, float, float, float, float]]:
113
+ return [] # an image has no text layer
114
+
115
+ def render_rgb(self, scale: float):
116
+ import numpy as np
117
+
118
+ w, h = max(1, round(self.width * scale)), max(1, round(self.height * scale))
119
+ img = self._image if (w, h) == self._image.size else self._image.resize((w, h))
120
+ return np.asarray(img, dtype=np.uint8).copy()
121
+
122
+
123
+ class Document:
124
+ def __init__(self, pages: Iterator[Any], count: int, close) -> None:
125
+ self._pages, self.page_count, self._close = pages, count, close
126
+
127
+ def pages(self) -> Iterator[Any]:
128
+ return self._pages
129
+
130
+ def close(self) -> None:
131
+ self._close()
132
+
133
+
134
+ def open_document(source: Path | bytes, *, image: bool = False) -> Document:
135
+ if image:
136
+ try:
137
+ from PIL import Image
138
+ except ImportError as e:
139
+ raise EngineFault("images need Pillow: pip install 'openbooks-statements[scan]'") from e
140
+ try:
141
+ img = Image.open(io.BytesIO(source) if isinstance(source, (bytes, bytearray)) else source)
142
+ img.load()
143
+ except Exception:
144
+ raise Refusal("UNSUPPORTED_INPUT", "unreadable document") from None
145
+ return Document(iter([ImagePage(img)]), 1, img.close)
146
+ pdfium = _pdfium()
147
+ try:
148
+ pdf = pdfium.PdfDocument(bytes(source) if isinstance(source, (bytes, bytearray)) else str(source))
149
+ except pdfium.PdfiumError as e:
150
+ if getattr(e, "err_code", None) == pdfium.raw.FPDF_ERR_PASSWORD:
151
+ raise Refusal("UNSUPPORTED_INPUT", "encrypted document") from None
152
+ raise Refusal("UNSUPPORTED_INPUT", "unreadable document") from None
153
+ except Exception:
154
+ raise Refusal("UNSUPPORTED_INPUT", "unreadable document") from None
155
+
156
+ def pages() -> Iterator[PdfPage]:
157
+ for i in range(len(pdf)):
158
+ page = pdf[i]
159
+ try:
160
+ yield PdfPage(page, pdfium)
161
+ finally:
162
+ page.close()
163
+
164
+ return Document(pages(), len(pdf), pdf.close)