openbooks-statements 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openbooks_statements/__init__.py +4 -0
- openbooks_statements/_version.py +1 -0
- openbooks_statements/assets.py +28 -0
- openbooks_statements/calibration.py +47 -0
- openbooks_statements/data/roles_calibration.json +1 -0
- openbooks_statements/data/roles_head.pt +0 -0
- openbooks_statements/data/roles_model_card.json +13 -0
- openbooks_statements/encoder.py +151 -0
- openbooks_statements/frontend.py +17 -0
- openbooks_statements/gates.py +10 -0
- openbooks_statements/labels.py +3 -0
- openbooks_statements/parsers.py +185 -0
- openbooks_statements/pdf_backend.py +164 -0
- openbooks_statements/pdf_front.py +478 -0
- openbooks_statements/py.typed +0 -0
- openbooks_statements/roles.py +168 -0
- openbooks_statements-0.1.0.dist-info/METADATA +45 -0
- openbooks_statements-0.1.0.dist-info/RECORD +21 -0
- openbooks_statements-0.1.0.dist-info/WHEEL +4 -0
- openbooks_statements-0.1.0.dist-info/entry_points.txt +12 -0
- openbooks_statements-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Where the model assets are, with no torch import.
|
|
2
|
+
|
|
3
|
+
Order: the directory named by OPENBOOKS_MODELS_DIR, then the assets packaged inside this distribution (data/), then the
|
|
4
|
+
repo's models/ directory when running from a source checkout. The calibration is bound to the weights by sha256 and the
|
|
5
|
+
loader refuses a mismatch, so a stray or older file elsewhere can never be used silently."""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
from importlib import resources
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
WEIGHTS_NAME, CALIBRATION_NAME = "roles_head.pt", "roles_calibration.json"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def models_dir() -> Path:
|
|
16
|
+
env = os.environ.get("OPENBOOKS_MODELS_DIR")
|
|
17
|
+
if env:
|
|
18
|
+
return Path(env)
|
|
19
|
+
packaged = Path(str(resources.files("openbooks_statements").joinpath("data")))
|
|
20
|
+
if (packaged / WEIGHTS_NAME).exists():
|
|
21
|
+
return packaged
|
|
22
|
+
for parent in Path(__file__).resolve().parents:
|
|
23
|
+
if (parent / "models" / WEIGHTS_NAME).exists():
|
|
24
|
+
return parent / "models"
|
|
25
|
+
return packaged # absent: load() reports that there are no weights
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
MODELS_DIR = models_dir()
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Calibration and abstention for the typed heads at inference: temperature-scaled probabilities and split-conformal sets.
|
|
2
|
+
|
|
3
|
+
A token is only trusted when its conformal set holds exactly one class. Anything else abstains, which the pipeline reports
|
|
4
|
+
as LOW_CONFIDENCE. Coverage (the true class is in the set) is guaranteed at 1 - alpha only if calibration and test data are
|
|
5
|
+
exchangeable; on layouts unlike the calibration set it can fail. The numbers (temperature, qhat, alpha) ship in
|
|
6
|
+
roles_calibration.json, bound to the exact weights by hash. Fitting them is not part of this package.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import math
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def softmax(z: np.ndarray) -> np.ndarray:
|
|
19
|
+
z = z - z.max(axis=-1, keepdims=True)
|
|
20
|
+
e = np.exp(z)
|
|
21
|
+
return e / e.sum(axis=-1, keepdims=True)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def prediction_set(probs: np.ndarray, qhat: float) -> list[int]:
|
|
25
|
+
return [i for i, p in enumerate(probs) if 1.0 - p <= qhat]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class Calibrator:
|
|
30
|
+
temperature: float = 1.0
|
|
31
|
+
qhat: float = float("inf")
|
|
32
|
+
alpha: float = 0.05
|
|
33
|
+
|
|
34
|
+
def probs(self, logits: np.ndarray) -> np.ndarray:
|
|
35
|
+
return softmax(np.asarray(logits, dtype=np.float64) / self.temperature)
|
|
36
|
+
|
|
37
|
+
def sets(self, logits: np.ndarray) -> list[list[int]]:
|
|
38
|
+
return [prediction_set(p, self.qhat) for p in self.probs(logits)]
|
|
39
|
+
|
|
40
|
+
def save(self, path: Path) -> None:
|
|
41
|
+
path.write_text(json.dumps({"temperature": self.temperature, "qhat": None if math.isinf(self.qhat) else self.qhat,
|
|
42
|
+
"alpha": self.alpha}), encoding="utf-8")
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def load(cls, path: Path) -> "Calibrator":
|
|
46
|
+
d = json.loads(path.read_text(encoding="utf-8"))
|
|
47
|
+
return cls(d["temperature"], float("inf") if d["qhat"] is None else d["qhat"], d["alpha"])
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"role": {"temperature": 2.019380728929039, "qhat": 0.9976497888565063, "alpha": 0.001}, "vtype": {"temperature": 1.0769541104862101, "qhat": 0.9351925849914551, "alpha": 0.001}, "weights_sha256": "d101e39610b74fa3195399a9e4aed89dad2916c2ac874284b08a38b6a0d31056"}
|
|
Binary file
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"model": "RoleNet (typed heads: column role + value type) over a LayoutLM-style encoder",
|
|
3
|
+
"parameters": 18878602,
|
|
4
|
+
"weights_mb_int8_at_rest": 19.04,
|
|
5
|
+
"weights_sha256": "d101e39610b74fa3195399a9e4aed89dad2916c2ac874284b08a38b6a0d31056",
|
|
6
|
+
"initialisation": "random; no pretrained weights",
|
|
7
|
+
"training_data": "synthetic statements only; no third-party dataset and no real client document",
|
|
8
|
+
"calibration": {
|
|
9
|
+
"method": "temperature scaling + split conformal",
|
|
10
|
+
"alpha": 0.001
|
|
11
|
+
},
|
|
12
|
+
"role": "cross-check only: the heads can add refusals, never cause a release"
|
|
13
|
+
}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""Layout encoder shared by the typed heads: hashed word and shape embeddings plus 2D box embeddings.
|
|
2
|
+
|
|
3
|
+
Release-one size: about 19M parameters (budget 25M; int8 at rest is about 1 MB per million). Trained from scratch on
|
|
4
|
+
synthetic statements only. Nothing here loads a pretrained checkpoint.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
import zlib
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
from torch import nn
|
|
13
|
+
|
|
14
|
+
NB, NS, MAXLEN, D, LAYERS, HEADS = 16384, 512, 512, 384, 6, 6
|
|
15
|
+
MONEY = re.compile(r"^\(?-?\d[\d,]*\.\d{2}\)?(?:Dr|Cr|DR|CR)?$")
|
|
16
|
+
NOT_MONEY, MAX_RANK, MAX_N = 5, 4, 6
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def column_heads(boxes: list[list[int]], hashes: list[int], money: list[bool]) -> list[list[int]]:
|
|
20
|
+
"""For each amount token, the hashed words (up to 3) of the nearest non-amount line above it in the same x-range.
|
|
21
|
+
|
|
22
|
+
On a statement that is the column header ('Withdrawal Amt.', 'Credit'); the model learns what those words mean. Nothing
|
|
23
|
+
here knows any header vocabulary, so a new header wording is just a new hash, not a rule to maintain.
|
|
24
|
+
"""
|
|
25
|
+
import numpy as np
|
|
26
|
+
|
|
27
|
+
heads = [[0, 0, 0] for _ in boxes]
|
|
28
|
+
if not any(money):
|
|
29
|
+
return heads
|
|
30
|
+
b, mon = np.asarray(boxes), np.asarray(money)
|
|
31
|
+
for i in np.nonzero(mon)[0]:
|
|
32
|
+
ok = ~mon & (np.minimum(b[:, 2], b[i, 2]) - np.maximum(b[:, 0], b[i, 0]) > 0) & (b[:, 3] <= b[i, 1])
|
|
33
|
+
idx = np.nonzero(ok)[0]
|
|
34
|
+
if not len(idx):
|
|
35
|
+
continue
|
|
36
|
+
near = idx[np.argmax(b[idx, 3])]
|
|
37
|
+
h = max(4, b[near, 3] - b[near, 1])
|
|
38
|
+
line = idx[np.abs((b[idx, 1] + b[idx, 3]) / 2 - (b[near, 1] + b[near, 3]) / 2) <= 0.6 * h]
|
|
39
|
+
ids = [hashes[k] for k in line[np.argsort(b[line, 0])][:3]]
|
|
40
|
+
heads[i] = ids + [0] * (3 - len(ids))
|
|
41
|
+
return heads
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def row_features(boxes: list[list[int]], money: list[bool]) -> list[list[int]]:
|
|
45
|
+
"""Per token (rank from the right among the amount tokens of its row, number of amounts in the row).
|
|
46
|
+
|
|
47
|
+
A statement row reads description ... [debit] [credit] balance, so the balance is always rank 0. This is geometry plus
|
|
48
|
+
the surface money test only: no header words and no rule-based column assignment, so it stays independent of the extractor.
|
|
49
|
+
"""
|
|
50
|
+
feats = [[NOT_MONEY, 0] for _ in boxes]
|
|
51
|
+
order = sorted((i for i, m in enumerate(money) if m), key=lambda i: boxes[i][1] + boxes[i][3])
|
|
52
|
+
rows: list[list[int]] = []
|
|
53
|
+
for i in order:
|
|
54
|
+
cy, h = (boxes[i][1] + boxes[i][3]) / 2, max(4, boxes[i][3] - boxes[i][1])
|
|
55
|
+
if rows and abs(cy - sum((boxes[k][1] + boxes[k][3]) / 2 for k in rows[-1]) / len(rows[-1])) <= 0.6 * h:
|
|
56
|
+
rows[-1].append(i)
|
|
57
|
+
else:
|
|
58
|
+
rows.append([i])
|
|
59
|
+
for row in rows:
|
|
60
|
+
for rank, i in enumerate(sorted(row, key=lambda k: -boxes[k][2])):
|
|
61
|
+
feats[i] = [min(rank, MAX_RANK), min(len(row), MAX_N)]
|
|
62
|
+
return feats
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def h_word(w: str) -> int:
|
|
66
|
+
return zlib.crc32(w.lower().encode()) % (NB - 2) + 2
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def h_shape(w: str) -> int:
|
|
70
|
+
s = "".join("d" if c.isdigit() else "A" if c.isupper() else "a" if c.isalpha() else c for c in w)
|
|
71
|
+
return zlib.crc32((re.sub(r"(.)\1+", r"\1", s) + str(min(len(w), 12))).encode()) % (NS - 1) + 1
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class Enc(nn.Module):
|
|
75
|
+
"""Token + shape + 2D box embeddings (LayoutLM-style), LAYERS x width D transformer."""
|
|
76
|
+
|
|
77
|
+
def __init__(self, d: int = D, layers: int = LAYERS, heads: int = HEADS):
|
|
78
|
+
super().__init__()
|
|
79
|
+
self.word, self.shape = nn.Embedding(NB, d, padding_idx=0), nn.Embedding(NS, d, padding_idx=0)
|
|
80
|
+
self.xe, self.ye, self.we, self.he = (nn.Embedding(1001, d) for _ in range(4))
|
|
81
|
+
self.pos = nn.Embedding(MAXLEN, d)
|
|
82
|
+
self.rank, self.nmon = nn.Embedding(NOT_MONEY + 1, d), nn.Embedding(MAX_N + 1, d)
|
|
83
|
+
layer = nn.TransformerEncoderLayer(d, heads, 4 * d, 0.1, activation="gelu", batch_first=True, norm_first=True)
|
|
84
|
+
self.tf = nn.TransformerEncoder(layer, layers, enable_nested_tensor=False)
|
|
85
|
+
self.norm = nn.LayerNorm(d)
|
|
86
|
+
|
|
87
|
+
def forward(self, W, S, X, pad, R, H):
|
|
88
|
+
pos = torch.arange(W.shape[1], device=W.device)
|
|
89
|
+
wd, ht = (X[..., 2] - X[..., 0]).clamp(0, 1000), (X[..., 3] - X[..., 1]).clamp(0, 1000)
|
|
90
|
+
x = (self.word(W) + self.shape(S) + self.xe(X[..., 0]) + self.xe(X[..., 2]) + self.ye(X[..., 1])
|
|
91
|
+
+ self.ye(X[..., 3]) + self.we(wd) + self.he(ht) + self.pos(pos) + self.rank(R[..., 0]) + self.nmon(R[..., 1])
|
|
92
|
+
+ self.word(H).sum(2))
|
|
93
|
+
return self.norm(self.tf(x, src_key_padding_mask=pad))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class RoleNet(nn.Module):
|
|
97
|
+
"""Encoder plus two typed heads: the column role of a token and the value type it holds."""
|
|
98
|
+
|
|
99
|
+
def __init__(self, n_roles: int, n_types: int):
|
|
100
|
+
super().__init__()
|
|
101
|
+
self.enc = Enc()
|
|
102
|
+
self.role, self.vtype = nn.Linear(D, n_roles), nn.Linear(D, n_types)
|
|
103
|
+
|
|
104
|
+
def forward(self, W, S, X, pad, R, H):
|
|
105
|
+
h = self.enc(W, S, X, pad, R, H)
|
|
106
|
+
return self.role(h), self.vtype(h)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def windows(words) -> list[list[int]]:
|
|
110
|
+
"""Index windows of at most MAXLEN tokens per page, in reading order (page, y, x)."""
|
|
111
|
+
out: list[list[int]] = []
|
|
112
|
+
by_page: dict[int, list[int]] = {}
|
|
113
|
+
for i, w in enumerate(words):
|
|
114
|
+
by_page.setdefault(w.page, []).append(i)
|
|
115
|
+
for idx in by_page.values():
|
|
116
|
+
idx.sort(key=lambda i: (round(words[i].cy / 4), words[i].x0))
|
|
117
|
+
out += [idx[k:k + MAXLEN] for k in range(0, len(idx), MAXLEN)]
|
|
118
|
+
return out
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def tensors(words, idx: list[int], dev):
|
|
122
|
+
"""Collate one window into (W, S, X, pad, R) batch-of-one tensors, boxes scaled to 0..1000 by page size."""
|
|
123
|
+
ws = [words[i] for i in idx]
|
|
124
|
+
clamp = lambda v: min(1000, max(0, int(v))) # noqa: E731
|
|
125
|
+
boxes = [[clamp(w.x0 * 1000 / w.pw), clamp(w.y0 * 1000 / w.ph), clamp(w.x1 * 1000 / w.pw), clamp(w.y1 * 1000 / w.ph)] for w in ws]
|
|
126
|
+
W = torch.tensor([[h_word(w.text) for w in ws]], device=dev)
|
|
127
|
+
S = torch.tensor([[h_shape(w.text) for w in ws]], device=dev)
|
|
128
|
+
money = [bool(MONEY.match(w.text)) for w in ws]
|
|
129
|
+
R = torch.tensor([row_features(boxes, money)], device=dev)
|
|
130
|
+
H = torch.tensor([column_heads(boxes, [h_word(w.text) for w in ws], money)], device=dev)
|
|
131
|
+
return W, S, torch.tensor([boxes], device=dev), torch.zeros(1, len(ws), dtype=torch.bool, device=dev), R, H
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def n_params(net: nn.Module) -> int:
|
|
135
|
+
return sum(p.numel() for p in net.parameters())
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def quantize_state(state: dict, min_numel: int = 4096) -> dict:
|
|
139
|
+
"""int8 at rest: per-row symmetric scales for every large 2D weight (embeddings and linears), fp16 for the rest."""
|
|
140
|
+
out = {}
|
|
141
|
+
for k, v in state.items():
|
|
142
|
+
if v.ndim == 2 and v.numel() >= min_numel:
|
|
143
|
+
scale = (v.abs().amax(dim=1, keepdim=True) / 127).clamp(min=1e-8)
|
|
144
|
+
out[k] = {"q": torch.round(v / scale).clamp(-127, 127).to(torch.int8), "s": scale.squeeze(1).half()}
|
|
145
|
+
else:
|
|
146
|
+
out[k] = v.half()
|
|
147
|
+
return out
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def dequantize_state(blob: dict) -> dict:
|
|
151
|
+
return {k: (v["q"].float() * v["s"].float().unsqueeze(1) if isinstance(v, dict) else v.float()) for k, v in blob.items()}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""PDF and image front end as a registry provider: extract() reads the document, identity() names the model that cross-checked it."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from .pdf_front import extract_document
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class _DocumentFrontend:
|
|
8
|
+
def extract(self, path, params):
|
|
9
|
+
return extract_document(path, params)
|
|
10
|
+
|
|
11
|
+
def identity(self, params):
|
|
12
|
+
from .roles import RoleModel
|
|
13
|
+
|
|
14
|
+
return getattr(RoleModel.load(), "identity", None)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
FRONTEND = _DocumentFrontend()
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Gate for statements read by OCR. Lived in extra_gate next to the tax rules; it belongs to the statements family."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def ocr_dates(b, tx):
|
|
6
|
+
"""Dates must be in order on an OCR'd statement (guards OCR date misreads)."""
|
|
7
|
+
if b.get("source_method") == "ocr":
|
|
8
|
+
ds = [t["date"] for t in tx]
|
|
9
|
+
ok = sum(1 for a, c in zip(ds, ds[1:]) if a <= c)
|
|
10
|
+
yield f"{b['id']}: dates are in order (guards OCR date misreads)", f"{max(len(ds) - 1, 0)} steps", f"{ok} steps", False
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""Bank statement parsers: bank CSV, MT940 and OFX. Moved verbatim from pipeline.py; become openbooks-statements.
|
|
2
|
+
"""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from decimal import Decimal
|
|
8
|
+
|
|
9
|
+
from openbooks_core.registry import bank_cols
|
|
10
|
+
from openbooks_core.builders import (D, body_rows as _body, cell_getter as _cell, currency_of as _currency, empty_data as _empty_data,
|
|
11
|
+
finish_batch as _finish, money_fields as _money_fields, settlement_batch as _settlement,
|
|
12
|
+
split_total as _split_total, stated_totals as _stated_totals, statement_doc)
|
|
13
|
+
from openbooks_core.errors import Refusal
|
|
14
|
+
from openbooks_core.evidence import evidence
|
|
15
|
+
from openbooks_core.normalise import date_order, fmt, money, to_date, to_dec
|
|
16
|
+
from openbooks_core.params import Params
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def parse_bank_csv(args, p: Params):
|
|
20
|
+
rows, hi, hs = args
|
|
21
|
+
idx = bank_cols(hs)
|
|
22
|
+
body = _body(rows, hi, hs)
|
|
23
|
+
cur = _currency(hs, [r for _, r in body], p)
|
|
24
|
+
if p.opening is None and not p.allow_derived_opening:
|
|
25
|
+
raise Refusal("PARSE_ERROR", "statement has no stated opening balance; supply --opening")
|
|
26
|
+
order = date_order([r[idx["date"]] for _, r in body], p.date_order, default="dmy", where="date column")
|
|
27
|
+
recs = []
|
|
28
|
+
for n, row in body:
|
|
29
|
+
def g(role: str) -> str:
|
|
30
|
+
return row[idx[role]].strip() if role in idx else ""
|
|
31
|
+
|
|
32
|
+
deb = to_dec(g("debit"), f"row {n} debit", blank_zero=True)
|
|
33
|
+
cre = to_dec(g("credit"), f"row {n} credit", blank_zero=True)
|
|
34
|
+
if (deb and cre) or not (deb or cre) or deb < 0 or cre < 0:
|
|
35
|
+
raise Refusal("PARSE_ERROR", f"row {n}: need exactly one positive debit or credit")
|
|
36
|
+
recs.append({
|
|
37
|
+
"row": n, "date": to_date(g("date"), f"row {n}", order), "dir": "credit" if cre else "debit",
|
|
38
|
+
"amt": cre or deb, "amt_raw": g("credit") if cre else g("debit"), "date_raw": g("date"),
|
|
39
|
+
"bal": to_dec(g("balance"), f"row {n} balance", dr_cr=True), "bal_raw": g("balance"),
|
|
40
|
+
"desc": g("desc"), "ref": g("ref") or None,
|
|
41
|
+
})
|
|
42
|
+
return statement_doc(recs, cur, p.account, p.opening)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
_TAG = re.compile(r"^:(\d{2}[A-Z]?):(.*)$")
|
|
46
|
+
_MT_BAL = re.compile(r"([CD])(\d{6})([A-Z]{3})(\d+,\d*)")
|
|
47
|
+
_MT_TXN = re.compile(r"(\d{6})(\d{4})?(R?[CD])([A-Z])?(\d+,\d*)N([A-Z0-9]{3})(.*)", re.S)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _mt_fields(text: str) -> list[list[str]]:
|
|
51
|
+
out: list[list[str]] = []
|
|
52
|
+
for line in text.replace("\r\n", "\n").replace("\r", "\n").split("\n"):
|
|
53
|
+
m = _TAG.match(line)
|
|
54
|
+
if m:
|
|
55
|
+
out.append([m.group(1), m.group(2)])
|
|
56
|
+
elif out and line.strip() not in ("", "-", "-}") and not line.startswith(("{", "}")):
|
|
57
|
+
out[-1][1] += "\n" + line
|
|
58
|
+
return out
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _mt_date(yymmdd: str, where: str) -> str:
|
|
62
|
+
try:
|
|
63
|
+
return datetime.strptime(yymmdd, "%y%m%d").date().isoformat()
|
|
64
|
+
except ValueError:
|
|
65
|
+
raise Refusal("PARSE_ERROR", f"{where}: unreadable date {yymmdd!r}") from None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _mt_bal(v: str, where: str):
|
|
69
|
+
m = _MT_BAL.fullmatch(v.strip())
|
|
70
|
+
if not m:
|
|
71
|
+
raise Refusal("PARSE_ERROR", f"{where}: unreadable balance {v!r}")
|
|
72
|
+
sign = 1 if m.group(1) == "C" else -1
|
|
73
|
+
return sign * Decimal(m.group(4).replace(",", ".")), _mt_date(m.group(2), where), m.group(3)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def parse_mt940(text: str, p: Params):
|
|
77
|
+
stmts: list[dict] = []
|
|
78
|
+
for tag, val in _mt_fields(text):
|
|
79
|
+
if tag == "20":
|
|
80
|
+
stmts.append({"txns": [], "open": None, "close": None})
|
|
81
|
+
elif not stmts:
|
|
82
|
+
continue
|
|
83
|
+
elif tag == "25":
|
|
84
|
+
stmts[-1]["acct"] = val.strip()
|
|
85
|
+
elif tag in ("60F", "60M") and stmts[-1]["open"] is None:
|
|
86
|
+
stmts[-1]["open"] = (_mt_bal(val, "opening"), val.strip())
|
|
87
|
+
elif tag in ("62F", "62M"):
|
|
88
|
+
stmts[-1]["close"] = (_mt_bal(val, "closing"), val.strip())
|
|
89
|
+
elif tag == "61":
|
|
90
|
+
m = _MT_TXN.fullmatch(val.split("\n")[0].strip())
|
|
91
|
+
if not m:
|
|
92
|
+
raise Refusal("PARSE_ERROR", f":61: unreadable {val!r}")
|
|
93
|
+
stmts[-1]["txns"].append({"m": m, "raw": val.split("\n")[0].strip(), "info": ""})
|
|
94
|
+
elif tag == "86" and stmts[-1]["txns"]:
|
|
95
|
+
stmts[-1]["txns"][-1]["info"] = " ".join(x.strip() for x in val.split("\n"))
|
|
96
|
+
data, ev, accts = _empty_data(), {}, {}
|
|
97
|
+
for si, s in enumerate(stmts, 1):
|
|
98
|
+
if not (s.get("acct") and s["open"] and s["close"]):
|
|
99
|
+
raise Refusal("PARSE_ERROR", f"statement {si}: missing :25:, :60F: or :62F:")
|
|
100
|
+
(ob, od, ocur), oraw = s["open"]
|
|
101
|
+
(cb, cd, ccur), craw = s["close"]
|
|
102
|
+
if ocur != ccur:
|
|
103
|
+
raise Refusal("PARSE_ERROR", f"statement {si}: currency changes inside statement")
|
|
104
|
+
aid = accts.setdefault(s["acct"], f"acct-{len(accts) + 1}")
|
|
105
|
+
if not any(a["id"] == aid for a in data["accounts"]):
|
|
106
|
+
data["accounts"].append({"id": aid, "number": s["acct"], "currency": ocur})
|
|
107
|
+
bid = f"batch-{si}"
|
|
108
|
+
data["batches"].append({"id": bid, "kind": "statement", "account_id": aid, "currency": ocur,
|
|
109
|
+
"period_start": od, "period_end": cd, "opening": money(ob, ocur),
|
|
110
|
+
"closing": money(cb, ocur), "opening_derived": False})
|
|
111
|
+
for f, raw in (("opening", oraw), ("closing", craw), ("period_start", oraw), ("period_end", craw)):
|
|
112
|
+
evidence(ev, bid, f, None, raw)
|
|
113
|
+
for k, t in enumerate(s["txns"], 1):
|
|
114
|
+
m = t["m"]
|
|
115
|
+
tid = f"txn-{si}-{k:05d}"
|
|
116
|
+
credit = m.group(3) in ("C", "RD")
|
|
117
|
+
ref = m.group(7).split("//")[0].strip()
|
|
118
|
+
data["transactions"].append({
|
|
119
|
+
"id": tid, "batch_id": bid, "account_id": aid, "date": _mt_date(m.group(1), tid),
|
|
120
|
+
"direction": "credit" if credit else "debit",
|
|
121
|
+
"amount": money(Decimal(m.group(5).replace(",", ".")), ocur),
|
|
122
|
+
"description": t["info"], "reference": None if ref in ("", "NONREF") else ref,
|
|
123
|
+
})
|
|
124
|
+
evidence(ev, tid, "date", None, t["raw"])
|
|
125
|
+
evidence(ev, tid, "amount", None, t["raw"])
|
|
126
|
+
if not data["transactions"]:
|
|
127
|
+
raise Refusal("PARSE_ERROR", "no :61: transactions")
|
|
128
|
+
return data, ev
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _ofx_tag(block: str, name: str) -> str | None:
|
|
132
|
+
m = re.search(rf"<{name}>([^<\r\n]*)", block, re.I)
|
|
133
|
+
return m.group(1).strip() if m else None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def parse_ofx(text: str, p: Params):
|
|
137
|
+
chunks = [c for c in re.split(r"(?=<STMTRS>|<CCSTMTRS>)", text, flags=re.I) if re.match(r"<(CC)?STMTRS>", c, re.I)]
|
|
138
|
+
if not chunks:
|
|
139
|
+
raise Refusal("PARSE_ERROR", "no statement response in OFX")
|
|
140
|
+
if p.opening is None:
|
|
141
|
+
raise Refusal("PARSE_ERROR", "OFX carries no opening balance; supply --opening")
|
|
142
|
+
if len(chunks) > 1:
|
|
143
|
+
raise Refusal("PARSE_ERROR", "multi-statement OFX with one --opening is ambiguous")
|
|
144
|
+
c = chunks[0]
|
|
145
|
+
cur = (_ofx_tag(c, "CURDEF") or p.currency or "").upper()
|
|
146
|
+
acct = _ofx_tag(c, "ACCTID")
|
|
147
|
+
bal = _ofx_tag(c, "BALAMT")
|
|
148
|
+
if not (cur and bal):
|
|
149
|
+
raise Refusal("PARSE_ERROR", "OFX missing CURDEF or LEDGERBAL")
|
|
150
|
+
blocks = re.findall(r"<STMTTRN>(.*?)(?=</STMTTRN>|<STMTTRN>|</BANKTRANLIST>|</CCSTMTTRNRS>|$)", c, re.S | re.I)
|
|
151
|
+
data, ev = _empty_data(), {}
|
|
152
|
+
data["accounts"].append({"id": "acct-1", "number": acct, "currency": cur})
|
|
153
|
+
seen, dates = set(), []
|
|
154
|
+
for k, b in enumerate(blocks, 1):
|
|
155
|
+
fit, amt_raw, dt = _ofx_tag(b, "FITID"), _ofx_tag(b, "TRNAMT"), _ofx_tag(b, "DTPOSTED")
|
|
156
|
+
if not (fit and amt_raw and dt) or fit in seen:
|
|
157
|
+
raise Refusal("PARSE_ERROR", f"transaction {k}: missing field or duplicate FITID")
|
|
158
|
+
seen.add(fit)
|
|
159
|
+
amt = to_dec(amt_raw, f"txn {k} amount")
|
|
160
|
+
if amt == 0:
|
|
161
|
+
raise Refusal("PARSE_ERROR", f"txn {k}: zero amount")
|
|
162
|
+
tid = f"txn-{k:05d}"
|
|
163
|
+
if not re.match(r"\d{8}", dt):
|
|
164
|
+
raise Refusal("PARSE_ERROR", f"{tid}: unreadable DTPOSTED {dt!r}")
|
|
165
|
+
d = datetime.strptime(dt[:8], "%Y%m%d").date().isoformat()
|
|
166
|
+
dates.append(d)
|
|
167
|
+
data["transactions"].append({
|
|
168
|
+
"id": tid, "batch_id": "batch-1", "account_id": "acct-1", "date": d,
|
|
169
|
+
"direction": "credit" if amt > 0 else "debit", "amount": money(abs(amt), cur),
|
|
170
|
+
"description": _ofx_tag(b, "NAME") or _ofx_tag(b, "MEMO") or "", "reference": fit,
|
|
171
|
+
})
|
|
172
|
+
evidence(ev, tid, "date", None, dt)
|
|
173
|
+
evidence(ev, tid, "amount", None, amt_raw)
|
|
174
|
+
if not dates:
|
|
175
|
+
raise Refusal("PARSE_ERROR", "no transactions")
|
|
176
|
+
data["batches"].append({
|
|
177
|
+
"id": "batch-1", "kind": "statement", "account_id": "acct-1", "currency": cur,
|
|
178
|
+
"period_start": min(dates), "period_end": max(dates), "opening": money(p.opening, cur),
|
|
179
|
+
"closing": money(to_dec(bal, "LEDGERBAL"), cur), "opening_derived": False,
|
|
180
|
+
})
|
|
181
|
+
evidence(ev, "batch-1", "opening", None, "supplied by caller")
|
|
182
|
+
evidence(ev, "batch-1", "closing", None, bal)
|
|
183
|
+
evidence(ev, "batch-1", "period_start", None, min(dates))
|
|
184
|
+
evidence(ev, "batch-1", "period_end", None, max(dates))
|
|
185
|
+
return data, ev
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Reading PDFs and images with permissively licensed libraries: pypdfium2 (Apache-2.0 / BSD-3, wrapping Google's PDFium) for
|
|
2
|
+
PDFs and Pillow for images. Nothing here is AGPL.
|
|
3
|
+
|
|
4
|
+
The rest of the engine sees only two things per page: its words as (text, x0, y0, x1, y1) in points with the origin at the
|
|
5
|
+
top-left (y grows downward), and an RGB render for OCR. The word rules are the ones the engine was built and measured with:
|
|
6
|
+
words break at whitespace and wherever one text object ends and the next begins (a table cell is its own text object), and text
|
|
7
|
+
that is not horizontal (watermarks, rotated stamps) is ignored.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import ctypes
|
|
12
|
+
import io
|
|
13
|
+
import math
|
|
14
|
+
from collections.abc import Iterator
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from openbooks_core.errors import EngineFault, Refusal
|
|
19
|
+
|
|
20
|
+
HORIZONTAL_COS, HORIZONTAL_SIN = 0.9, 0.05 # a line within ~2.9 degrees of horizontal counts as horizontal
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _pdfium():
|
|
24
|
+
try:
|
|
25
|
+
import pypdfium2 as pdfium
|
|
26
|
+
except ImportError as e:
|
|
27
|
+
raise EngineFault("PDF and image statements need pypdfium2: pip install 'openbooks-statements[pdf]'") from e
|
|
28
|
+
return pdfium
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class PdfPage:
|
|
32
|
+
def __init__(self, page: Any, pdfium: Any) -> None:
|
|
33
|
+
self._page, self._pdfium = page, pdfium
|
|
34
|
+
l, b, r, t = page.get_bbox()
|
|
35
|
+
self._left, self._top = l, t
|
|
36
|
+
self.width, self.height = r - l, t - b
|
|
37
|
+
self.rotation = page.get_rotation()
|
|
38
|
+
|
|
39
|
+
def words(self) -> list[tuple[str, float, float, float, float]]:
|
|
40
|
+
if self.rotation:
|
|
41
|
+
raise Refusal("UNSUPPORTED_INPUT", "a page with a rotated text layer is not supported; export it upright or scan it")
|
|
42
|
+
raw = self._pdfium.raw
|
|
43
|
+
tp = self._page.get_textpage()
|
|
44
|
+
try:
|
|
45
|
+
n = tp.count_chars()
|
|
46
|
+
ptr = tp.raw
|
|
47
|
+
out: list[tuple[str, float, float, float, float]] = []
|
|
48
|
+
cur: list[str] = []
|
|
49
|
+
box: list[float] = []
|
|
50
|
+
obj = None
|
|
51
|
+
|
|
52
|
+
def flush() -> None:
|
|
53
|
+
if cur:
|
|
54
|
+
out.append(("".join(cur), *box))
|
|
55
|
+
cur.clear()
|
|
56
|
+
box.clear()
|
|
57
|
+
|
|
58
|
+
high = 0 # pdfium reports UTF-16 code units: a character outside the BMP (an emoji in a payee name) is two of them
|
|
59
|
+
for i in range(n):
|
|
60
|
+
code = raw.FPDFText_GetUnicode(ptr, i)
|
|
61
|
+
if 0xDC00 <= code <= 0xDFFF and high:
|
|
62
|
+
code, high = 0x10000 + ((high - 0xD800) << 10) + (code - 0xDC00), 0
|
|
63
|
+
if cur:
|
|
64
|
+
cur[-1] = chr(code) # replaces the high half appended at the previous index
|
|
65
|
+
l, b, r, t = tp.get_charbox(i, loose=True)
|
|
66
|
+
box[0], box[1] = min(box[0], l - self._left), min(box[1], self._top - t)
|
|
67
|
+
box[2], box[3] = max(box[2], r - self._left), max(box[3], self._top - b)
|
|
68
|
+
continue
|
|
69
|
+
high = 0
|
|
70
|
+
if 0xD800 <= code <= 0xDBFF:
|
|
71
|
+
high = code
|
|
72
|
+
if 0xD800 <= code <= 0xDFFF:
|
|
73
|
+
code = 0xFFFD # kept only if a low half follows; a lone half is never passed on (it cannot be encoded)
|
|
74
|
+
ch = chr(code) if code else " "
|
|
75
|
+
if ch.isspace() or code == 0:
|
|
76
|
+
flush()
|
|
77
|
+
continue
|
|
78
|
+
angle = raw.FPDFText_GetCharAngle(ptr, i)
|
|
79
|
+
if math.cos(angle) < HORIZONTAL_COS or abs(math.sin(angle)) > HORIZONTAL_SIN:
|
|
80
|
+
flush()
|
|
81
|
+
continue
|
|
82
|
+
this_obj = ctypes.cast(raw.FPDFText_GetTextObject(ptr, i), ctypes.c_void_p).value
|
|
83
|
+
if cur and this_obj != obj:
|
|
84
|
+
flush()
|
|
85
|
+
obj = this_obj
|
|
86
|
+
l, b, r, t = tp.get_charbox(i, loose=True)
|
|
87
|
+
x0, x1 = l - self._left, r - self._left
|
|
88
|
+
y0, y1 = self._top - t, self._top - b
|
|
89
|
+
if not cur:
|
|
90
|
+
box.extend([x0, y0, x1, y1])
|
|
91
|
+
else:
|
|
92
|
+
box[0], box[1], box[2], box[3] = min(box[0], x0), min(box[1], y0), max(box[2], x1), max(box[3], y1)
|
|
93
|
+
cur.append(ch)
|
|
94
|
+
flush()
|
|
95
|
+
return out
|
|
96
|
+
finally:
|
|
97
|
+
tp.close()
|
|
98
|
+
|
|
99
|
+
def render_rgb(self, scale: float):
|
|
100
|
+
bitmap = self._page.render(scale=scale, rev_byteorder=True)
|
|
101
|
+
return bitmap.to_numpy()[:, :, :3].copy()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class ImagePage:
|
|
105
|
+
"""A photo or scan as a one-page document. Its size in points follows the image's own DPI (72 if it states none)."""
|
|
106
|
+
|
|
107
|
+
def __init__(self, image: Any) -> None:
|
|
108
|
+
dpi = image.info.get("dpi", (72, 72))[0] or 72
|
|
109
|
+
self._image = image.convert("RGB")
|
|
110
|
+
self.width, self.height = image.width * 72 / dpi, image.height * 72 / dpi
|
|
111
|
+
|
|
112
|
+
def words(self) -> list[tuple[str, float, float, float, float]]:
|
|
113
|
+
return [] # an image has no text layer
|
|
114
|
+
|
|
115
|
+
def render_rgb(self, scale: float):
|
|
116
|
+
import numpy as np
|
|
117
|
+
|
|
118
|
+
w, h = max(1, round(self.width * scale)), max(1, round(self.height * scale))
|
|
119
|
+
img = self._image if (w, h) == self._image.size else self._image.resize((w, h))
|
|
120
|
+
return np.asarray(img, dtype=np.uint8).copy()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class Document:
|
|
124
|
+
def __init__(self, pages: Iterator[Any], count: int, close) -> None:
|
|
125
|
+
self._pages, self.page_count, self._close = pages, count, close
|
|
126
|
+
|
|
127
|
+
def pages(self) -> Iterator[Any]:
|
|
128
|
+
return self._pages
|
|
129
|
+
|
|
130
|
+
def close(self) -> None:
|
|
131
|
+
self._close()
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def open_document(source: Path | bytes, *, image: bool = False) -> Document:
|
|
135
|
+
if image:
|
|
136
|
+
try:
|
|
137
|
+
from PIL import Image
|
|
138
|
+
except ImportError as e:
|
|
139
|
+
raise EngineFault("images need Pillow: pip install 'openbooks-statements[scan]'") from e
|
|
140
|
+
try:
|
|
141
|
+
img = Image.open(io.BytesIO(source) if isinstance(source, (bytes, bytearray)) else source)
|
|
142
|
+
img.load()
|
|
143
|
+
except Exception:
|
|
144
|
+
raise Refusal("UNSUPPORTED_INPUT", "unreadable document") from None
|
|
145
|
+
return Document(iter([ImagePage(img)]), 1, img.close)
|
|
146
|
+
pdfium = _pdfium()
|
|
147
|
+
try:
|
|
148
|
+
pdf = pdfium.PdfDocument(bytes(source) if isinstance(source, (bytes, bytearray)) else str(source))
|
|
149
|
+
except pdfium.PdfiumError as e:
|
|
150
|
+
if getattr(e, "err_code", None) == pdfium.raw.FPDF_ERR_PASSWORD:
|
|
151
|
+
raise Refusal("UNSUPPORTED_INPUT", "encrypted document") from None
|
|
152
|
+
raise Refusal("UNSUPPORTED_INPUT", "unreadable document") from None
|
|
153
|
+
except Exception:
|
|
154
|
+
raise Refusal("UNSUPPORTED_INPUT", "unreadable document") from None
|
|
155
|
+
|
|
156
|
+
def pages() -> Iterator[PdfPage]:
|
|
157
|
+
for i in range(len(pdf)):
|
|
158
|
+
page = pdf[i]
|
|
159
|
+
try:
|
|
160
|
+
yield PdfPage(page, pdfium)
|
|
161
|
+
finally:
|
|
162
|
+
page.close()
|
|
163
|
+
|
|
164
|
+
return Document(pages(), len(pdf), pdf.close)
|