memvo 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memvo/__init__.py +3 -0
- memvo/__main__.py +3 -0
- memvo/audit.py +73 -0
- memvo/cli.py +237 -0
- memvo/dates.py +88 -0
- memvo/importers/__init__.py +4 -0
- memvo/importers/csv_rows.py +168 -0
- memvo/importers/ofx_rows.py +90 -0
- memvo/mcp.py +233 -0
- memvo/money.py +89 -0
- memvo/reconcile.py +133 -0
- memvo/vault.py +445 -0
- memvo-0.1.0a1.dist-info/METADATA +207 -0
- memvo-0.1.0a1.dist-info/RECORD +17 -0
- memvo-0.1.0a1.dist-info/WHEEL +4 -0
- memvo-0.1.0a1.dist-info/entry_points.txt +2 -0
- memvo-0.1.0a1.dist-info/licenses/LICENSE +21 -0
memvo/__init__.py
ADDED
memvo/__main__.py
ADDED
memvo/audit.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Audit: prove that every stored entry still matches the file it came from.
|
|
2
|
+
|
|
3
|
+
For each source the file is re-read and re-hashed. If the hash matches, every
|
|
4
|
+
entry from it is checked line by line against the bytes on disk. If the file
|
|
5
|
+
changed or disappeared, its entries are reported, never silently rewritten.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from .vault import Vault, decode_text, sha256_bytes, sha256_text
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class SourceCheck:
|
|
18
|
+
source_id: int
|
|
19
|
+
path: str
|
|
20
|
+
status: str # ok | changed | missing
|
|
21
|
+
entries: int
|
|
22
|
+
mismatched: list[str] = field(default_factory=list)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class AuditReport:
|
|
27
|
+
sources: list[SourceCheck]
|
|
28
|
+
duplicates: list[list[str]]
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def ok(self) -> bool:
|
|
32
|
+
return all(s.status == "ok" and not s.mismatched for s in self.sources)
|
|
33
|
+
|
|
34
|
+
def as_dict(self) -> dict:
|
|
35
|
+
return {
|
|
36
|
+
"ok": self.ok,
|
|
37
|
+
"sources": [s.__dict__ for s in self.sources],
|
|
38
|
+
"possible_duplicates": self.duplicates,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def audit(vault: Vault) -> AuditReport:
|
|
43
|
+
checks: list[SourceCheck] = []
|
|
44
|
+
for src in vault.sources():
|
|
45
|
+
rows = vault.db.execute(
|
|
46
|
+
"SELECT id, first_line, last_line, raw, raw_sha256 FROM entries WHERE source_id=? ORDER BY first_line",
|
|
47
|
+
(src["id"],),
|
|
48
|
+
).fetchall()
|
|
49
|
+
path = Path(src["path"])
|
|
50
|
+
if not path.exists():
|
|
51
|
+
checks.append(SourceCheck(src["id"], src["path"], "missing", len(rows)))
|
|
52
|
+
continue
|
|
53
|
+
data = path.read_bytes()
|
|
54
|
+
if sha256_bytes(data) != src["sha256"]:
|
|
55
|
+
checks.append(SourceCheck(src["id"], src["path"], "changed", len(rows)))
|
|
56
|
+
continue
|
|
57
|
+
lines = decode_text(data).splitlines(keepends=True)
|
|
58
|
+
bad = []
|
|
59
|
+
# CSV rows equal their line range; an OFX block may share a line with others.
|
|
60
|
+
for eid, first, last, raw, raw_sha in rows:
|
|
61
|
+
on_disk = "".join(lines[first - 1 : last]).rstrip("\r\n")
|
|
62
|
+
if raw not in on_disk or sha256_text(raw) != raw_sha:
|
|
63
|
+
bad.append(eid)
|
|
64
|
+
checks.append(SourceCheck(src["id"], src["path"], "ok", len(rows), bad))
|
|
65
|
+
|
|
66
|
+
dupes = [
|
|
67
|
+
ids.split(",")
|
|
68
|
+
for (ids,) in vault.db.execute(
|
|
69
|
+
"""SELECT GROUP_CONCAT(id) FROM entries GROUP BY fingerprint
|
|
70
|
+
HAVING COUNT(DISTINCT source_id) > 1 ORDER BY MIN(day)"""
|
|
71
|
+
)
|
|
72
|
+
]
|
|
73
|
+
return AuditReport(checks, dupes)
|
memvo/cli.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from . import __version__
|
|
9
|
+
from .audit import audit
|
|
10
|
+
from .money import AmountError, parse_amount, to_scaled
|
|
11
|
+
from .reconcile import reconcile
|
|
12
|
+
from .vault import Vault, VaultError, default_vault_path
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _vault(args) -> Vault:
|
|
16
|
+
return Vault(args.vault or default_vault_path())
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _emit(args, payload, text: str) -> None:
|
|
20
|
+
if args.json:
|
|
21
|
+
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
22
|
+
else:
|
|
23
|
+
print(text)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _parse_map(pairs: list[str] | None) -> dict[str, str]:
|
|
27
|
+
out = {}
|
|
28
|
+
for pair in pairs or []:
|
|
29
|
+
role, sep, col = pair.partition("=")
|
|
30
|
+
if not sep or not role or not col:
|
|
31
|
+
raise SystemExit(f"--map expects role=column, got {pair!r}")
|
|
32
|
+
out[role.strip()] = col.strip()
|
|
33
|
+
return out
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def cmd_init(args) -> int:
|
|
37
|
+
path = Path(args.vault or default_vault_path())
|
|
38
|
+
existed = path.exists()
|
|
39
|
+
with Vault.init(path) as v:
|
|
40
|
+
_emit(args, v.stats(), f"{'Vault already at' if existed else 'Vault created at'} {path}")
|
|
41
|
+
return 0
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def cmd_import(args) -> int:
|
|
45
|
+
with _vault(args) as v:
|
|
46
|
+
r = v.import_csv(args.file, args.account, kind=args.kind, columns=_parse_map(args.map))
|
|
47
|
+
lines = [
|
|
48
|
+
f"{r.account}: {r.added} entries added from {Path(r.path).name} ({r.format})",
|
|
49
|
+
f"source sha256 {r.sha256[:16]}…",
|
|
50
|
+
]
|
|
51
|
+
if r.already_present:
|
|
52
|
+
lines[0] = f"{r.account}: file already imported ({r.already_present} entries), nothing added"
|
|
53
|
+
if r.skipped:
|
|
54
|
+
lines.append(f"{len(r.skipped)} rows skipped:")
|
|
55
|
+
lines += [f" line {n}: {why}" for n, why in r.skipped[:10]]
|
|
56
|
+
if r.columns:
|
|
57
|
+
lines.append("columns: " + ", ".join(f"{k}={v}" for k, v in r.columns.items()))
|
|
58
|
+
if r.statement_balance is not None:
|
|
59
|
+
lines.append(f"statement balance {r.statement_balance} as of {r.statement_as_of}")
|
|
60
|
+
_emit(args, r.__dict__, "\n".join(lines))
|
|
61
|
+
return 0
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _entry_line(e) -> str:
|
|
65
|
+
who = e.counterparty or e.description
|
|
66
|
+
return f"{e.day} {e.amount_text:>14} {e.account:<12} {who[:48]}\n {e.id} {Path(e.source_path).name}:{e.first_line}"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def cmd_search(args) -> int:
|
|
70
|
+
with _vault(args) as v:
|
|
71
|
+
hits = v.search(
|
|
72
|
+
" ".join(args.query), account=args.account, period=args.period,
|
|
73
|
+
since=args.since, until=args.until, limit=args.limit,
|
|
74
|
+
)
|
|
75
|
+
text = "\n".join(_entry_line(e) for e in hits) if hits else "No entries match."
|
|
76
|
+
_emit(args, [e.as_dict() for e in hits], text)
|
|
77
|
+
return 0 if hits else 1
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def cmd_show(args) -> int:
|
|
81
|
+
with _vault(args) as v:
|
|
82
|
+
e = v.get(args.entry)
|
|
83
|
+
if not e:
|
|
84
|
+
print(f"no entry {args.entry}", file=sys.stderr)
|
|
85
|
+
return 1
|
|
86
|
+
r = e.receipt()
|
|
87
|
+
text = (
|
|
88
|
+
f"{e.id}\n{e.day} {e.amount_text} {e.currency or ''} {e.account}\n"
|
|
89
|
+
f"{e.description}\n\nreceipt\n source {r['source']} (line{'s' if '-' in r['lines'] else ''} {r['lines']})\n"
|
|
90
|
+
f" file sha256 {r['source_sha256']}\n row sha256 {r['raw_sha256']}\n raw {r['raw']}"
|
|
91
|
+
)
|
|
92
|
+
_emit(args, e.as_dict(), text)
|
|
93
|
+
return 0
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def cmd_accounts(args) -> int:
|
|
97
|
+
with _vault(args) as v:
|
|
98
|
+
rows = v.accounts()
|
|
99
|
+
text = "\n".join(
|
|
100
|
+
f"{a['name']:<16} {a['kind']:<8} {a['entries']:>6} entries net {a['net']:>14} {a['first'] or '-'} → {a['last'] or '-'}"
|
|
101
|
+
for a in rows
|
|
102
|
+
) or "No accounts yet. Import a statement with `memvo import FILE --account NAME`."
|
|
103
|
+
_emit(args, rows, text)
|
|
104
|
+
return 0
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def cmd_periods(args) -> int:
|
|
108
|
+
with _vault(args) as v:
|
|
109
|
+
rows = v.periods(args.account)
|
|
110
|
+
text = "\n".join(
|
|
111
|
+
f"{p['account']:<16} {p['period']} {p['entries']:>5} in {p['in']:>14} out {p['out']:>14}" for p in rows
|
|
112
|
+
) or "No entries yet."
|
|
113
|
+
_emit(args, rows, text)
|
|
114
|
+
return 0
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def cmd_audit(args) -> int:
|
|
118
|
+
with _vault(args) as v:
|
|
119
|
+
rep = audit(v)
|
|
120
|
+
lines = []
|
|
121
|
+
for s in rep.sources:
|
|
122
|
+
mark = "ok" if s.status == "ok" and not s.mismatched else s.status.upper()
|
|
123
|
+
extra = f", {len(s.mismatched)} rows differ" if s.mismatched else ""
|
|
124
|
+
lines.append(f"[{mark}] {Path(s.path).name}: {s.entries} entries{extra}")
|
|
125
|
+
if rep.duplicates:
|
|
126
|
+
lines.append(f"{len(rep.duplicates)} possible duplicates across files (same account, day, amount, text):")
|
|
127
|
+
lines += [" " + " ".join(d) for d in rep.duplicates[:10]]
|
|
128
|
+
lines.append("All entries match their sources." if rep.ok else "Audit found problems.")
|
|
129
|
+
_emit(args, rep.as_dict(), "\n".join(lines))
|
|
130
|
+
return 0 if rep.ok else 2
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def cmd_reconcile(args) -> int:
|
|
134
|
+
try:
|
|
135
|
+
use_statement = args.expect == "statement"
|
|
136
|
+
expected = to_scaled(parse_amount(args.expect)) if args.expect and not use_statement else None
|
|
137
|
+
opening = to_scaled(parse_amount(args.opening)) if args.opening else 0
|
|
138
|
+
except AmountError as exc:
|
|
139
|
+
raise SystemExit(str(exc)) from exc
|
|
140
|
+
with _vault(args) as v:
|
|
141
|
+
rep = reconcile(v, args.account, expected=expected, as_of=args.as_of, opening=opening,
|
|
142
|
+
use_statement=use_statement)
|
|
143
|
+
lines = []
|
|
144
|
+
for c in rep.chains:
|
|
145
|
+
state = "unbroken" if not c.breaks else f"{len(c.breaks)} breaks"
|
|
146
|
+
lines.append(f"running balance in {Path(c.path).name}: {c.rows_checked} rows, {c.order}, {state}")
|
|
147
|
+
for b in c.breaks[:10]:
|
|
148
|
+
lines.append(f" {b.day} {b.entry}: expected {b.expected}, statement says {b.stated} (off by {b.diff})")
|
|
149
|
+
if rep.expected is not None:
|
|
150
|
+
lines.append(
|
|
151
|
+
f"as of {rep.as_of or 'latest'}: entries add up to {rep.computed}, "
|
|
152
|
+
f"{'statement says' if rep.expected_from == 'statement' else 'expected'} {rep.expected}, "
|
|
153
|
+
f"difference {rep.difference}"
|
|
154
|
+
)
|
|
155
|
+
if not lines:
|
|
156
|
+
lines.append("Nothing to check: no balance column in the sources and no --expect given.")
|
|
157
|
+
_emit(args, rep.as_dict(), "\n".join(lines))
|
|
158
|
+
return 0 if rep.ok else 2
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def cmd_mcp(args) -> int:
|
|
162
|
+
from .mcp import Server
|
|
163
|
+
|
|
164
|
+
Server(args.vault or default_vault_path()).serve()
|
|
165
|
+
return 0
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def cmd_stats(args) -> int:
|
|
169
|
+
with _vault(args) as v:
|
|
170
|
+
s = v.stats()
|
|
171
|
+
_emit(args, s, "\n".join(f"{k:<9} {v}" for k, v in s.items()))
|
|
172
|
+
return 0
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
176
|
+
p = argparse.ArgumentParser(prog="memvo", description="Local-first financial memory with a receipt for every answer.")
|
|
177
|
+
p.add_argument("--version", action="version", version=f"memvo {__version__}")
|
|
178
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
179
|
+
common.add_argument("--vault", help="vault file (default: $MEMVO_HOME/vault.db or ~/.memvo/vault.db)")
|
|
180
|
+
common.add_argument("--json", action="store_true", help="machine-readable output")
|
|
181
|
+
sub = p.add_subparsers(dest="command", required=True)
|
|
182
|
+
|
|
183
|
+
sub.add_parser("init", parents=[common], help="create a vault").set_defaults(fn=cmd_init)
|
|
184
|
+
|
|
185
|
+
s = sub.add_parser("import", parents=[common], help="import a CSV or OFX/QFX statement")
|
|
186
|
+
s.add_argument("file")
|
|
187
|
+
s.add_argument("--account", required=True)
|
|
188
|
+
s.add_argument("--kind", default="bank", choices=["bank", "card", "broker", "wallet", "cash", "other"])
|
|
189
|
+
s.add_argument("--map", action="append", metavar="ROLE=COLUMN",
|
|
190
|
+
help="override column detection, e.g. --map amount=Betrag")
|
|
191
|
+
s.set_defaults(fn=cmd_import)
|
|
192
|
+
|
|
193
|
+
s = sub.add_parser("search", parents=[common], help="find entries; each hit carries its receipt")
|
|
194
|
+
s.add_argument("query", nargs="*")
|
|
195
|
+
s.add_argument("--account")
|
|
196
|
+
s.add_argument("--period", help="YYYY-MM")
|
|
197
|
+
s.add_argument("--since", help="YYYY-MM-DD")
|
|
198
|
+
s.add_argument("--until", help="YYYY-MM-DD")
|
|
199
|
+
s.add_argument("--limit", type=int, default=10)
|
|
200
|
+
s.set_defaults(fn=cmd_search)
|
|
201
|
+
|
|
202
|
+
s = sub.add_parser("show", parents=[common], help="show one entry and its receipt")
|
|
203
|
+
s.add_argument("entry")
|
|
204
|
+
s.set_defaults(fn=cmd_show)
|
|
205
|
+
|
|
206
|
+
sub.add_parser("accounts", parents=[common], help="list accounts").set_defaults(fn=cmd_accounts)
|
|
207
|
+
|
|
208
|
+
s = sub.add_parser("periods", parents=[common], help="money in and out per month")
|
|
209
|
+
s.add_argument("--account")
|
|
210
|
+
s.set_defaults(fn=cmd_periods)
|
|
211
|
+
|
|
212
|
+
sub.add_parser("audit", parents=[common], help="check every entry against its source file").set_defaults(fn=cmd_audit)
|
|
213
|
+
|
|
214
|
+
s = sub.add_parser("reconcile", parents=[common], help="check balances")
|
|
215
|
+
s.add_argument("--account", required=True)
|
|
216
|
+
s.add_argument("--expect", help="balance shown on the statement or wallet, or 'statement' to use the "
|
|
217
|
+
"ledger balance from the newest imported OFX")
|
|
218
|
+
s.add_argument("--as-of", help="YYYY-MM-DD, default: all entries")
|
|
219
|
+
s.add_argument("--opening", help="opening balance before the first entry")
|
|
220
|
+
s.set_defaults(fn=cmd_reconcile)
|
|
221
|
+
|
|
222
|
+
sub.add_parser("stats", parents=[common], help="vault summary").set_defaults(fn=cmd_stats)
|
|
223
|
+
sub.add_parser("mcp", parents=[common], help="run the read-only MCP server on stdio").set_defaults(fn=cmd_mcp)
|
|
224
|
+
return p
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def main(argv: list[str] | None = None) -> int:
|
|
228
|
+
args = build_parser().parse_args(argv)
|
|
229
|
+
try:
|
|
230
|
+
return args.fn(args)
|
|
231
|
+
except (VaultError, ValueError, FileNotFoundError, UnicodeDecodeError) as exc:
|
|
232
|
+
print(f"memvo: {exc}", file=sys.stderr)
|
|
233
|
+
return 1
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
if __name__ == "__main__":
|
|
237
|
+
raise SystemExit(main())
|
memvo/dates.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Date parsing for statement exports.
|
|
2
|
+
|
|
3
|
+
Slash dates are ambiguous (03/04/2026). The importer looks at every date in a
|
|
4
|
+
file before choosing day-first or month-first, so one file is never parsed
|
|
5
|
+
two different ways.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from datetime import date, datetime, timezone
|
|
12
|
+
|
|
13
|
+
_MONTHS = {
|
|
14
|
+
m: i
|
|
15
|
+
for i, names in enumerate(
|
|
16
|
+
[
|
|
17
|
+
("jan", "january", "januari"),
|
|
18
|
+
("feb", "february", "februari"),
|
|
19
|
+
("mar", "march", "maret"),
|
|
20
|
+
("apr", "april"),
|
|
21
|
+
("may", "mei"),
|
|
22
|
+
("jun", "june", "juni"),
|
|
23
|
+
("jul", "july", "juli"),
|
|
24
|
+
("aug", "august", "agu", "agustus"),
|
|
25
|
+
("sep", "sept", "september"),
|
|
26
|
+
("oct", "october", "okt", "oktober"),
|
|
27
|
+
("nov", "november", "nop", "nopember"),
|
|
28
|
+
("dec", "december", "des", "desember"),
|
|
29
|
+
],
|
|
30
|
+
start=1,
|
|
31
|
+
)
|
|
32
|
+
for m in names
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
_ISO = re.compile(r"^(\d{4})-(\d{1,2})-(\d{1,2})(?:[T ].*)?$")
|
|
36
|
+
_SLASH = re.compile(r"^(\d{1,2})[/.\-](\d{1,2})[/.\-](\d{2,4})(?:\s.*)?$")
|
|
37
|
+
_YMD_SLASH = re.compile(r"^(\d{4})/(\d{1,2})/(\d{1,2})(?:\s.*)?$")
|
|
38
|
+
_NAMED = re.compile(r"^(\d{1,2})[\s\-]+([A-Za-z]+)\.?[\s\-,]+(\d{2,4})$")
|
|
39
|
+
_NAMED_US = re.compile(r"^([A-Za-z]+)\.?\s+(\d{1,2}),?\s+(\d{4})$")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class DateError(ValueError):
|
|
43
|
+
pass
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _year(y: str) -> int:
|
|
47
|
+
n = int(y)
|
|
48
|
+
return n + 2000 if len(y) == 2 else n
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def slash_order(samples: list[str]) -> str:
|
|
52
|
+
"""Return "dmy" or "mdy" for a column of slash dates."""
|
|
53
|
+
for s in samples:
|
|
54
|
+
m = _SLASH.match(s.strip())
|
|
55
|
+
if not m:
|
|
56
|
+
continue
|
|
57
|
+
a, b = int(m.group(1)), int(m.group(2))
|
|
58
|
+
if a > 12 >= b:
|
|
59
|
+
return "dmy"
|
|
60
|
+
if b > 12 >= a:
|
|
61
|
+
return "mdy"
|
|
62
|
+
return "dmy"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def parse_date(text: str, order: str = "dmy") -> date:
|
|
66
|
+
s = text.strip()
|
|
67
|
+
try:
|
|
68
|
+
if m := _ISO.match(s):
|
|
69
|
+
return date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
|
70
|
+
if m := _YMD_SLASH.match(s):
|
|
71
|
+
return date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
|
72
|
+
if m := _SLASH.match(s):
|
|
73
|
+
a, b, y = int(m.group(1)), int(m.group(2)), _year(m.group(3))
|
|
74
|
+
return date(y, b, a) if order == "dmy" else date(y, a, b)
|
|
75
|
+
if m := _NAMED.match(s):
|
|
76
|
+
month = _MONTHS.get(m.group(2).lower())
|
|
77
|
+
if month:
|
|
78
|
+
return date(_year(m.group(3)), month, int(m.group(1)))
|
|
79
|
+
if m := _NAMED_US.match(s):
|
|
80
|
+
month = _MONTHS.get(m.group(1).lower())
|
|
81
|
+
if month:
|
|
82
|
+
return date(int(m.group(3)), month, int(m.group(2)))
|
|
83
|
+
if s.isdigit() and len(s) in (10, 13):
|
|
84
|
+
ts = int(s) / (1000 if len(s) == 13 else 1)
|
|
85
|
+
return datetime.fromtimestamp(ts, timezone.utc).date()
|
|
86
|
+
except ValueError as exc:
|
|
87
|
+
raise DateError(f"not a date: {text!r}") from exc
|
|
88
|
+
raise DateError(f"not a date: {text!r}")
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""CSV statement reader.
|
|
2
|
+
|
|
3
|
+
Each record keeps the exact source text it was parsed from (`raw`) and the
|
|
4
|
+
1-based line range it spans, so a receipt can point back at the file.
|
|
5
|
+
Quoted fields with embedded newlines span more than one line; that is why
|
|
6
|
+
the range is tracked instead of a single line number.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import csv
|
|
12
|
+
import io
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from datetime import date
|
|
15
|
+
from decimal import Decimal
|
|
16
|
+
|
|
17
|
+
from ..dates import DateError, parse_date, slash_order
|
|
18
|
+
from ..money import AmountError, parse_amount
|
|
19
|
+
|
|
20
|
+
_ALIASES: dict[str, tuple[str, ...]] = {
|
|
21
|
+
"date": (
|
|
22
|
+
"date", "posted", "posting date", "transaction date", "trans date", "booking date",
|
|
23
|
+
"value date", "tanggal", "tgl", "time", "timestamp", "datetime", "completed date",
|
|
24
|
+
),
|
|
25
|
+
"amount": ("amount", "jumlah", "nominal", "value", "net amount", "total", "mutasi"),
|
|
26
|
+
"debit": ("debit", "withdrawal", "withdrawals", "paid out", "money out", "out", "debet"),
|
|
27
|
+
"credit": ("credit", "deposit", "deposits", "paid in", "money in", "in", "kredit"),
|
|
28
|
+
"description": (
|
|
29
|
+
"description", "memo", "details", "narrative", "keterangan", "transaction",
|
|
30
|
+
"reference", "remarks", "particulars", "type",
|
|
31
|
+
),
|
|
32
|
+
"counterparty": ("payee", "merchant", "counterparty", "name", "beneficiary", "to", "from"),
|
|
33
|
+
"currency": ("currency", "ccy", "mata uang"),
|
|
34
|
+
"balance": ("balance", "running balance", "saldo", "closing balance"),
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class ColumnMap:
|
|
40
|
+
date: str
|
|
41
|
+
amount: str | None = None
|
|
42
|
+
debit: str | None = None
|
|
43
|
+
credit: str | None = None
|
|
44
|
+
description: str | None = None
|
|
45
|
+
counterparty: str | None = None
|
|
46
|
+
currency: str | None = None
|
|
47
|
+
balance: str | None = None
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def detect(cls, header: list[str], overrides: dict[str, str] | None = None) -> ColumnMap:
|
|
51
|
+
found: dict[str, str] = {}
|
|
52
|
+
lowered = {h.strip().lower(): h for h in header}
|
|
53
|
+
for role, names in _ALIASES.items():
|
|
54
|
+
for name in names:
|
|
55
|
+
if name in lowered and lowered[name] not in found.values():
|
|
56
|
+
found[role] = lowered[name]
|
|
57
|
+
break
|
|
58
|
+
for role, col in (overrides or {}).items():
|
|
59
|
+
if role not in _ALIASES:
|
|
60
|
+
raise ValueError(f"unknown column role {role!r}")
|
|
61
|
+
if col not in header:
|
|
62
|
+
raise ValueError(f"column {col!r} not in header {header}")
|
|
63
|
+
found[role] = col
|
|
64
|
+
if "date" not in found:
|
|
65
|
+
raise ValueError(f"no date column found in {header}; pass --map date=<column>")
|
|
66
|
+
if "amount" not in found and not ({"debit", "credit"} & found.keys()):
|
|
67
|
+
raise ValueError(f"no amount or debit/credit column in {header}; pass --map amount=<column>")
|
|
68
|
+
return cls(**found)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True)
|
|
72
|
+
class ImportedRow:
|
|
73
|
+
first_line: int
|
|
74
|
+
last_line: int
|
|
75
|
+
raw: str
|
|
76
|
+
when: date
|
|
77
|
+
amount: Decimal
|
|
78
|
+
description: str
|
|
79
|
+
counterparty: str
|
|
80
|
+
currency: str | None
|
|
81
|
+
balance: Decimal | None
|
|
82
|
+
fields: dict[str, str] = field(default_factory=dict)
|
|
83
|
+
ref: str = ""
|
|
84
|
+
pos: int | None = None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass
|
|
88
|
+
class CsvResult:
|
|
89
|
+
columns: ColumnMap
|
|
90
|
+
rows: list[ImportedRow]
|
|
91
|
+
skipped: list[tuple[int, str]]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _records(text: str, dialect: type[csv.Dialect] | csv.Dialect):
|
|
95
|
+
lines = text.splitlines(keepends=True)
|
|
96
|
+
reader = csv.reader(io.StringIO(text), dialect)
|
|
97
|
+
start = 1
|
|
98
|
+
for record in reader:
|
|
99
|
+
end = reader.line_num
|
|
100
|
+
raw = "".join(lines[start - 1 : end]).rstrip("\r\n")
|
|
101
|
+
yield start, end, raw, record
|
|
102
|
+
start = end + 1
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def read_csv(text: str, overrides: dict[str, str] | None = None) -> CsvResult:
|
|
106
|
+
if text.startswith(""):
|
|
107
|
+
text = text[1:]
|
|
108
|
+
sample = text[:8192]
|
|
109
|
+
try:
|
|
110
|
+
dialect: type[csv.Dialect] | csv.Dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|")
|
|
111
|
+
except csv.Error:
|
|
112
|
+
dialect = csv.excel
|
|
113
|
+
|
|
114
|
+
records = [r for r in _records(text, dialect) if any(c.strip() for c in r[3])]
|
|
115
|
+
if not records:
|
|
116
|
+
return CsvResult(ColumnMap(date=""), [], [])
|
|
117
|
+
|
|
118
|
+
header = [h.strip() for h in records[0][3]]
|
|
119
|
+
columns = ColumnMap.detect(header, overrides)
|
|
120
|
+
idx = {h: i for i, h in enumerate(header)}
|
|
121
|
+
|
|
122
|
+
def cell(rec: list[str], col: str | None) -> str:
|
|
123
|
+
if col is None:
|
|
124
|
+
return ""
|
|
125
|
+
i = idx[col]
|
|
126
|
+
return rec[i].strip() if i < len(rec) else ""
|
|
127
|
+
|
|
128
|
+
order = slash_order([cell(r[3], columns.date) for r in records[1:]])
|
|
129
|
+
rows: list[ImportedRow] = []
|
|
130
|
+
skipped: list[tuple[int, str]] = []
|
|
131
|
+
for first, last, raw, rec in records[1:]:
|
|
132
|
+
try:
|
|
133
|
+
when = parse_date(cell(rec, columns.date), order)
|
|
134
|
+
amount = _amount(rec, columns, cell)
|
|
135
|
+
bal_text = cell(rec, columns.balance)
|
|
136
|
+
balance = parse_amount(bal_text) if bal_text else None
|
|
137
|
+
except (DateError, AmountError) as exc:
|
|
138
|
+
skipped.append((first, str(exc)))
|
|
139
|
+
continue
|
|
140
|
+
rows.append(
|
|
141
|
+
ImportedRow(
|
|
142
|
+
first_line=first,
|
|
143
|
+
last_line=last,
|
|
144
|
+
raw=raw,
|
|
145
|
+
when=when,
|
|
146
|
+
amount=amount,
|
|
147
|
+
description=cell(rec, columns.description),
|
|
148
|
+
counterparty=cell(rec, columns.counterparty),
|
|
149
|
+
currency=cell(rec, columns.currency) or None,
|
|
150
|
+
balance=balance,
|
|
151
|
+
fields={h: (rec[i] if i < len(rec) else "") for h, i in idx.items()},
|
|
152
|
+
)
|
|
153
|
+
)
|
|
154
|
+
return CsvResult(columns, rows, skipped)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _amount(rec: list[str], columns: ColumnMap, cell) -> Decimal:
|
|
158
|
+
if columns.amount:
|
|
159
|
+
return parse_amount(cell(rec, columns.amount))
|
|
160
|
+
debit, credit = cell(rec, columns.debit), cell(rec, columns.credit)
|
|
161
|
+
if not debit and not credit:
|
|
162
|
+
raise AmountError("row has neither debit nor credit")
|
|
163
|
+
total = Decimal(0)
|
|
164
|
+
if credit:
|
|
165
|
+
total += abs(parse_amount(credit))
|
|
166
|
+
if debit:
|
|
167
|
+
total -= abs(parse_amount(debit))
|
|
168
|
+
return total
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""OFX / QFX reader (both the 1.x SGML form and the 2.x XML form).
|
|
2
|
+
|
|
3
|
+
Each <STMTTRN>…</STMTTRN> block becomes one row. `raw` is the block exactly
|
|
4
|
+
as it appears in the file, so a receipt shows the bank's own record.
|
|
5
|
+
FITID, the bank's transaction id, is kept as `ref`; overlapping downloads of
|
|
6
|
+
the same account repeat it, which is how duplicates are recognised.
|
|
7
|
+
The statement's <LEDGERBAL> is kept so reconcile can check against it.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import html
|
|
13
|
+
import re
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from datetime import date
|
|
16
|
+
from decimal import Decimal
|
|
17
|
+
|
|
18
|
+
from ..money import AmountError, parse_amount
|
|
19
|
+
from .csv_rows import ImportedRow
|
|
20
|
+
|
|
21
|
+
_BLOCK = re.compile(r"<STMTTRN>.*?</STMTTRN>", re.S | re.I)
|
|
22
|
+
_LEDGER = re.compile(r"<LEDGERBAL>(.*?)</LEDGERBAL>", re.S | re.I)
|
|
23
|
+
_CURDEF = re.compile(r"<CURDEF>\s*([A-Za-z]{3})", re.I)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _tag(block: str, name: str) -> str:
|
|
27
|
+
m = re.search(rf"<{name}>([^<\r\n]*)", block, re.I)
|
|
28
|
+
return html.unescape(m.group(1).strip()) if m else ""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _ofx_date(value: str) -> date:
|
|
32
|
+
m = re.match(r"(\d{4})(\d{2})(\d{2})", value)
|
|
33
|
+
if not m:
|
|
34
|
+
raise ValueError(f"not an OFX date: {value!r}")
|
|
35
|
+
return date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def looks_like_ofx(text: str) -> bool:
|
|
39
|
+
head = text[:2048].upper()
|
|
40
|
+
return "OFXHEADER" in head or "<OFX>" in head
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class OfxResult:
|
|
45
|
+
rows: list[ImportedRow]
|
|
46
|
+
skipped: list[tuple[int, str]]
|
|
47
|
+
currency: str | None
|
|
48
|
+
ledger_balance: Decimal | None
|
|
49
|
+
ledger_as_of: date | None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def read_ofx(text: str) -> OfxResult:
|
|
53
|
+
currency = (m.group(1).upper() if (m := _CURDEF.search(text)) else None)
|
|
54
|
+
rows: list[ImportedRow] = []
|
|
55
|
+
skipped: list[tuple[int, str]] = []
|
|
56
|
+
for m in _BLOCK.finditer(text):
|
|
57
|
+
block = m.group(0)
|
|
58
|
+
first = text.count("\n", 0, m.start()) + 1
|
|
59
|
+
last = first + block.count("\n")
|
|
60
|
+
try:
|
|
61
|
+
when = _ofx_date(_tag(block, "DTPOSTED"))
|
|
62
|
+
amount = parse_amount(_tag(block, "TRNAMT"))
|
|
63
|
+
except (ValueError, AmountError) as exc:
|
|
64
|
+
skipped.append((first, str(exc)))
|
|
65
|
+
continue
|
|
66
|
+
name, memo = _tag(block, "NAME") or _tag(block, "PAYEE"), _tag(block, "MEMO")
|
|
67
|
+
rows.append(
|
|
68
|
+
ImportedRow(
|
|
69
|
+
first_line=first,
|
|
70
|
+
last_line=last,
|
|
71
|
+
raw=block,
|
|
72
|
+
when=when,
|
|
73
|
+
amount=amount,
|
|
74
|
+
description=" · ".join(x for x in (name, memo) if x) or _tag(block, "TRNTYPE"),
|
|
75
|
+
counterparty=name,
|
|
76
|
+
currency=_tag(block, "CURRENCY") or currency,
|
|
77
|
+
balance=None,
|
|
78
|
+
ref=_tag(block, "FITID"),
|
|
79
|
+
pos=m.start(),
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
balance = as_of = None
|
|
84
|
+
if lm := _LEDGER.search(text):
|
|
85
|
+
try:
|
|
86
|
+
balance = parse_amount(_tag(lm.group(1), "BALAMT"))
|
|
87
|
+
as_of = _ofx_date(_tag(lm.group(1), "DTASOF"))
|
|
88
|
+
except (ValueError, AmountError):
|
|
89
|
+
balance = as_of = None
|
|
90
|
+
return OfxResult(rows, skipped, currency, balance, as_of)
|