runningtab 0.1.0.dev3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. runningtab/__init__.py +3 -0
  2. runningtab/access.py +419 -0
  3. runningtab/books.py +225 -0
  4. runningtab/chart.py +408 -0
  5. runningtab/cli.py +2216 -0
  6. runningtab/coa.py +212 -0
  7. runningtab/doc.py +185 -0
  8. runningtab/forecast/__init__.py +1 -0
  9. runningtab/forecast/compute.py +224 -0
  10. runningtab/forecast/doctype.py +228 -0
  11. runningtab/forecast/model.py +251 -0
  12. runningtab/forecast/names.py +8 -0
  13. runningtab/forecast/sources.py +85 -0
  14. runningtab/gl_codes.py +50 -0
  15. runningtab/health.py +661 -0
  16. runningtab/mercury/__init__.py +7 -0
  17. runningtab/mercury/categories.py +500 -0
  18. runningtab/mercury/classify.py +101 -0
  19. runningtab/mercury/client.py +360 -0
  20. runningtab/mercury/ingest.py +733 -0
  21. runningtab/mercury/mapping.py +130 -0
  22. runningtab/mercury/models.py +544 -0
  23. runningtab/mercury/posting.py +757 -0
  24. runningtab/mercury/receipts.py +304 -0
  25. runningtab/mercury/session.py +65 -0
  26. runningtab/mercury/state.py +285 -0
  27. runningtab/mercury/vouchers.py +951 -0
  28. runningtab/period/__init__.py +9 -0
  29. runningtab/period/balances.py +283 -0
  30. runningtab/period/bank.py +154 -0
  31. runningtab/period/close.py +510 -0
  32. runningtab/period/crosscheck.py +257 -0
  33. runningtab/period/csvfmt.py +86 -0
  34. runningtab/period/documents.py +260 -0
  35. runningtab/period/fence.py +277 -0
  36. runningtab/period/ledger.py +205 -0
  37. runningtab/period/lock.py +294 -0
  38. runningtab/period/manifest.py +259 -0
  39. runningtab/period/open_items.py +84 -0
  40. runningtab/period/package.py +140 -0
  41. runningtab/period/qbo.py +521 -0
  42. runningtab/period/qbo_map.py +174 -0
  43. runningtab/period/quarter.py +90 -0
  44. runningtab/period/readiness.py +146 -0
  45. runningtab/period/record.py +247 -0
  46. runningtab/period/status.py +312 -0
  47. runningtab/site.py +409 -0
  48. runningtab-0.1.0.dev3.dist-info/METADATA +726 -0
  49. runningtab-0.1.0.dev3.dist-info/RECORD +52 -0
  50. runningtab-0.1.0.dev3.dist-info/WHEEL +4 -0
  51. runningtab-0.1.0.dev3.dist-info/entry_points.txt +2 -0
  52. runningtab-0.1.0.dev3.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,951 @@
1
+ """The draft voucher that sits beside a coded Bank Transaction.
2
+
3
+ Every settled transaction is already a submitted Bank Transaction -- the bank's
4
+ record. Reconciling one means opening ERPNext's create-voucher dialog, picking
5
+ the account, and confirming the amount, once per row. For any row a person has
6
+ coded in Mercury the account is already known: the GL code list the bank shows
7
+ is our own chart, exported by `runningtab coa export-gl-codes`, so
8
+ `generalLedgerCodeName` comes back naming a real account.
9
+
10
+ So this module drafts that voucher. **A proposal and nothing else.** It sets no
11
+ `docstatus` in any request, submits nothing, reconciles nothing, and deletes
12
+ nothing -- ERPNext's reconciliation query filters `je.docstatus == 1`, so an
13
+ unsubmitted voucher is not even offered as a match until a person submits it.
14
+ A draft somebody else has since edited is left alone and not reported: that is
15
+ a person taking the proposal over, which is the point.
16
+
17
+ The shape is ERPNext's own. `bank_reconciliation_tool.create_journal_entry_bts`
18
+ -- what the create-voucher dialog calls -- builds a `Bank Entry` whose
19
+ `cheque_no` is the statement line's reference and whose two lines are the Bank
20
+ Account's chart account and the account the person picked. The tool's matching
21
+ query ranks on `je.cheque_no == transaction.reference_number` and filters on
22
+ `jea.account == Bank Account.account`, so both of those have to be right or the
23
+ draft is invisible to the very screen it exists for.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import dataclasses
29
+ import datetime as dt
30
+ import enum
31
+ import re
32
+ from collections.abc import Iterable, Iterator
33
+ from decimal import Decimal
34
+ from typing import Any, Protocol
35
+
36
+ from icedcoffee_sync.apply.model import DesiredAccount
37
+
38
+ from runningtab.gl_codes import postable_accounts
39
+ from runningtab.mercury.classify import classify, is_internal_transfer
40
+ from runningtab.mercury.models import Transaction
41
+ from runningtab.mercury.posting import (
42
+ BANK_TRANSACTION,
43
+ JOURNAL_ENTRY,
44
+ TRANSACTION_ID_FIELD,
45
+ PostingContext,
46
+ PostingError,
47
+ )
48
+ from runningtab.period.quarter import Quarter
49
+
50
+ # ERPNext's own choice for a Bank Account that is not flagged `is_credit_card`,
51
+ # which is all of ours -- `install_bank_accounts` sets `account`, `company` and
52
+ # `is_company_account` and nothing else. `validate_cheque_info` makes
53
+ # `cheque_no` and `cheque_date` mandatory for this type, which is why both are
54
+ # always set below.
55
+ VOUCHER_TYPE = "Bank Entry"
56
+
57
+ # What ERPNext truncates a reference to when its own dialog builds one. Mercury
58
+ # ids and ACH trace numbers are far shorter; the cap is here so a longer
59
+ # identifier would be cut the same way rather than refused by the server.
60
+ REFERENCE_LIMIT = 140
61
+
62
+ # The Bank Transaction status that means a person has already matched the row
63
+ # to a voucher of their own. A draft still sitting beside it is clutter.
64
+ RECONCILED = "Reconciled"
65
+ # Frappe's docstatus for a cancelled document.
66
+ CANCELLED = 2
67
+ # More vouchers than this for one Mercury transaction is not an amendment
68
+ # chain; `find_existing` refuses rather than read a truncated page.
69
+ _VOUCHERS_PER_TRANSACTION = 20
70
+
71
+ # The two account families where a two-line entry against the bank account is
72
+ # the whole truth: a bill paid, and a member putting money in. Revenue is
73
+ # deliberately absent -- a customer payment clears a Sales Invoice through a
74
+ # Payment Entry, and a bare entry crediting revenue would book the income twice
75
+ # and leave the invoice outstanding.
76
+ EXPENSE_PREFIX = "5"
77
+ MEMBER_CAPITAL_PREFIX = "31"
78
+
79
+ # `"<number> - <name>"`, the spelling `runningtab coa export-gl-codes` writes and the
80
+ # bank reports back. Only the number is read: an account renamed in the chart
81
+ # after the list was uploaded is still the same account.
82
+ _GL_CODE = re.compile(r"^\s*(?P<number>\d+)\s*-\s*(?P<name>\S.*?)\s*$")
83
+
84
+ # The scalar fields a draft carries, compared before it is rewritten.
85
+ _SCALARS = (
86
+ "voucher_type",
87
+ "company",
88
+ "posting_date",
89
+ "cheque_no",
90
+ "cheque_date",
91
+ "title",
92
+ "remark",
93
+ "custom_remark",
94
+ TRANSACTION_ID_FIELD,
95
+ )
96
+
97
+
98
+ class Outcome(enum.StrEnum):
99
+ """What happened to one transaction's voucher."""
100
+
101
+ DRAFTED = "drafted"
102
+ UPDATED = "updated"
103
+ UNCHANGED = "unchanged"
104
+ # A person edited the draft. Theirs now; nothing is written and nothing is
105
+ # reported, because a report line every run would train them to ignore it.
106
+ TAKEN_OVER = "taken-over"
107
+ # Submitted or cancelled. A person committed it; not ours to touch.
108
+ COMMITTED = "committed"
109
+ # Its Bank Transaction is reconciled to something else, so the draft is no
110
+ # longer needed. Reported, never deleted.
111
+ UNNEEDED = "unneeded"
112
+ # Its Bank Transaction was reconciled before this ever drafted anything.
113
+ # The person has matched that statement line; there is nothing left to
114
+ # propose, so nothing is drafted and nothing is said.
115
+ MATCHED = "matched"
116
+
117
+
118
+ class Ledger(Protocol):
119
+ """The slice of `icedcoffee_sync.frappe.client.Client` this module uses.
120
+
121
+ `get` rather than a wider `list`, because the amounts and the accounts a
122
+ draft carries live in a child table that no field list returns.
123
+ """
124
+
125
+ def list(self, doctype: str, **kwargs: Any) -> Any: ...
126
+ def get(self, doctype: str, name: str) -> Any: ...
127
+ def insert(self, doctype: str, doc: dict[str, Any]) -> Any: ...
128
+ def update(self, doctype: str, name: str, doc: dict[str, Any]) -> Any: ...
129
+
130
+
131
+ @dataclasses.dataclass(frozen=True)
132
+ class Chart:
133
+ """The postable accounts of `chart.coa`, by number.
134
+
135
+ Read from the repository rather than from the site, and through the same
136
+ `postable_accounts` that produces the list uploaded to Mercury -- so the
137
+ vocabulary the bank offers and the vocabulary this checks against are one
138
+ thing by construction.
139
+ """
140
+
141
+ postable: dict[str, str]
142
+
143
+ @classmethod
144
+ def load(cls, accounts: Iterable[DesiredAccount]) -> Chart:
145
+ """The chart as this module reads it, from the books directory's parsed chart."""
146
+ chart = list(accounts)
147
+ return cls({account.number: account.name for account in postable_accounts(chart)})
148
+
149
+ def drafts_a_voucher(self, number: str) -> bool:
150
+ """Whether a two-line entry against the bank account describes this account completely."""
151
+ if number not in self.postable:
152
+ return False
153
+ return number.startswith(MEMBER_CAPITAL_PREFIX) or number.startswith(EXPENSE_PREFIX)
154
+
155
+
156
+ @dataclasses.dataclass(frozen=True)
157
+ class Unneeded:
158
+ """A draft Journal Entry of ours that nothing needs any more, which a person deletes.
159
+
160
+ Never deleted here: a person may have edited it. Carried as the document's
161
+ name rather than folded into prose so the report can link straight to it.
162
+ """
163
+
164
+ transaction_id: str
165
+ journal_entry: str
166
+ why: str
167
+
168
+
169
+ @dataclasses.dataclass
170
+ class VoucherReport:
171
+ """What one voucher pass did. Folded into the run's `IngestReport` by the caller.
172
+
173
+ Returned rather than written into `IngestReport` directly so this module
174
+ has no import back into `ingest`, and so its whole decision table is
175
+ exercisable without a run.
176
+ """
177
+
178
+ drafted: list[str] = dataclasses.field(default_factory=list)
179
+ updated: list[str] = dataclasses.field(default_factory=list)
180
+ planned: list[str] = dataclasses.field(default_factory=list)
181
+ attention: list[tuple[str, str]] = dataclasses.field(default_factory=list)
182
+ # A coded row dated on or before Books Closed Through. Its Bank
183
+ # Transaction still posts -- that happens before this module ever sees the
184
+ # row -- but no voucher is drafted or updated beside it: ERPNext would
185
+ # refuse one dated in a locked quarter on submit. A person reopens that
186
+ # quarter, books it there, and closes it again. See `_reopen_advice`.
187
+ closed: list[str] = dataclasses.field(default_factory=list)
188
+ unneeded: list[Unneeded] = dataclasses.field(default_factory=list)
189
+ # Rows Mercury has no code for that the ledger accounts for anyway: a
190
+ # voucher already carries their Mercury id, their Bank Transaction is
191
+ # reconciled, or their payee's history drafted one. The run stops calling
192
+ # these `uncoded`, because nobody has to code them any more.
193
+ handled_in_ledger: set[str] = dataclasses.field(default_factory=set)
194
+
195
+
196
+ def coded_account(transaction: Transaction, chart: Chart) -> tuple[str | None, str | None]:
197
+ """The chart account this transaction's confirmed GL code names, or why it drafts nothing.
198
+
199
+ `(number, None)` when a voucher should be drafted. `(None, None)` when the
200
+ row simply is not one this drafts -- no code at all, which the run already
201
+ reports as `uncoded`, or a code outside the two account families, which a
202
+ person picked deliberately and which rides to the reconciliation screen on
203
+ the Bank Transaction either way. `(None, the code)` when the code itself is
204
+ the problem, which is one problem for the whole run rather than one per row.
205
+
206
+ Read from `Transaction.gl_code` -- `generalLedgerCodeName`, else the
207
+ person's custom category, which is a GL-code label -- and never from
208
+ `glAllocations`: the
209
+ allocations are what Mercury's own rule engine derived, not what a person
210
+ confirmed, and a split allocation is a shape this voucher does not model.
211
+ """
212
+ code = transaction.gl_code
213
+ if not code:
214
+ return None, None
215
+ match = _GL_CODE.match(code)
216
+ if match is None:
217
+ return None, code
218
+ number = match["number"]
219
+ if number not in chart.postable:
220
+ return None, code
221
+ if not chart.drafts_a_voucher(number):
222
+ return None, None
223
+ return number, None
224
+
225
+
226
+ def funding_account(transaction: Transaction, ctx: PostingContext) -> str:
227
+ """The chart account holding the Mercury account this transaction moved through.
228
+
229
+ The same `[accounts]` entry `bank_account_documents` links a Bank Account
230
+ to, so the voucher's bank line and the Bank Account's chart account are the
231
+ same account -- which is what the reconciliation tool filters on.
232
+ """
233
+ number = ctx.mapping.account_for_mercury_account(
234
+ transaction.account_id, ctx.mercury_account_names.get(transaction.account_id)
235
+ )
236
+ if number is None:
237
+ raise PostingError(
238
+ f"Mercury account {transaction.account_id} has no [accounts] entry in mercury.toml; "
239
+ "a voucher's bank line has to be the chart account holding that balance"
240
+ )
241
+ return number
242
+
243
+
244
+ def reference_number(transaction: Transaction) -> str:
245
+ """The reference a voucher and its Bank Transaction are matched on.
246
+
247
+ The ACH trace number where the transaction has one, because that is what a
248
+ statement export carries as `Tracking ID` and what a hand-imported row
249
+ therefore holds; the Mercury id otherwise. Composed from the same two
250
+ fields the Bank Transaction's `reference_number` is, and it has to stay
251
+ that way: the tool's Journal Entry query ranks candidates on
252
+ `je.cheque_no == transaction.reference_number`.
253
+ """
254
+ return (transaction.tracking_number or transaction.id)[:REFERENCE_LIMIT]
255
+
256
+
257
+ # The separator between the parts of a remark, matching the one a Bank
258
+ # Transaction's description uses, so the two read as one voice.
259
+ _REMARK_SEPARATOR = " · "
260
+
261
+
262
+ def _payee(transaction: Transaction) -> str | None:
263
+ """Who the money went to, or came from. None when Mercury names no counterparty.
264
+
265
+ Named on the voucher because a reviewer looking at a list of drafts sees
266
+ the remark and nothing else; without it, telling one `5204` entry from the
267
+ next means opening the Bank Transaction behind each.
268
+ """
269
+ if not transaction.counterparty_name:
270
+ return None
271
+ direction = "paid to" if transaction.amount < 0 else "received from"
272
+ return f"{direction} {transaction.counterparty_name}"
273
+
274
+
275
+ def _remark(transaction: Transaction, code: str | None) -> str:
276
+ """What the person reviewing the draft reads: their note, their code, the payee, and the row it came from."""
277
+ parts = (transaction.note, code, _payee(transaction), f"mercury {transaction.id}")
278
+ return _REMARK_SEPARATOR.join(part for part in parts if part)
279
+
280
+
281
+ def _title(transaction: Transaction, code: str | None) -> str:
282
+ """What the Journal Entry list shows for a draft: who was paid, and what it was coded as."""
283
+ return _REMARK_SEPARATOR.join(part for part in (transaction.counterparty_name, code) if part)
284
+
285
+
286
+ def _amount(value: Decimal) -> float:
287
+ """A ledger amount as JSON. The one place `Decimal` becomes a float, after every decision is made."""
288
+ return float(value)
289
+
290
+
291
+ def journal_entry(
292
+ transaction: Transaction,
293
+ number: str,
294
+ ctx: PostingContext,
295
+ *,
296
+ code: str | None = None,
297
+ copied_from: str | None = None,
298
+ ) -> dict[str, Any]:
299
+ """The draft voucher for one coded transaction, without a `docstatus` of any kind.
300
+
301
+ `code` is what the row was coded as, when that is not the transaction's
302
+ own GL code, and `copied_from` the submitted voucher a payee's history
303
+ took it from; the remark names both.
304
+
305
+ The description goes in `remark` with `custom_remark` set, and the list
306
+ title is the payee and the code. ERPNext's `user_remark` on Journal Entry
307
+ is hidden -- no list, report or form shows it -- and without
308
+ `custom_remark` ERPNext rebuilds `remark` from the reference on every
309
+ save. Left unset, `title` falls back to the first account line, which made
310
+ every draft in the list read as the bank account the money moved through.
311
+
312
+ Spend debits the coded account and credits the account the money left; a
313
+ refund is the same entry with the sides swapped, which falls out of the
314
+ sign rather than needing its own branch.
315
+
316
+ Only `debit_in_account_currency` and `credit_in_account_currency` are
317
+ written. ERPNext derives `debit` and `credit` from them on validate
318
+ (`set_amounts_in_company_currency`), so stating both would be restating a
319
+ number the server computes.
320
+
321
+ No party on either line: member identity is the equity account itself, and
322
+ ERPNext rejects a Shareholder party on an equity line.
323
+ """
324
+ label = code if code is not None else transaction.gl_code
325
+ remark_code = f"{label} as {copied_from}" if label and copied_from else label
326
+ coded = ctx.link(number)
327
+ bank = ctx.link(funding_account(transaction, ctx))
328
+ posting_date = transaction.accounting_date.isoformat()
329
+ magnitude = abs(transaction.amount)
330
+ outflow = transaction.amount < 0
331
+ bank_line: dict[str, Any] = {
332
+ "account": bank,
333
+ # The dialog's own shape. The matching query reads `account`, but a
334
+ # voucher a person opens should show the Bank Account link theirs does.
335
+ "bank_account": ctx.bank_account(transaction),
336
+ "debit_in_account_currency": 0 if outflow else _amount(magnitude),
337
+ "credit_in_account_currency": _amount(magnitude) if outflow else 0,
338
+ }
339
+ coded_line: dict[str, Any] = {
340
+ "account": coded,
341
+ "debit_in_account_currency": _amount(magnitude) if outflow else 0,
342
+ "credit_in_account_currency": 0 if outflow else _amount(magnitude),
343
+ }
344
+ return {
345
+ "voucher_type": VOUCHER_TYPE,
346
+ "company": ctx.company,
347
+ "posting_date": posting_date,
348
+ "cheque_no": reference_number(transaction),
349
+ "cheque_date": posting_date,
350
+ "title": _title(transaction, label),
351
+ "remark": _remark(transaction, remark_code),
352
+ "custom_remark": 1,
353
+ TRANSACTION_ID_FIELD: transaction.id,
354
+ "accounts": [bank_line, coded_line],
355
+ }
356
+
357
+
358
+ def find_existing(ledger: Ledger, transaction_id: str) -> dict[str, Any] | None:
359
+ """The voucher standing for a Mercury transaction: the live one, else a cancelled one, else none.
360
+
361
+ A person corrects a submitted voucher by cancelling and amending it, and
362
+ ERPNext's Amend copies the Mercury id onto the amendment, so one id can
363
+ name a cancelled original and its live replacement. The cancelled ones are
364
+ history. Only a second *live* voucher is a duplicate, and the field cannot
365
+ be `unique` to stop one -- see `posting.custom_field_documents` -- so this
366
+ lookup is what refuses it.
367
+
368
+ With nothing live, a cancelled voucher is still returned: a person
369
+ cancelling ours without replacing it is their decision about the row, and
370
+ the callers leave it alone rather than drafting it again.
371
+ """
372
+ rows = ledger.list(
373
+ JOURNAL_ENTRY,
374
+ fields=["name", "docstatus", "modified_by"],
375
+ filters=[[TRANSACTION_ID_FIELD, "=", transaction_id]],
376
+ limit=_VOUCHERS_PER_TRANSACTION,
377
+ )
378
+ if len(rows) >= _VOUCHERS_PER_TRANSACTION:
379
+ # A full page cannot say whether a live voucher sits beyond it.
380
+ raise PostingError(
381
+ f"{JOURNAL_ENTRY} has {_VOUCHERS_PER_TRANSACTION} or more documents for Mercury transaction "
382
+ f"{transaction_id}; a person looks at why before ingesting again"
383
+ )
384
+ live = [row for row in rows if int(row.get("docstatus") or 0) != CANCELLED]
385
+ if len(live) > 1:
386
+ # Rewriting one of them at random leaves the other standing.
387
+ names = ", ".join(str(row.get("name")) for row in live)
388
+ raise PostingError(
389
+ f"{JOURNAL_ENTRY} has more than one live document for Mercury transaction {transaction_id} ({names}); "
390
+ f"a person cancels or deletes the duplicate before ingesting again"
391
+ )
392
+ if live:
393
+ return live[0]
394
+ return rows[0] if rows else None
395
+
396
+
397
+ def bank_transaction_reconciled(ledger: Ledger, transaction_id: str) -> bool:
398
+ """Whether this transaction's Bank Transaction has already been matched to a voucher."""
399
+ rows = ledger.list(
400
+ BANK_TRANSACTION,
401
+ fields=["name", "status"],
402
+ filters=[[TRANSACTION_ID_FIELD, "=", transaction_id]],
403
+ limit=1,
404
+ )
405
+ return bool(rows) and str(rows[0].get("status") or "") == RECONCILED
406
+
407
+
408
+ def _text(value: Any) -> str:
409
+ """A field as text for comparison. Frappe returns dates as strings and unset values as null."""
410
+ return "" if value is None else str(value)
411
+
412
+
413
+ def _rows(doc: dict[str, Any]) -> list[tuple[str, Decimal, Decimal]]:
414
+ """A voucher's lines as (account, debit, credit), exactly, for comparison."""
415
+ return [
416
+ (
417
+ _text(row.get("account")),
418
+ Decimal(_text(row.get("debit_in_account_currency") or 0)),
419
+ Decimal(_text(row.get("credit_in_account_currency") or 0)),
420
+ )
421
+ for row in doc.get("accounts") or ()
422
+ ]
423
+
424
+
425
+ def differences(current: dict[str, Any], doc: dict[str, Any]) -> list[str]:
426
+ """Which fields of a draft the run would now write differently.
427
+
428
+ Read back whole rather than by field list: a recoded GL code changes an
429
+ account and an amount that settled differently changes a number, and both
430
+ of those live in the child table.
431
+ """
432
+ changed = [field for field in _SCALARS if _text(current.get(field)) != _text(doc[field])]
433
+ if _rows(current) != _rows(doc):
434
+ changed.append("accounts")
435
+ return changed
436
+
437
+
438
+ def plan_voucher(ledger: Ledger, transaction: Transaction, doc: dict[str, Any], bot_user: str) -> tuple[Outcome, str]:
439
+ """Decide what to do with one transaction's voucher, writing nothing.
440
+
441
+ The order matters. A submitted or cancelled voucher and one a person has
442
+ edited are both theirs, and neither is reported -- the second is a person
443
+ taking the proposal over, which is what the proposal is for. Only once a
444
+ draft is established as ours does the question of whether it is still
445
+ wanted, or still right, arise.
446
+
447
+ A row whose Bank Transaction is already reconciled gets nothing new. The
448
+ person has matched that statement line to a voucher of their own, so
449
+ drafting one now would be clutter, and clutter this would then report as
450
+ unneeded on every later run. A dry run plans the draft anyway, because a
451
+ dry run has posted no Bank Transaction for this to find.
452
+ """
453
+ existing = find_existing(ledger, transaction.id)
454
+ if existing is None:
455
+ if bank_transaction_reconciled(ledger, transaction.id):
456
+ return Outcome.MATCHED, ""
457
+ return Outcome.DRAFTED, ""
458
+ name = str(existing["name"])
459
+ if int(existing.get("docstatus") or 0) != 0:
460
+ return Outcome.COMMITTED, name
461
+ if _text(existing.get("modified_by")) != bot_user:
462
+ return Outcome.TAKEN_OVER, name
463
+ if bank_transaction_reconciled(ledger, transaction.id):
464
+ return Outcome.UNNEEDED, name
465
+ if not differences(ledger.get(JOURNAL_ENTRY, name), doc):
466
+ return Outcome.UNCHANGED, name
467
+ return Outcome.UPDATED, name
468
+
469
+
470
+ def _record(
471
+ ledger: Ledger,
472
+ transaction: Transaction,
473
+ doc: dict[str, Any],
474
+ report: VoucherReport,
475
+ *,
476
+ bot_user: str,
477
+ dry_run: bool,
478
+ ) -> None:
479
+ """Carry out the plan for one transaction, or say what it would have been."""
480
+ outcome, name = plan_voucher(ledger, transaction, doc, bot_user)
481
+ if outcome is Outcome.UNNEEDED:
482
+ report.unneeded.append(Unneeded(transaction.id, str(name), "its Bank Transaction is reconciled"))
483
+ return
484
+ if outcome is Outcome.DRAFTED:
485
+ if dry_run:
486
+ report.planned.append(f"draft {JOURNAL_ENTRY} for {transaction.id}")
487
+ return
488
+ created = ledger.insert(JOURNAL_ENTRY, doc)
489
+ report.drafted.append(str(created.get("name", "")) if isinstance(created, dict) else "")
490
+ return
491
+ if outcome is Outcome.UPDATED:
492
+ if dry_run:
493
+ report.planned.append(f"update {JOURNAL_ENTRY} {name} for {transaction.id}")
494
+ return
495
+ ledger.update(JOURNAL_ENTRY, name, doc)
496
+ report.updated.append(name)
497
+
498
+
499
+ class PayeeHistory:
500
+ """The one account a person has booked a payee to, read from the vouchers they submitted.
501
+
502
+ A payee is Mercury's `counterpartyId`, which every row from one merchant
503
+ shares. History counts only a submitted voucher carrying a Mercury id: a
504
+ draft is a proposal nobody has agreed to, and a cancelled voucher is one
505
+ somebody took back. It answers only when every such voucher went to the
506
+ same single account. A payee booked to several -- a person reimbursed for
507
+ many kinds of spend -- has no answer, because copying whichever came last
508
+ would be a guess wearing a precedent.
509
+
510
+ Read lazily, once per payee per run, and only for payees with an uncoded
511
+ row: each answer costs a lookup per row of that payee's.
512
+ """
513
+
514
+ def __init__(self, ledger: Ledger, transactions: list[Transaction], ctx: PostingContext, chart: Chart) -> None:
515
+ self._ledger = ledger
516
+ self._transactions = transactions
517
+ self._ctx = ctx
518
+ self._chart = chart
519
+ self._numbers = {name: number for number, name in ctx.account_names.items()}
520
+ self._answers: dict[str, tuple[str, str] | None] = {}
521
+
522
+ def account_for(self, payee: str | None) -> tuple[str, str] | None:
523
+ """`(chart number, the latest voucher booked to it)` for this payee, or None when history cannot say."""
524
+ if payee is None:
525
+ return None
526
+ if payee not in self._answers:
527
+ self._answers[payee] = self._read(payee)
528
+ return self._answers[payee]
529
+
530
+ def _read(self, payee: str) -> tuple[str, str] | None:
531
+ booked: list[tuple[str, str, str]] = []
532
+ for transaction in self._transactions:
533
+ if transaction.counterparty_id != payee:
534
+ continue
535
+ existing = find_existing(self._ledger, transaction.id)
536
+ if existing is None or int(existing.get("docstatus") or 0) != 1:
537
+ continue
538
+ name = str(existing["name"])
539
+ voucher = self._ledger.get(JOURNAL_ENTRY, name)
540
+ bank = self._ctx.link(funding_account(transaction, self._ctx))
541
+ others = {_text(row.get("account")) for row in voucher.get("accounts") or ()} - {bank}
542
+ numbers = {self._numbers.get(account) for account in others}
543
+ if len(numbers) != 1 or None in numbers:
544
+ # A split voucher, or an account outside the chart: no single answer.
545
+ return None
546
+ booked.append((_text(voucher.get("posting_date")), name, numbers.pop() or ""))
547
+ if not booked or len({number for _, _, number in booked}) != 1:
548
+ return None
549
+ _, latest, number = max(booked)
550
+ if number not in self._chart.postable:
551
+ return None
552
+ return number, latest
553
+
554
+
555
+ def _draft_from_history(
556
+ ledger: Ledger,
557
+ transaction: Transaction,
558
+ ctx: PostingContext,
559
+ chart: Chart,
560
+ history: PayeeHistory,
561
+ report: VoucherReport,
562
+ *,
563
+ dry_run: bool,
564
+ ) -> None:
565
+ """For a row Mercury has no code for: leave a ledger decision alone, else repeat the payee's.
566
+
567
+ A voucher already carrying the row's Mercury id is a person's coding in
568
+ the ledger, or a history draft from an earlier run; either way it is
569
+ neither rewritten nor reported. A cancelled one is neither, and leaves the
570
+ row uncoded. A reconciled Bank Transaction is a person
571
+ having decided where the row goes. Unlike a Mercury code, history drafts
572
+ any postable account, revenue included: it repeats a voucher a person
573
+ already submitted for this payee -- card cashback booked to income -- so
574
+ the invoice concern that keeps a revenue code from drafting does not apply.
575
+ """
576
+ # Looked up first even when the row is reconciled: it is also what refuses
577
+ # two vouchers carrying one Mercury id.
578
+ existing = find_existing(ledger, transaction.id)
579
+ if bank_transaction_reconciled(ledger, transaction.id):
580
+ report.handled_in_ledger.add(transaction.id)
581
+ return
582
+ if existing is not None and int(existing.get("docstatus") or 0) == CANCELLED:
583
+ # Cancelled: it booked nothing, and its Mercury id still blocks a new
584
+ # draft. The row stays uncoded, which is the honest report.
585
+ return
586
+ if existing is not None:
587
+ report.handled_in_ledger.add(transaction.id)
588
+ return
589
+ learned = history.account_for(transaction.counterparty_id)
590
+ if learned is None:
591
+ return
592
+ number, source = learned
593
+ report.handled_in_ledger.add(transaction.id)
594
+ label = f"{number} - {chart.postable[number]}"
595
+ closed_through = ctx.lock.closed_through
596
+ if closed_through is not None and transaction.accounting_date <= closed_through:
597
+ _report_closed_row(ledger, transaction, label, closed_through, report, ctx.bot_user)
598
+ return
599
+ try:
600
+ doc = journal_entry(transaction, number, ctx, code=label, copied_from=source)
601
+ except PostingError as exc:
602
+ report.attention.append((transaction.id, str(exc)))
603
+ return
604
+ _record(ledger, transaction, doc, report, bot_user=ctx.bot_user, dry_run=dry_run)
605
+
606
+
607
+ # How far apart the two halves of one transfer may be dated. They settle the
608
+ # same day in every pair observed; the slack absorbs a weekend, and is far
609
+ # short of the month between two autopays of the same amount.
610
+ TRANSFER_WINDOW_DAYS = 5
611
+
612
+
613
+ def transfer_pairs(
614
+ transactions: Iterable[Transaction], own_account_ids: Iterable[str]
615
+ ) -> tuple[list[tuple[Transaction, Transaction]], list[tuple[Transaction, str]]]:
616
+ """Pair the two statement lines of each movement between our own accounts.
617
+
618
+ Returns `(out, in)` pairs -- the half the money left, then the half it
619
+ arrived on -- and every settled half that could not be paired, with why.
620
+ Two halves pair only when each is the other's one and only match: the
621
+ same amount with opposite signs, each naming the other's account as its
622
+ counterparty, dated within `TRANSFER_WINDOW_DAYS`. Anything else is
623
+ reported rather than guessed, because a wrong pair books a payment
624
+ against the wrong statement line.
625
+ """
626
+ own = frozenset(own_account_ids)
627
+ halves = [t for t in transactions if classify(t).posts and is_internal_transfer(t, own)]
628
+
629
+ def partners(half: Transaction) -> list[Transaction]:
630
+ return [
631
+ other
632
+ for other in halves
633
+ if other.account_id == half.counterparty_id
634
+ and other.counterparty_id == half.account_id
635
+ and other.amount == -half.amount
636
+ and abs((other.accounting_date - half.accounting_date).days) <= TRANSFER_WINDOW_DAYS
637
+ ]
638
+
639
+ pairs: list[tuple[Transaction, Transaction]] = []
640
+ unpaired: list[tuple[Transaction, str]] = []
641
+ for half in halves:
642
+ found = partners(half)
643
+ if len(found) == 1 and partners(found[0]) == [half]:
644
+ if half.amount < 0:
645
+ pairs.append((half, found[0]))
646
+ elif not found:
647
+ unpaired.append((half, "no matching other half on the account it names"))
648
+ else:
649
+ unpaired.append((half, f"{len(found)} possible other halves, so none is assumed"))
650
+ return pairs, unpaired
651
+
652
+
653
+ def transfer_entry(out: Transaction, into: Transaction, ctx: PostingContext) -> dict[str, Any]:
654
+ """The draft voucher both halves of one transfer are matched to.
655
+
656
+ Debits the account the money arrived on and credits the one it left --
657
+ for a card payment, the card's liability falls and checking falls with it.
658
+ Identified by the outgoing half's Mercury id, so the unique field keeps it
659
+ to one voucher; the remark names both halves.
660
+ """
661
+ magnitude = abs(out.amount)
662
+ posting_date = out.accounting_date.isoformat()
663
+ destination = f"Transfer to {out.counterparty_name}" if out.counterparty_name else "Transfer"
664
+ remark = _REMARK_SEPARATOR.join(
665
+ part for part in (out.note, f"transfer {magnitude:.2f}", f"mercury {out.id}", f"mercury {into.id}") if part
666
+ )
667
+ return {
668
+ "voucher_type": VOUCHER_TYPE,
669
+ "company": ctx.company,
670
+ "posting_date": posting_date,
671
+ "cheque_no": reference_number(out),
672
+ "cheque_date": posting_date,
673
+ "title": f"{destination} · {magnitude:.2f}",
674
+ "remark": remark,
675
+ "custom_remark": 1,
676
+ TRANSACTION_ID_FIELD: out.id,
677
+ "accounts": [
678
+ {
679
+ "account": ctx.link(funding_account(into, ctx)),
680
+ "bank_account": ctx.bank_account(into),
681
+ "debit_in_account_currency": _amount(magnitude),
682
+ "credit_in_account_currency": 0,
683
+ },
684
+ {
685
+ "account": ctx.link(funding_account(out, ctx)),
686
+ "bank_account": ctx.bank_account(out),
687
+ "debit_in_account_currency": 0,
688
+ "credit_in_account_currency": _amount(magnitude),
689
+ },
690
+ ],
691
+ }
692
+
693
+
694
+ def _draft_transfers(
695
+ ledger: Ledger,
696
+ transactions: list[Transaction],
697
+ ctx: PostingContext,
698
+ report: VoucherReport,
699
+ *,
700
+ own_account_ids: Iterable[str],
701
+ dry_run: bool,
702
+ ) -> None:
703
+ """Draft one voucher per paired transfer, and report every half that could not be paired."""
704
+ pairs, unpaired = transfer_pairs(transactions, own_account_ids)
705
+ for half, why in unpaired:
706
+ existing = find_existing(ledger, half.id)
707
+ draft = (
708
+ f"; its earlier draft {existing['name']} was paired before and should be checked before anyone submits it"
709
+ if existing is not None and int(existing.get("docstatus") or 0) == 0
710
+ else "; no transfer voucher is drafted until both halves pair"
711
+ )
712
+ report.attention.append((
713
+ half.id,
714
+ f"transfer of {abs(half.amount):.2f} dated {half.accounting_date.isoformat()}: {why}{draft}",
715
+ ))
716
+ closed_through = ctx.lock.closed_through
717
+ for out, into in pairs:
718
+ if closed_through is not None and min(out.accounting_date, into.accounting_date) <= closed_through:
719
+ _report_closed_transfer(ledger, out, into, closed_through, report)
720
+ continue
721
+ if _booked_by_a_person(ledger, out, into, report):
722
+ continue
723
+ try:
724
+ doc = transfer_entry(out, into, ctx)
725
+ except PostingError as exc:
726
+ report.attention.append((out.id, str(exc)))
727
+ continue
728
+ _record(ledger, out, doc, report, bot_user=ctx.bot_user, dry_run=dry_run)
729
+
730
+
731
+ def _report_closed_transfer(
732
+ ledger: Ledger, out: Transaction, into: Transaction, closed_through: dt.date, report: VoucherReport
733
+ ) -> None:
734
+ """A transfer with either half on or before Books Closed Through: never drafted, reported until settled.
735
+
736
+ One rule: report it until both halves are booked -- reconciled, or covered
737
+ by our transfer voucher once submitted -- and no draft of ours remains.
738
+ The line says which half is booked and names any leftover draft, and
739
+ promises no future draft: a person decides how a closed quarter is
740
+ corrected, and a leftover draft could otherwise be submitted to book the
741
+ movement again.
742
+ """
743
+ existing = find_existing(ledger, out.id)
744
+ status = int(existing.get("docstatus") or 0) if existing is not None else None
745
+ ours_submitted = status == 1
746
+ halves = [(half.id, ours_submitted or bank_transaction_reconciled(ledger, half.id)) for half in (out, into)]
747
+ leftover = str(existing["name"]) if existing is not None and status == 0 else None
748
+ if all(booked for _, booked in halves) and leftover is None:
749
+ return
750
+ states = ", ".join(f"{half_id} {'booked' if booked else 'not booked'}" for half_id, booked in halves)
751
+ quarter = Quarter.containing(min(out.accounting_date, into.accounting_date))
752
+ advice = (
753
+ f"reopen {quarter.label} in ERPNext (move Books Closed Through back to {quarter.previous.end.isoformat()}), "
754
+ "book a transfer voucher by hand and match each unbooked half to it, then run "
755
+ f"`runningtab period close --quarter {quarter.label}` again"
756
+ )
757
+ if closed_through > quarter.end:
758
+ advice += " and re-close every closed quarter after it, in order"
759
+ if leftover is not None:
760
+ advice += f"; draft {JOURNAL_ENTRY} {leftover} was made before the close and a person deletes it"
761
+ report.closed.append(
762
+ f"{out.id} {out.accounting_date.isoformat()} transfer of {abs(out.amount):.2f}: closed period — "
763
+ f"{states}; {advice}"
764
+ )
765
+
766
+
767
+ def _booked_by_a_person(ledger: Ledger, out: Transaction, into: Transaction, report: VoucherReport) -> bool:
768
+ """Whether a person reconciled a half to something other than our voucher; reported, and nothing drafted.
769
+
770
+ A reconciled half can only be matched to a submitted voucher, so a half
771
+ reconciled while ours is absent or still a draft was booked by something
772
+ else. Ours would book the movement twice.
773
+ """
774
+ existing = find_existing(ledger, out.id)
775
+ if bank_transaction_reconciled(ledger, into.id):
776
+ if existing is None:
777
+ report.attention.append((
778
+ out.id,
779
+ f"transfer of {abs(out.amount):.2f}: its incoming half ({into.id}) is already reconciled, "
780
+ "so no transfer voucher is drafted; match this half to the same voucher",
781
+ ))
782
+ return True
783
+ if int(existing.get("docstatus") or 0) == 0:
784
+ report.unneeded.append(
785
+ Unneeded(out.id, str(existing["name"]), f"its incoming half {into.id} is reconciled to another voucher")
786
+ )
787
+ return True
788
+ return False
789
+ if existing is None and bank_transaction_reconciled(ledger, out.id):
790
+ report.attention.append((
791
+ out.id,
792
+ f"transfer of {abs(out.amount):.2f}: this half is reconciled to a voucher of a person's own, "
793
+ f"so no transfer voucher is drafted; match its incoming half ({into.id}) to the same voucher",
794
+ ))
795
+ return True
796
+ return False
797
+
798
+
799
+ def candidates(transactions: Iterable[Transaction], own_account_ids: Iterable[str]) -> Iterator[Transaction]:
800
+ """The settled, non-transfer rows a voucher could be drafted for.
801
+
802
+ A transfer between two of our own accounts is one economic event seen
803
+ twice, and a voucher per leg would double-count a balance-sheet movement.
804
+ The reconciliation tool already matches the pair as a transfer.
805
+ """
806
+ own = frozenset(own_account_ids)
807
+ for transaction in transactions:
808
+ if classify(transaction).posts and not is_internal_transfer(transaction, own):
809
+ yield transaction
810
+
811
+
812
+ @dataclasses.dataclass(frozen=True)
813
+ class _ClosedRow:
814
+ """Where a row dated in a closed quarter stands: booked or not, and any draft of ours it left behind."""
815
+
816
+ booked: bool
817
+ leftover_draft: str | None = None
818
+
819
+
820
+ def _closed_row(ledger: Ledger, transaction_id: str, bot_user: str) -> _ClosedRow:
821
+ """Whether a row dated in a closed quarter already reached the ledger before the close.
822
+
823
+ That is every row a quarter was closed with: its voucher submitted
824
+ (docstatus 1 -- a cancelled one, 2, booked nothing), or its Bank
825
+ Transaction reconciled against a voucher of the person's own. The voucher
826
+ pass sweeps every transaction on every run, so only a row with neither --
827
+ late bank activity, or a draft left unsubmitted when the quarter closed --
828
+ is the person's to reopen for. A reconciled row can still carry a draft
829
+ this module proposed and nobody took over; that draft is clutter a person
830
+ deletes, the same as `Outcome.UNNEEDED` in an open quarter.
831
+ """
832
+ existing = find_existing(ledger, transaction_id)
833
+ status = int(existing.get("docstatus") or 0) if existing is not None else None
834
+ if status == 1:
835
+ return _ClosedRow(booked=True)
836
+ if not bank_transaction_reconciled(ledger, transaction_id):
837
+ return _ClosedRow(booked=False)
838
+ ours = existing is not None and status == 0 and _text(existing.get("modified_by")) == bot_user
839
+ return _ClosedRow(booked=True, leftover_draft=str(existing["name"]) if ours and existing is not None else None)
840
+
841
+
842
+ def _report_closed_row(
843
+ ledger: Ledger,
844
+ transaction: Transaction,
845
+ account: str,
846
+ closed_through: dt.date,
847
+ report: VoucherReport,
848
+ bot_user: str,
849
+ ) -> None:
850
+ """Report a coded row dated on or before Books Closed Through, if it is the person's to act on.
851
+
852
+ The world as it is, not an error (spec's "Late bank activity"). No draft
853
+ here, ever, and no update to one drafted before the quarter closed: an
854
+ existing draft beside a now-closed row would only be a proposal ERPNext
855
+ refuses the moment somebody submits it, and this run is not the one to
856
+ decide what replaces it. A row booked before the close is left alone.
857
+ """
858
+ try:
859
+ row = _closed_row(ledger, transaction.id, bot_user)
860
+ except PostingError as exc:
861
+ report.attention.append((transaction.id, str(exc)))
862
+ return
863
+ if row.leftover_draft is not None:
864
+ report.unneeded.append(Unneeded(transaction.id, row.leftover_draft, "its Bank Transaction is reconciled"))
865
+ if row.booked:
866
+ return
867
+ report.closed.append(
868
+ f"{transaction.id} {transaction.accounting_date.isoformat()} {account}: "
869
+ f"closed period — {_reopen_advice(transaction.accounting_date, closed_through)}"
870
+ )
871
+
872
+
873
+ def _reopen_advice(accounting_date: dt.date, closed_through: dt.date) -> str:
874
+ """How a row dated in a closed quarter reaches the ledger: reopen that quarter, book it there, close it again.
875
+
876
+ Booking it in the open quarter instead would leave the closed quarter's
877
+ rebuilt `bank-reconciliation.csv` different from its stored package for
878
+ good, so `period status` could never pass again. Reopening moves Books
879
+ Closed Through back, which is also what lets the next sync draft the
880
+ voucher this run withheld; closing again stores a new revision, and any
881
+ closed quarter after it is re-closed after it, in order.
882
+ """
883
+ quarter = Quarter.containing(accounting_date)
884
+ advice = (
885
+ f"reopen {quarter.label} in ERPNext (move Books Closed Through back to {quarter.previous.end.isoformat()}), "
886
+ "book it there (the next sync drafts its voucher), and run "
887
+ f"`runningtab period close --quarter {quarter.label}` again, which stores a new revision"
888
+ )
889
+ if closed_through > quarter.end:
890
+ advice += "; then re-close every closed quarter after it, in order"
891
+ return advice
892
+
893
+
894
+ def draft_vouchers(
895
+ transactions: Iterable[Transaction],
896
+ ctx: PostingContext,
897
+ ledger: Ledger,
898
+ *,
899
+ own_account_ids: Iterable[str],
900
+ dry_run: bool = False,
901
+ ) -> VoucherReport:
902
+ """Draft, or bring up to date, the voucher every coded row should have.
903
+
904
+ Sweep-shaped, and it has to be, for the reason the GL-code hint sweep
905
+ already is: **coding a transaction at Mercury emits no event and bumps no
906
+ `resourceVersion`**, so a pass that waited for the log would never hear
907
+ about the one field it keys on. It adds no Mercury read -- the run has
908
+ already fetched every transaction for the balance invariant.
909
+
910
+ A GL code the chart cannot place is reported **once for the run**, naming
911
+ every such code. The cause is one thing, the list in Mercury having drifted
912
+ from `chart.coa`, and a line per row would bury it.
913
+ """
914
+ report = VoucherReport()
915
+ chart = ctx.chart
916
+ transactions = list(transactions)
917
+ history = PayeeHistory(ledger, transactions, ctx, chart)
918
+ drifted: list[str] = []
919
+ for transaction in candidates(transactions, own_account_ids):
920
+ number, unplaceable = coded_account(transaction, chart)
921
+ if unplaceable is not None:
922
+ if unplaceable not in drifted:
923
+ drifted.append(unplaceable)
924
+ continue
925
+ if number is None:
926
+ # A code that drafts nothing (revenue) is a person's decision too;
927
+ # history is only for a row nobody coded.
928
+ if transaction.gl_code is None:
929
+ _draft_from_history(ledger, transaction, ctx, chart, history, report, dry_run=dry_run)
930
+ continue
931
+ closed_through = ctx.lock.closed_through
932
+ if closed_through is not None and transaction.accounting_date <= closed_through:
933
+ _report_closed_row(
934
+ ledger, transaction, f"{number} - {chart.postable[number]}", closed_through, report, ctx.bot_user
935
+ )
936
+ continue
937
+ try:
938
+ doc = journal_entry(transaction, number, ctx)
939
+ except PostingError as exc:
940
+ report.attention.append((transaction.id, str(exc)))
941
+ continue
942
+ _record(ledger, transaction, doc, report, bot_user=ctx.bot_user, dry_run=dry_run)
943
+ _draft_transfers(ledger, transactions, ctx, report, own_account_ids=own_account_ids, dry_run=dry_run)
944
+ if drifted:
945
+ report.attention.append((
946
+ "gl codes",
947
+ f"{', '.join(repr(code) for code in drifted)} name no postable account in chart.coa, so the rows "
948
+ "carrying them get no draft voucher; re-run `runningtab coa export-gl-codes` and re-upload the list "
949
+ "to Mercury, or add the account to the chart",
950
+ ))
951
+ return report