runningtab 0.1.0.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runningtab/__init__.py +3 -0
- runningtab/access.py +419 -0
- runningtab/books.py +225 -0
- runningtab/chart.py +408 -0
- runningtab/cli.py +2216 -0
- runningtab/coa.py +212 -0
- runningtab/doc.py +185 -0
- runningtab/forecast/__init__.py +1 -0
- runningtab/forecast/compute.py +224 -0
- runningtab/forecast/doctype.py +228 -0
- runningtab/forecast/model.py +251 -0
- runningtab/forecast/names.py +8 -0
- runningtab/forecast/sources.py +85 -0
- runningtab/gl_codes.py +50 -0
- runningtab/health.py +661 -0
- runningtab/mercury/__init__.py +7 -0
- runningtab/mercury/categories.py +500 -0
- runningtab/mercury/classify.py +101 -0
- runningtab/mercury/client.py +360 -0
- runningtab/mercury/ingest.py +733 -0
- runningtab/mercury/mapping.py +130 -0
- runningtab/mercury/models.py +544 -0
- runningtab/mercury/posting.py +757 -0
- runningtab/mercury/receipts.py +304 -0
- runningtab/mercury/session.py +65 -0
- runningtab/mercury/state.py +285 -0
- runningtab/mercury/vouchers.py +951 -0
- runningtab/period/__init__.py +9 -0
- runningtab/period/balances.py +283 -0
- runningtab/period/bank.py +154 -0
- runningtab/period/close.py +510 -0
- runningtab/period/crosscheck.py +257 -0
- runningtab/period/csvfmt.py +86 -0
- runningtab/period/documents.py +260 -0
- runningtab/period/fence.py +277 -0
- runningtab/period/ledger.py +205 -0
- runningtab/period/lock.py +294 -0
- runningtab/period/manifest.py +259 -0
- runningtab/period/open_items.py +84 -0
- runningtab/period/package.py +140 -0
- runningtab/period/qbo.py +521 -0
- runningtab/period/qbo_map.py +174 -0
- runningtab/period/quarter.py +90 -0
- runningtab/period/readiness.py +146 -0
- runningtab/period/record.py +247 -0
- runningtab/period/status.py +312 -0
- runningtab/site.py +409 -0
- runningtab-0.1.0.dev3.dist-info/METADATA +726 -0
- runningtab-0.1.0.dev3.dist-info/RECORD +52 -0
- runningtab-0.1.0.dev3.dist-info/WHEEL +4 -0
- runningtab-0.1.0.dev3.dist-info/entry_points.txt +2 -0
- runningtab-0.1.0.dev3.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
"""The receipt Mercury holds, attached to the Bank Transaction it belongs to.
|
|
2
|
+
|
|
3
|
+
A Mercury transaction carries the receipts somebody uploaded against it as
|
|
4
|
+
`attachments`, each a presigned S3 URL good for twelve hours. This module
|
|
5
|
+
fetches them and attaches them, private, to that transaction's **Bank
|
|
6
|
+
Transaction** -- the bank's record of the row, which exists for every settled
|
|
7
|
+
transaction, rather than the draft voucher, which exists only for the coded
|
|
8
|
+
ones and which a person may take over or delete. A reviewer reading a draft is
|
|
9
|
+
one hop away: its `remark` ends with the Mercury id the receipt hangs off.
|
|
10
|
+
|
|
11
|
+
Identity is the attachment UUID out of the S3 path, carried into the `File` as
|
|
12
|
+
`<uuid>-<fileName>`, so a run can compare what is already on the row against
|
|
13
|
+
what Mercury reports without downloading anything. A missing one is fetched and
|
|
14
|
+
uploaded; one already there is left alone.
|
|
15
|
+
|
|
16
|
+
**Nothing here deletes or replaces a `File`, ever** -- not one this wrote, and
|
|
17
|
+
not one it did not. A receipt a person can see on the bank's record is their
|
|
18
|
+
record, no re-run reproduces it (the URL behind it expired), and Mercury
|
|
19
|
+
dropping an attachment is not a reason to take a document away from somebody.
|
|
20
|
+
A `File` on the row with no UUID prefix is somebody else's and is left alone
|
|
21
|
+
without a report line, for the reason the voucher pass says nothing about a
|
|
22
|
+
draft a person has edited.
|
|
23
|
+
|
|
24
|
+
The fetch is a GET to S3, not to Mercury's API host, which is why it lives here
|
|
25
|
+
and not on `MercuryClient`: that class's promise is that the only verb it can
|
|
26
|
+
send is GET against one base URL, and an honest promise is better than one with
|
|
27
|
+
an asterisk on it.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import dataclasses
|
|
33
|
+
import re
|
|
34
|
+
from collections.abc import Iterable
|
|
35
|
+
|
|
36
|
+
import httpx2
|
|
37
|
+
from icedcoffee_sync.frappe.errors import FrappeError, PermissionError_, RateLimitError
|
|
38
|
+
|
|
39
|
+
from runningtab.mercury.models import Attachment, Transaction
|
|
40
|
+
from runningtab.mercury.posting import BANK_TRANSACTION, Ledger, PostingError, find_existing, plain_message
|
|
41
|
+
|
|
42
|
+
FILE = "File"
|
|
43
|
+
|
|
44
|
+
# What one receipt may weigh. A scanned invoice is a few hundred kilobytes and
|
|
45
|
+
# a phone photograph a few megabytes; twenty is generous for both and small
|
|
46
|
+
# enough that a run cannot be made to buffer something absurd. The stream is
|
|
47
|
+
# measured as it arrives and abandoned at the limit, so an enormous file costs
|
|
48
|
+
# this much traffic and not its own size.
|
|
49
|
+
RECEIPT_SIZE_LIMIT = 20 * 1024 * 1024
|
|
50
|
+
|
|
51
|
+
# Frappe stores `File.file_name` in a `Data` column, which is 140 characters.
|
|
52
|
+
# A longer name would be refused by the server, or silently cut somewhere we
|
|
53
|
+
# did not choose.
|
|
54
|
+
FILE_NAME_LIMIT = 140
|
|
55
|
+
|
|
56
|
+
_UUID = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
|
|
57
|
+
|
|
58
|
+
# The `<uuid>-` prefix this module writes, read back off a `File` that is
|
|
59
|
+
# already on the row. Anchored and case-insensitive; a name that does not match
|
|
60
|
+
# was not written here.
|
|
61
|
+
_OURS = re.compile(rf"^(?P<id>{_UUID})-", re.IGNORECASE)
|
|
62
|
+
|
|
63
|
+
# What Frappe itself strips from a `file_name` on save (`File.set_file_name`
|
|
64
|
+
# removes `/`). The newline is ours: it would break the multipart header this
|
|
65
|
+
# name rides in long before Frappe ever saw it.
|
|
66
|
+
_UNSAFE = re.compile(r"[/\r\n]")
|
|
67
|
+
|
|
68
|
+
# The name given to an attachment whose own is empty once sanitised. It never
|
|
69
|
+
# stands alone -- the UUID prefix is what identifies the file -- so it only has
|
|
70
|
+
# to be a word rather than a blank.
|
|
71
|
+
_UNNAMED = "receipt"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class ReceiptError(RuntimeError):
|
|
75
|
+
"""A receipt could not be fetched. One report line; the run goes on."""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class ReceiptTooLargeError(ReceiptError):
|
|
79
|
+
"""The receipt is past `RECEIPT_SIZE_LIMIT` and was abandoned mid-stream."""
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclasses.dataclass
|
|
83
|
+
class ReceiptReport:
|
|
84
|
+
"""What one receipt pass did. Folded into the run's `IngestReport` by the caller.
|
|
85
|
+
|
|
86
|
+
Returned rather than written into `IngestReport` directly so this module
|
|
87
|
+
has no import back into `ingest`, and so its whole decision table is
|
|
88
|
+
exercisable without a run.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
# `<file name> -> <Bank Transaction name>`, one per upload.
|
|
92
|
+
attached: list[str] = dataclasses.field(default_factory=list)
|
|
93
|
+
planned: list[str] = dataclasses.field(default_factory=list)
|
|
94
|
+
attention: list[tuple[str, str]] = dataclasses.field(default_factory=list)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _trimmed(name: str, limit: int) -> str:
|
|
98
|
+
"""A file name cut to `limit`, keeping its extension.
|
|
99
|
+
|
|
100
|
+
The extension is what tells a person -- and Frappe's own PDF scan -- what
|
|
101
|
+
the file is, so it survives and the middle of the name goes.
|
|
102
|
+
"""
|
|
103
|
+
if len(name) <= limit:
|
|
104
|
+
return name
|
|
105
|
+
stem, dot, extension = name.rpartition(".")
|
|
106
|
+
if dot and len(extension) + 1 < limit:
|
|
107
|
+
return f"{stem[: limit - len(extension) - 1]}.{extension}"
|
|
108
|
+
return name[:limit]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def file_name_for(attachment: Attachment, identity: str) -> str:
|
|
112
|
+
"""The `File.file_name` one attachment becomes: `<uuid>-<sanitised fileName>`.
|
|
113
|
+
|
|
114
|
+
Sanitised the way Frappe would sanitise it, so the name sent is the name
|
|
115
|
+
that comes back and the comparison on the next run is against what we
|
|
116
|
+
wrote. The UUID prefix is fixed-width hex and dashes, so the trim to the
|
|
117
|
+
column's 140 characters can never reach it.
|
|
118
|
+
"""
|
|
119
|
+
name = _UNSAFE.sub("", attachment.file_name).strip() or _UNNAMED
|
|
120
|
+
return f"{identity}-{_trimmed(name, FILE_NAME_LIMIT - len(identity) - 1)}"
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def attachment_id_of(file_name: str) -> str | None:
|
|
124
|
+
"""The attachment UUID a `File` we wrote carries, or None for one we did not."""
|
|
125
|
+
match = _OURS.match(file_name or "")
|
|
126
|
+
return match["id"].lower() if match else None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class ReceiptFetcher:
|
|
130
|
+
"""Downloads one presigned receipt URL, streamed, with a size cap.
|
|
131
|
+
|
|
132
|
+
Its own client rather than the Mercury one: this reaches S3 with the
|
|
133
|
+
signature in the query string, not Mercury's API host with a bearer token,
|
|
134
|
+
and the two have no header, base URL or timeout in common.
|
|
135
|
+
|
|
136
|
+
A context manager, so the connection pool closes with the run that opened
|
|
137
|
+
it. Redirects are followed because S3 answers a cross-region read with one.
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
def __init__(
|
|
141
|
+
self,
|
|
142
|
+
*,
|
|
143
|
+
transport: httpx2.BaseTransport | None = None,
|
|
144
|
+
timeout: float = 60.0,
|
|
145
|
+
size_limit: int = RECEIPT_SIZE_LIMIT,
|
|
146
|
+
) -> None:
|
|
147
|
+
self.size_limit = size_limit
|
|
148
|
+
self._http = httpx2.Client(timeout=timeout, transport=transport, follow_redirects=True)
|
|
149
|
+
|
|
150
|
+
def __enter__(self) -> ReceiptFetcher:
|
|
151
|
+
return self
|
|
152
|
+
|
|
153
|
+
def __exit__(self, *exc: object) -> None:
|
|
154
|
+
self.close()
|
|
155
|
+
|
|
156
|
+
def close(self) -> None:
|
|
157
|
+
"""Release the underlying connection pool."""
|
|
158
|
+
self._http.close()
|
|
159
|
+
|
|
160
|
+
def fetch(self, url: str) -> bytes:
|
|
161
|
+
"""The receipt's bytes, or `ReceiptError` describing why there are none.
|
|
162
|
+
|
|
163
|
+
Everything that can go wrong with one download comes back as one
|
|
164
|
+
exception type, because the caller does the same thing with all of
|
|
165
|
+
them: name the row, name the file, and carry on with the rest of the
|
|
166
|
+
run. Ending the run instead would cost every other receipt in it, and
|
|
167
|
+
the URLs expire twelve hours after Mercury minted them.
|
|
168
|
+
"""
|
|
169
|
+
try:
|
|
170
|
+
with self._http.stream("GET", url) as response:
|
|
171
|
+
if response.status_code >= 400:
|
|
172
|
+
response.read()
|
|
173
|
+
raise ReceiptError(f"the receipt URL answered {response.status_code}: {response.text[:200]}")
|
|
174
|
+
chunks: list[bytes] = []
|
|
175
|
+
size = 0
|
|
176
|
+
for chunk in response.iter_bytes():
|
|
177
|
+
size += len(chunk)
|
|
178
|
+
if size > self.size_limit:
|
|
179
|
+
raise ReceiptTooLargeError(
|
|
180
|
+
f"the receipt is larger than the {self.size_limit} byte limit; "
|
|
181
|
+
"a person downloads it from Mercury and attaches it"
|
|
182
|
+
)
|
|
183
|
+
chunks.append(chunk)
|
|
184
|
+
except httpx2.RequestError as exc:
|
|
185
|
+
raise ReceiptError(f"the receipt could not be fetched: {exc!r}") from exc
|
|
186
|
+
return b"".join(chunks)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def receipts_on(ledger: Ledger, name: str) -> dict[str, str]:
|
|
190
|
+
"""The attachment UUIDs already on one Bank Transaction, mapped to their `File` names.
|
|
191
|
+
|
|
192
|
+
Paged with `iter_all` rather than one capped `list`: a read cut short at a
|
|
193
|
+
page limit looks exactly like "that receipt is not attached yet", which is
|
|
194
|
+
the answer that uploads a second copy.
|
|
195
|
+
|
|
196
|
+
A `File` whose name carries no UUID prefix is not in the result and is not
|
|
197
|
+
reported. Somebody attached it, or another integration did; it is not this
|
|
198
|
+
module's business either way.
|
|
199
|
+
"""
|
|
200
|
+
have: dict[str, str] = {}
|
|
201
|
+
for row in ledger.iter_all(
|
|
202
|
+
FILE,
|
|
203
|
+
fields=["name", "file_name"],
|
|
204
|
+
filters=[["attached_to_doctype", "=", BANK_TRANSACTION], ["attached_to_name", "=", name]],
|
|
205
|
+
):
|
|
206
|
+
identity = attachment_id_of(str(row.get("file_name") or ""))
|
|
207
|
+
if identity is not None:
|
|
208
|
+
have[identity] = str(row.get("name", ""))
|
|
209
|
+
return have
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _attach(
|
|
213
|
+
ledger: Ledger,
|
|
214
|
+
transaction: Transaction,
|
|
215
|
+
attachment: Attachment,
|
|
216
|
+
row: str,
|
|
217
|
+
have: dict[str, str],
|
|
218
|
+
report: ReceiptReport,
|
|
219
|
+
*,
|
|
220
|
+
fetcher: ReceiptFetcher,
|
|
221
|
+
dry_run: bool,
|
|
222
|
+
) -> None:
|
|
223
|
+
"""Put one attachment on one Bank Transaction, or say why it is not there.
|
|
224
|
+
|
|
225
|
+
A refusal of this upload is a report line and the run goes on, mirroring
|
|
226
|
+
`ingest._write`. A site that is rate limiting us and a credential it will
|
|
227
|
+
not accept are each one problem the whole run has rather than one file's,
|
|
228
|
+
so they propagate: a line per receipt would bury the single real problem.
|
|
229
|
+
"""
|
|
230
|
+
identity = attachment.attachment_id
|
|
231
|
+
if identity is None:
|
|
232
|
+
report.attention.append((
|
|
233
|
+
transaction.id,
|
|
234
|
+
f"the receipt {attachment.file_name!r} has no id in its URL path, so no run can tell it from one "
|
|
235
|
+
"already attached and every run would attach it again; a person downloads it from Mercury",
|
|
236
|
+
))
|
|
237
|
+
return
|
|
238
|
+
if identity in have:
|
|
239
|
+
return
|
|
240
|
+
name = file_name_for(attachment, identity)
|
|
241
|
+
if dry_run:
|
|
242
|
+
report.planned.append(f"attach {name} to {BANK_TRANSACTION} {row} for {transaction.id}")
|
|
243
|
+
return
|
|
244
|
+
try:
|
|
245
|
+
content = fetcher.fetch(attachment.url)
|
|
246
|
+
except ReceiptError as exc:
|
|
247
|
+
report.attention.append((transaction.id, f"receipt {name}: {exc}"))
|
|
248
|
+
return
|
|
249
|
+
try:
|
|
250
|
+
# `is_private` stated rather than inherited: a receipt is financial
|
|
251
|
+
# evidence and has no business behind a world-readable URL.
|
|
252
|
+
ledger.upload_file(BANK_TRANSACTION, row, name, content, is_private=True)
|
|
253
|
+
except RateLimitError, PermissionError_:
|
|
254
|
+
raise
|
|
255
|
+
except FrappeError as exc:
|
|
256
|
+
report.attention.append((
|
|
257
|
+
transaction.id,
|
|
258
|
+
f"attaching {name} to {BANK_TRANSACTION} {row} refused by the ledger: {plain_message(str(exc))}",
|
|
259
|
+
))
|
|
260
|
+
return
|
|
261
|
+
have[identity] = name
|
|
262
|
+
report.attached.append(f"{name} -> {row}")
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def attach_receipts(
|
|
266
|
+
transactions: Iterable[Transaction],
|
|
267
|
+
ledger: Ledger,
|
|
268
|
+
*,
|
|
269
|
+
fetcher: ReceiptFetcher,
|
|
270
|
+
dry_run: bool = False,
|
|
271
|
+
) -> ReceiptReport:
|
|
272
|
+
"""Attach every Mercury receipt its Bank Transaction does not already carry.
|
|
273
|
+
|
|
274
|
+
Sweep-shaped, and it has to be, for the reason the GL-code hint sweep and
|
|
275
|
+
the voucher pass already are: **the attachment list emits no event.** That
|
|
276
|
+
is an assumption rather than a measurement -- nobody has captured an event
|
|
277
|
+
for an attachment upload -- but the two neighbouring fields we do know
|
|
278
|
+
about, the GL code and the note, provably emit none, and if an event does
|
|
279
|
+
turn out to fire the only cost is that this comparison finds nothing new.
|
|
280
|
+
It adds no Mercury read: the run already fetched every transaction for the
|
|
281
|
+
balance invariant.
|
|
282
|
+
|
|
283
|
+
Two rows are skipped in silence. One with **no Bank Transaction** has
|
|
284
|
+
nothing to attach to, and a settled row that failed to post has already
|
|
285
|
+
produced its own report line from the Bank Transaction path. One whose Bank
|
|
286
|
+
Transaction is **cancelled** was cancelled by a person deliberately;
|
|
287
|
+
whatever replaces it is a new row, and the receipt lands there.
|
|
288
|
+
"""
|
|
289
|
+
report = ReceiptReport()
|
|
290
|
+
for transaction in transactions:
|
|
291
|
+
if not transaction.attachments:
|
|
292
|
+
continue
|
|
293
|
+
try:
|
|
294
|
+
existing = find_existing(ledger, transaction.id)
|
|
295
|
+
except PostingError as exc:
|
|
296
|
+
report.attention.append((transaction.id, str(exc)))
|
|
297
|
+
continue
|
|
298
|
+
if existing is None or int(existing.get("docstatus") or 0) == 2:
|
|
299
|
+
continue
|
|
300
|
+
row = str(existing["name"])
|
|
301
|
+
have = receipts_on(ledger, row)
|
|
302
|
+
for attachment in transaction.attachments:
|
|
303
|
+
_attach(ledger, transaction, attachment, row, have, report, fetcher=fetcher, dry_run=dry_run)
|
|
304
|
+
return report
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Assembling a `PostingContext` from a live site, the chart, and the mapping.
|
|
2
|
+
|
|
3
|
+
Kept out of `runningtab.cli` so the wiring is testable without a terminal, and out of
|
|
4
|
+
`posting.py` so that module stays pure enough to exercise against captured
|
|
5
|
+
payloads alone.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from icedcoffee_sync.apply.model import DesiredAccount
|
|
11
|
+
from icedcoffee_sync.erpnext.accounts import read_chart
|
|
12
|
+
from icedcoffee_sync.frappe.client import Client
|
|
13
|
+
|
|
14
|
+
from runningtab.coa import sole_company
|
|
15
|
+
from runningtab.mercury.client import MercuryClient
|
|
16
|
+
from runningtab.mercury.mapping import Mapping, MappingError, validate_against_chart
|
|
17
|
+
from runningtab.mercury.posting import PostingContext
|
|
18
|
+
from runningtab.mercury.vouchers import Chart
|
|
19
|
+
from runningtab.period.lock import read_visible_lock
|
|
20
|
+
from runningtab.site import Profile
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def posting_context(
|
|
24
|
+
ledger: Client,
|
|
25
|
+
*,
|
|
26
|
+
mapping: Mapping,
|
|
27
|
+
chart: list[DesiredAccount],
|
|
28
|
+
bot_user: str,
|
|
29
|
+
mercury: MercuryClient | None = None,
|
|
30
|
+
company: str | None = None,
|
|
31
|
+
) -> PostingContext:
|
|
32
|
+
"""Everything a Bank Transaction needs that is not on the transaction.
|
|
33
|
+
|
|
34
|
+
Account document names come from the **live site**, not from rendering the
|
|
35
|
+
chart: an account's document name derives from its number and title, so a
|
|
36
|
+
site whose chart has not been applied yet would otherwise produce links that
|
|
37
|
+
look right and resolve to nothing.
|
|
38
|
+
|
|
39
|
+
Books Closed Through is read here too, alongside Company -- the one read
|
|
40
|
+
this module already makes -- so the voucher pass can leave a closed
|
|
41
|
+
quarter's rows to a person. A bot that cannot see the permlevel-1 lock
|
|
42
|
+
fields (never re-provisioned with its Company read grant, or a provision
|
|
43
|
+
that installed the fence and then failed) is refused by name, before
|
|
44
|
+
anything is posted: Frappe deletes the fields rather than returning them
|
|
45
|
+
empty, so reading on would treat a closed quarter as open. Nothing else
|
|
46
|
+
would catch that. ERPNext checks the frozen date only when GL entries
|
|
47
|
+
are made, on submit, and the ingest saves its vouchers as drafts.
|
|
48
|
+
"""
|
|
49
|
+
unknown = validate_against_chart(mapping, {a.number for a in chart})
|
|
50
|
+
if unknown:
|
|
51
|
+
raise MappingError(f"mercury.toml names accounts the chart does not have: {', '.join(unknown)}")
|
|
52
|
+
company = company or sole_company(ledger)
|
|
53
|
+
# First, so a bot that cannot see the lock is refused before any other read.
|
|
54
|
+
lock = read_visible_lock(ledger, company, Profile.BOT)
|
|
55
|
+
account_names = {a.number: a.document_name for a in read_chart(ledger, company) if a.number}
|
|
56
|
+
mercury_names = {a.id: a.name for a in mercury.accounts()} if mercury is not None else {}
|
|
57
|
+
return PostingContext(
|
|
58
|
+
company=company,
|
|
59
|
+
account_names=account_names,
|
|
60
|
+
mapping=mapping,
|
|
61
|
+
bot_user=bot_user,
|
|
62
|
+
chart=Chart.load(chart),
|
|
63
|
+
mercury_account_names=mercury_names,
|
|
64
|
+
lock=lock,
|
|
65
|
+
)
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
"""Where the ingest keeps its place in the event log.
|
|
2
|
+
|
|
3
|
+
Three things live here and nothing else: the cursor, the set of event ids
|
|
4
|
+
already applied, and the highest `resourceVersion` seen per resource. SQLite
|
|
5
|
+
rather than a JSON document because the applier commits after each event -- a
|
|
6
|
+
run that dies halfway keeps everything it already applied, instead of losing a
|
|
7
|
+
batch that was going to be written out at the end.
|
|
8
|
+
|
|
9
|
+
What deliberately does **not** live here is which ERPNext document was posted
|
|
10
|
+
for a Mercury transaction. That lookup is the `mercury_transaction_id` custom
|
|
11
|
+
field on the document itself, so losing this file can never cause a double
|
|
12
|
+
post: it costs a re-read, not a duplicate entry.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import dataclasses
|
|
18
|
+
import datetime as dt
|
|
19
|
+
import os
|
|
20
|
+
import pathlib
|
|
21
|
+
import sqlite3
|
|
22
|
+
|
|
23
|
+
# Mercury retains events for 90 days. A cursor older than that may point into a
|
|
24
|
+
# hole the log can no longer fill, and resuming from it would skip whatever
|
|
25
|
+
# happened in the gap without anything looking wrong.
|
|
26
|
+
RETENTION = dt.timedelta(days=90)
|
|
27
|
+
|
|
28
|
+
_SCHEMA = """
|
|
29
|
+
CREATE TABLE IF NOT EXISTS ingest_identity (
|
|
30
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
31
|
+
identity TEXT NOT NULL
|
|
32
|
+
);
|
|
33
|
+
CREATE TABLE IF NOT EXISTS ingest_cursor (
|
|
34
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
35
|
+
event_id TEXT,
|
|
36
|
+
synced_at TEXT NOT NULL
|
|
37
|
+
);
|
|
38
|
+
CREATE TABLE IF NOT EXISTS seen_event (
|
|
39
|
+
event_id TEXT PRIMARY KEY,
|
|
40
|
+
applied_at TEXT NOT NULL
|
|
41
|
+
);
|
|
42
|
+
CREATE TABLE IF NOT EXISTS resource_version (
|
|
43
|
+
resource_id TEXT PRIMARY KEY,
|
|
44
|
+
version INTEGER NOT NULL
|
|
45
|
+
);
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class NoCursorError(RuntimeError):
|
|
50
|
+
"""Nothing has been snapshotted yet, so there is no place in the log to resume from."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class StaleCursorError(RuntimeError):
|
|
54
|
+
"""The cursor is older than Mercury's retention window; re-snapshot rather than resume."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class StateIdentityError(RuntimeError):
|
|
58
|
+
"""This state file belongs to a different Mercury deployment or ledger."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclasses.dataclass(frozen=True)
|
|
62
|
+
class Cursor:
|
|
63
|
+
"""Where the last run stopped, and when it stopped there.
|
|
64
|
+
|
|
65
|
+
`synced_at` is our clock, not the event's. The retention rule needs to know
|
|
66
|
+
how long ago we last read the log, which is a fact about us; the event's own
|
|
67
|
+
timestamp field was never recorded by the investigation, and guessing its
|
|
68
|
+
name to answer a question we can already answer would be a worse trade.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
event_id: str | None
|
|
72
|
+
synced_at: dt.datetime
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def default_state_path(books_name: str, env: str, target: str) -> pathlib.Path:
|
|
76
|
+
"""`$XDG_STATE_HOME/<name>-books/mercury-<env>-<target>.sqlite3`, or the `~/.local/state` fallback.
|
|
77
|
+
|
|
78
|
+
One file per set of books, Mercury deployment and ledger, never one shared
|
|
79
|
+
file. A cursor is a place in one bank's event log, and the rows it
|
|
80
|
+
accounts for are in one ledger; a snapshot taken against the sandbox and a
|
|
81
|
+
test site would otherwise look to the production feed like a place it can
|
|
82
|
+
resume from, and everything before that cursor would be skipped with
|
|
83
|
+
nothing looking wrong. `books_name` is `books.toml`'s name, lowercased.
|
|
84
|
+
"""
|
|
85
|
+
root = os.environ.get("XDG_STATE_HOME")
|
|
86
|
+
base = pathlib.Path(root) if root else pathlib.Path.home() / ".local" / "state"
|
|
87
|
+
return base / f"{books_name.lower()}-books" / f"mercury-{env}-{target}.sqlite3"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class IngestState:
|
|
91
|
+
"""The ingest's place in the log. One writer, committed per call.
|
|
92
|
+
|
|
93
|
+
`identity` names the Mercury deployment and ledger this file belongs to,
|
|
94
|
+
as `"<env>/<target>"`. It is recorded when an empty file is first used and
|
|
95
|
+
checked on every later open, because `--state` can point at any file the
|
|
96
|
+
namespaced default path was arranged to keep apart. A file that already
|
|
97
|
+
holds a cursor but names no deployment is refused rather than adopted:
|
|
98
|
+
there is nothing to check against, and a file predating the binding is
|
|
99
|
+
exactly one that may have been written by another deployment. Omitting the
|
|
100
|
+
identity -- which only tests do -- binds nothing and checks nothing.
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
def __init__(self, path: pathlib.Path, *, identity: str | None = None) -> None:
|
|
104
|
+
self.path = path
|
|
105
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
106
|
+
self._db = sqlite3.connect(path)
|
|
107
|
+
self._db.executescript(_SCHEMA)
|
|
108
|
+
self._db.commit()
|
|
109
|
+
if identity is not None:
|
|
110
|
+
self._bind(identity)
|
|
111
|
+
|
|
112
|
+
def _is_empty(self) -> bool:
|
|
113
|
+
"""Whether this file has nothing to lose: no cursor, no applied events, no versions."""
|
|
114
|
+
return all(
|
|
115
|
+
self._db.execute(query).fetchone() is None
|
|
116
|
+
for query in (
|
|
117
|
+
"SELECT 1 FROM ingest_cursor WHERE id = 1",
|
|
118
|
+
"SELECT 1 FROM seen_event LIMIT 1",
|
|
119
|
+
"SELECT 1 FROM resource_version LIMIT 1",
|
|
120
|
+
)
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
def _bind(self, identity: str) -> None:
|
|
124
|
+
"""Claim an empty file, or refuse one that belongs to -- or might belong to -- another ingest."""
|
|
125
|
+
row = self._db.execute("SELECT identity FROM ingest_identity WHERE id = 1").fetchone()
|
|
126
|
+
if row is not None and row[0] != identity:
|
|
127
|
+
self.close()
|
|
128
|
+
raise StateIdentityError(
|
|
129
|
+
f"{self.path} holds the state of {row[0]} and this run is {identity}; "
|
|
130
|
+
"a cursor is a place in one bank's log and its rows are in one ledger, so resuming "
|
|
131
|
+
"from the wrong file would skip everything before an event this deployment never emitted"
|
|
132
|
+
)
|
|
133
|
+
if row is not None:
|
|
134
|
+
return
|
|
135
|
+
if not self._is_empty():
|
|
136
|
+
# Claiming a populated file would bless whatever cursor it holds as
|
|
137
|
+
# this deployment's, which is the confusion the binding exists to
|
|
138
|
+
# prevent -- and a file predating the binding is exactly one that
|
|
139
|
+
# may have been written against another deployment.
|
|
140
|
+
self.close()
|
|
141
|
+
raise StateIdentityError(
|
|
142
|
+
f"{self.path} already holds ingest state but names no deployment, so there is "
|
|
143
|
+
f"nothing to check {identity} against; snapshot into a fresh file rather than "
|
|
144
|
+
"adopting one whose bank and ledger cannot be established"
|
|
145
|
+
)
|
|
146
|
+
self._db.execute("INSERT INTO ingest_identity (id, identity) VALUES (1, ?)", (identity,))
|
|
147
|
+
self._db.commit()
|
|
148
|
+
|
|
149
|
+
def __enter__(self) -> IngestState:
|
|
150
|
+
return self
|
|
151
|
+
|
|
152
|
+
def __exit__(self, *exc: object) -> None:
|
|
153
|
+
self.close()
|
|
154
|
+
|
|
155
|
+
def close(self) -> None:
|
|
156
|
+
"""Close the database handle."""
|
|
157
|
+
self._db.close()
|
|
158
|
+
|
|
159
|
+
# --- cursor -----------------------------------------------------------
|
|
160
|
+
|
|
161
|
+
def cursor(self) -> Cursor | None:
|
|
162
|
+
"""Where the last snapshot or sync left off, or None if there has never been one."""
|
|
163
|
+
row = self._db.execute("SELECT event_id, synced_at FROM ingest_cursor WHERE id = 1").fetchone()
|
|
164
|
+
if row is None:
|
|
165
|
+
return None
|
|
166
|
+
return Cursor(event_id=row[0], synced_at=dt.datetime.fromisoformat(row[1]))
|
|
167
|
+
|
|
168
|
+
def record_cursor(self, event_id: str | None, at: dt.datetime) -> None:
|
|
169
|
+
"""Move the cursor to `event_id` as of `at`.
|
|
170
|
+
|
|
171
|
+
A None `event_id` is a real state, not a missing one: an account whose
|
|
172
|
+
event log is empty has been read completely, and the next sync starts
|
|
173
|
+
from the beginning of the log rather than refusing.
|
|
174
|
+
"""
|
|
175
|
+
self._db.execute(
|
|
176
|
+
"INSERT INTO ingest_cursor (id, event_id, synced_at) VALUES (1, ?, ?) "
|
|
177
|
+
"ON CONFLICT(id) DO UPDATE SET event_id = excluded.event_id, synced_at = excluded.synced_at",
|
|
178
|
+
(event_id, at.isoformat()),
|
|
179
|
+
)
|
|
180
|
+
self._db.commit()
|
|
181
|
+
|
|
182
|
+
def require_resumable(self, now: dt.datetime) -> Cursor:
|
|
183
|
+
"""The cursor, if resuming from it can still be complete.
|
|
184
|
+
|
|
185
|
+
Raises rather than resuming into a hole: past the retention window the
|
|
186
|
+
log cannot tell us what we missed, and the honest recovery is another
|
|
187
|
+
snapshot.
|
|
188
|
+
"""
|
|
189
|
+
cursor = self.cursor()
|
|
190
|
+
if cursor is None:
|
|
191
|
+
raise NoCursorError("no cursor recorded; run `runningtab mercury snapshot` before syncing")
|
|
192
|
+
age = now - cursor.synced_at
|
|
193
|
+
if age > RETENTION:
|
|
194
|
+
raise StaleCursorError(
|
|
195
|
+
f"last sync was {age.days} days ago and Mercury retains events for {RETENTION.days}; "
|
|
196
|
+
"the cursor may point into a gap the log can no longer fill -- re-snapshot instead of resuming"
|
|
197
|
+
)
|
|
198
|
+
return cursor
|
|
199
|
+
|
|
200
|
+
# --- dedup ------------------------------------------------------------
|
|
201
|
+
|
|
202
|
+
def has_seen(self, event_id: str) -> bool:
|
|
203
|
+
"""Whether this event id has already been applied. Mercury delivers at-least-once."""
|
|
204
|
+
return self._db.execute("SELECT 1 FROM seen_event WHERE event_id = ?", (event_id,)).fetchone() is not None
|
|
205
|
+
|
|
206
|
+
def mark_seen(self, event_id: str, at: dt.datetime) -> None:
|
|
207
|
+
"""Record an event id as applied, whether it changed anything or not.
|
|
208
|
+
|
|
209
|
+
An event skipped by the version guard is still recorded: it has been
|
|
210
|
+
considered, and reconsidering it on the next delivery would reach the
|
|
211
|
+
same conclusion more slowly.
|
|
212
|
+
"""
|
|
213
|
+
self._db.execute(
|
|
214
|
+
"INSERT INTO seen_event (event_id, applied_at) VALUES (?, ?) ON CONFLICT(event_id) DO NOTHING",
|
|
215
|
+
(event_id, at.isoformat()),
|
|
216
|
+
)
|
|
217
|
+
self._db.commit()
|
|
218
|
+
|
|
219
|
+
def seen_count(self) -> int:
|
|
220
|
+
"""How many event ids have been recorded. For `runningtab mercury status`."""
|
|
221
|
+
return int(self._db.execute("SELECT COUNT(*) FROM seen_event").fetchone()[0])
|
|
222
|
+
|
|
223
|
+
# --- ordering ---------------------------------------------------------
|
|
224
|
+
|
|
225
|
+
def version_of(self, resource_id: str) -> int | None:
|
|
226
|
+
"""The highest `resourceVersion` applied for a resource, or None."""
|
|
227
|
+
row = self._db.execute("SELECT version FROM resource_version WHERE resource_id = ?", (resource_id,)).fetchone()
|
|
228
|
+
return None if row is None else int(row[0])
|
|
229
|
+
|
|
230
|
+
def record_version(self, resource_id: str, version: int) -> None:
|
|
231
|
+
"""Raise a resource's high-water mark. Never lowers it: a late replay must not reopen the past."""
|
|
232
|
+
self._db.execute(
|
|
233
|
+
"INSERT INTO resource_version (resource_id, version) VALUES (?, ?) "
|
|
234
|
+
"ON CONFLICT(resource_id) DO UPDATE SET version = MAX(version, excluded.version)",
|
|
235
|
+
(resource_id, version),
|
|
236
|
+
)
|
|
237
|
+
self._db.commit()
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
class DryRunState:
|
|
241
|
+
"""An `IngestState` that remembers for the length of one run and then forgets.
|
|
242
|
+
|
|
243
|
+
So a dry run answers the question people actually ask it -- "what would a
|
|
244
|
+
real run do?" -- rather than a slightly different one. Buffering the writes
|
|
245
|
+
here instead of branching on `dry_run` at every write means the applier
|
|
246
|
+
takes the same path either way; the only thing a dry run skips is the
|
|
247
|
+
ledger write itself.
|
|
248
|
+
"""
|
|
249
|
+
|
|
250
|
+
def __init__(self, inner: IngestState) -> None:
|
|
251
|
+
self._inner = inner
|
|
252
|
+
self._seen: set[str] = set()
|
|
253
|
+
self._versions: dict[str, int] = {}
|
|
254
|
+
|
|
255
|
+
def cursor(self) -> Cursor | None:
|
|
256
|
+
return self._inner.cursor()
|
|
257
|
+
|
|
258
|
+
def require_resumable(self, now: dt.datetime) -> Cursor:
|
|
259
|
+
return self._inner.require_resumable(now)
|
|
260
|
+
|
|
261
|
+
def record_cursor(self, event_id: str | None, at: dt.datetime) -> None:
|
|
262
|
+
"""Deliberately nothing: a dry run must not move the real cursor."""
|
|
263
|
+
|
|
264
|
+
def has_seen(self, event_id: str) -> bool:
|
|
265
|
+
return event_id in self._seen or self._inner.has_seen(event_id)
|
|
266
|
+
|
|
267
|
+
def mark_seen(self, event_id: str, at: dt.datetime) -> None:
|
|
268
|
+
self._seen.add(event_id)
|
|
269
|
+
|
|
270
|
+
def seen_count(self) -> int:
|
|
271
|
+
return self._inner.seen_count() + len(self._seen)
|
|
272
|
+
|
|
273
|
+
def version_of(self, resource_id: str) -> int | None:
|
|
274
|
+
buffered = self._versions.get(resource_id)
|
|
275
|
+
recorded = self._inner.version_of(resource_id)
|
|
276
|
+
if buffered is None:
|
|
277
|
+
return recorded
|
|
278
|
+
return buffered if recorded is None else max(buffered, recorded)
|
|
279
|
+
|
|
280
|
+
def record_version(self, resource_id: str, version: int) -> None:
|
|
281
|
+
self._versions[resource_id] = max(self._versions.get(resource_id, version), version)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
type State = IngestState | DryRunState
|
|
285
|
+
"""What the applier writes its place to: the real file, or a run-scoped buffer."""
|