runningtab 0.1.0.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runningtab/__init__.py +3 -0
- runningtab/access.py +419 -0
- runningtab/books.py +225 -0
- runningtab/chart.py +408 -0
- runningtab/cli.py +2216 -0
- runningtab/coa.py +212 -0
- runningtab/doc.py +185 -0
- runningtab/forecast/__init__.py +1 -0
- runningtab/forecast/compute.py +224 -0
- runningtab/forecast/doctype.py +228 -0
- runningtab/forecast/model.py +251 -0
- runningtab/forecast/names.py +8 -0
- runningtab/forecast/sources.py +85 -0
- runningtab/gl_codes.py +50 -0
- runningtab/health.py +661 -0
- runningtab/mercury/__init__.py +7 -0
- runningtab/mercury/categories.py +500 -0
- runningtab/mercury/classify.py +101 -0
- runningtab/mercury/client.py +360 -0
- runningtab/mercury/ingest.py +733 -0
- runningtab/mercury/mapping.py +130 -0
- runningtab/mercury/models.py +544 -0
- runningtab/mercury/posting.py +757 -0
- runningtab/mercury/receipts.py +304 -0
- runningtab/mercury/session.py +65 -0
- runningtab/mercury/state.py +285 -0
- runningtab/mercury/vouchers.py +951 -0
- runningtab/period/__init__.py +9 -0
- runningtab/period/balances.py +283 -0
- runningtab/period/bank.py +154 -0
- runningtab/period/close.py +510 -0
- runningtab/period/crosscheck.py +257 -0
- runningtab/period/csvfmt.py +86 -0
- runningtab/period/documents.py +260 -0
- runningtab/period/fence.py +277 -0
- runningtab/period/ledger.py +205 -0
- runningtab/period/lock.py +294 -0
- runningtab/period/manifest.py +259 -0
- runningtab/period/open_items.py +84 -0
- runningtab/period/package.py +140 -0
- runningtab/period/qbo.py +521 -0
- runningtab/period/qbo_map.py +174 -0
- runningtab/period/quarter.py +90 -0
- runningtab/period/readiness.py +146 -0
- runningtab/period/record.py +247 -0
- runningtab/period/status.py +312 -0
- runningtab/site.py +409 -0
- runningtab-0.1.0.dev3.dist-info/METADATA +726 -0
- runningtab-0.1.0.dev3.dist-info/RECORD +52 -0
- runningtab-0.1.0.dev3.dist-info/WHEEL +4 -0
- runningtab-0.1.0.dev3.dist-info/entry_points.txt +2 -0
- runningtab-0.1.0.dev3.dist-info/licenses/LICENSE +202 -0
runningtab/cli.py
ADDED
|
@@ -0,0 +1,2216 @@
|
|
|
1
|
+
"""`runningtab` -- keeping a set of books in ERPNext: typer wiring and printing.
|
|
2
|
+
|
|
3
|
+
The work happens in `runningtab.coa` and `runningtab.access`; this module resolves
|
|
4
|
+
targets, asks the operator, renders outcomes and picks exit codes. There is
|
|
5
|
+
no default target: reaching the ledger means naming it.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import dataclasses
|
|
11
|
+
import datetime as dt
|
|
12
|
+
import difflib
|
|
13
|
+
import getpass
|
|
14
|
+
import hashlib
|
|
15
|
+
import pathlib
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
import tomllib
|
|
19
|
+
import urllib.parse
|
|
20
|
+
from collections.abc import Callable, Generator
|
|
21
|
+
from contextlib import contextmanager
|
|
22
|
+
from decimal import Decimal
|
|
23
|
+
from typing import Annotated, NoReturn
|
|
24
|
+
|
|
25
|
+
import httpx2
|
|
26
|
+
import typer
|
|
27
|
+
from google.api_core.exceptions import GoogleAPIError
|
|
28
|
+
from google.auth.exceptions import GoogleAuthError
|
|
29
|
+
from icedcoffee_sync.apply.diff import ChartPlan
|
|
30
|
+
from icedcoffee_sync.apply.model import DesiredAccount
|
|
31
|
+
from icedcoffee_sync.apply.prune import PrunePlan, PruneReport
|
|
32
|
+
from icedcoffee_sync.erpnext.accounts import company_abbr, document_name
|
|
33
|
+
from icedcoffee_sync.erpnext.company import UnknownCompanyFieldError
|
|
34
|
+
from icedcoffee_sync.frappe.access import PROBE_DOCTYPE, OrphanProbeDocumentError, SubmitProbe
|
|
35
|
+
from icedcoffee_sync.frappe.client import Client, SiteConfig
|
|
36
|
+
from icedcoffee_sync.frappe.errors import FrappeError, PermissionError_, ValidationError
|
|
37
|
+
|
|
38
|
+
from runningtab import coa, health
|
|
39
|
+
from runningtab.access import (
|
|
40
|
+
BOT_SUBMITS,
|
|
41
|
+
READER_GRANTS,
|
|
42
|
+
CredentialCheck,
|
|
43
|
+
CredentialCheckError,
|
|
44
|
+
KeyRotationFailedError,
|
|
45
|
+
Policy,
|
|
46
|
+
ProvisionReport,
|
|
47
|
+
check_credential,
|
|
48
|
+
check_reader_credential,
|
|
49
|
+
missing_grants_report,
|
|
50
|
+
provision,
|
|
51
|
+
skipped_grants,
|
|
52
|
+
)
|
|
53
|
+
from runningtab.books import Books, BooksError, resolve_books
|
|
54
|
+
from runningtab.chart import (
|
|
55
|
+
ChartError,
|
|
56
|
+
load_chart,
|
|
57
|
+
load_chart_document,
|
|
58
|
+
load_chart_entries,
|
|
59
|
+
parse_chart,
|
|
60
|
+
parse_defaults,
|
|
61
|
+
suspense_account,
|
|
62
|
+
)
|
|
63
|
+
from runningtab.coa import Context, PlanChangedError, PruneOutcome
|
|
64
|
+
from runningtab.doc import render_doc
|
|
65
|
+
from runningtab.forecast.compute import Forecast, compute
|
|
66
|
+
from runningtab.forecast.doctype import install_doctypes, is_installed, write_forecast
|
|
67
|
+
from runningtab.forecast.model import ForecastFileError, load_forecast
|
|
68
|
+
from runningtab.forecast.names import CASH_FORECAST
|
|
69
|
+
from runningtab.forecast.sources import SourceError, ledger_through, maintenance_message, opening_cash
|
|
70
|
+
from runningtab.gl_codes import gl_code_csv
|
|
71
|
+
from runningtab.mercury.categories import (
|
|
72
|
+
CategoryPlan,
|
|
73
|
+
MercuryCategoryWriter,
|
|
74
|
+
UnmanagedCategory,
|
|
75
|
+
apply_plan,
|
|
76
|
+
carried_categories,
|
|
77
|
+
desired_categories,
|
|
78
|
+
live_flags,
|
|
79
|
+
plan_categories,
|
|
80
|
+
)
|
|
81
|
+
from runningtab.mercury.client import BASE_URLS, MercuryClient, MercuryEnv, MercuryError
|
|
82
|
+
from runningtab.mercury.ingest import IngestReport, read_everything, run_snapshot, run_sync
|
|
83
|
+
from runningtab.mercury.mapping import MappingError, load_mapping
|
|
84
|
+
from runningtab.mercury.posting import (
|
|
85
|
+
BANK,
|
|
86
|
+
PostingError,
|
|
87
|
+
bank_account_documents,
|
|
88
|
+
install_bank_accounts,
|
|
89
|
+
install_custom_fields,
|
|
90
|
+
)
|
|
91
|
+
from runningtab.mercury.receipts import ReceiptFetcher
|
|
92
|
+
from runningtab.mercury.session import posting_context
|
|
93
|
+
from runningtab.mercury.state import (
|
|
94
|
+
RETENTION,
|
|
95
|
+
IngestState,
|
|
96
|
+
NoCursorError,
|
|
97
|
+
StaleCursorError,
|
|
98
|
+
StateIdentityError,
|
|
99
|
+
default_state_path,
|
|
100
|
+
)
|
|
101
|
+
from runningtab.mercury.vouchers import JOURNAL_ENTRY, Chart
|
|
102
|
+
from runningtab.period import PeriodError
|
|
103
|
+
from runningtab.period.balances import trial_balance
|
|
104
|
+
from runningtab.period.bank import read_ledgers as read_mercury_ledgers
|
|
105
|
+
from runningtab.period.close import Built, CloseSteps, Held, HeldForReview, close_quarter, reclose_steps, reopen_step
|
|
106
|
+
from runningtab.period.crosscheck import bind_crosscheck
|
|
107
|
+
from runningtab.period.documents import (
|
|
108
|
+
quarter_vouchers,
|
|
109
|
+
read_bank_transactions,
|
|
110
|
+
read_documents,
|
|
111
|
+
refuse_missing_bank_transactions,
|
|
112
|
+
refuse_unreadable_doctypes,
|
|
113
|
+
refuse_unreadable_vouchers,
|
|
114
|
+
)
|
|
115
|
+
from runningtab.period.fence import LOCK_FIELDS, FenceProof, check_fence_probe, install_fence, prove_fence
|
|
116
|
+
from runningtab.period.ledger import GLRow, index_accounts, read_fiscal_years, read_gl
|
|
117
|
+
from runningtab.period.lock import lock_history, read_lock, read_visible_lock, set_closed_through
|
|
118
|
+
from runningtab.period.manifest import ManifestFacts, render_manifest
|
|
119
|
+
from runningtab.period.open_items import open_items
|
|
120
|
+
from runningtab.period.package import Package, PackageInputs, build_package, sha256sums, write_package
|
|
121
|
+
from runningtab.period.qbo_map import load_qbo_map
|
|
122
|
+
from runningtab.period.quarter import FiscalYear, Quarter, fiscal_year_for
|
|
123
|
+
from runningtab.period.readiness import Blocker, read_blockers
|
|
124
|
+
from runningtab.period.record import (
|
|
125
|
+
GcsRecord,
|
|
126
|
+
SuspectPackage,
|
|
127
|
+
latest,
|
|
128
|
+
next_prefix,
|
|
129
|
+
read_lock_version,
|
|
130
|
+
read_sums,
|
|
131
|
+
revision_of,
|
|
132
|
+
stored_packages,
|
|
133
|
+
upload_package,
|
|
134
|
+
)
|
|
135
|
+
from runningtab.period.status import StatusReport, closed_periods_check, period_status
|
|
136
|
+
from runningtab.site import (
|
|
137
|
+
Profile,
|
|
138
|
+
SiteCredentialError,
|
|
139
|
+
Target,
|
|
140
|
+
admin_config,
|
|
141
|
+
ingest_config,
|
|
142
|
+
mercury_config,
|
|
143
|
+
mercury_write_config,
|
|
144
|
+
site_config,
|
|
145
|
+
store_books_credential,
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
app = typer.Typer(help="Keep a company's books in ERPNext.", no_args_is_help=True)
|
|
149
|
+
coa_app = typer.Typer(help="Chart of accounts.", no_args_is_help=True)
|
|
150
|
+
app.add_typer(coa_app, name="coa")
|
|
151
|
+
chart_app = typer.Typer(help="The chart.coa outline file and the doc generated from it.", no_args_is_help=True)
|
|
152
|
+
app.add_typer(chart_app, name="chart")
|
|
153
|
+
access_app = typer.Typer(help="Bot-user provisioning and submit-permission verification.", no_args_is_help=True)
|
|
154
|
+
app.add_typer(access_app, name="access")
|
|
155
|
+
mercury_app = typer.Typer(
|
|
156
|
+
help=(
|
|
157
|
+
"Read the bank, post each statement line into the ledger as a Bank Transaction. "
|
|
158
|
+
"The ingest is read-only; 'categories apply' is the one command here that writes to Mercury."
|
|
159
|
+
),
|
|
160
|
+
no_args_is_help=True,
|
|
161
|
+
)
|
|
162
|
+
app.add_typer(mercury_app, name="mercury")
|
|
163
|
+
categories_app = typer.Typer(
|
|
164
|
+
help=(
|
|
165
|
+
"Mirror the chart's codeable accounts as Mercury's custom categories. Never deletes one a transaction carries."
|
|
166
|
+
),
|
|
167
|
+
no_args_is_help=True,
|
|
168
|
+
)
|
|
169
|
+
mercury_app.add_typer(categories_app, name="categories")
|
|
170
|
+
|
|
171
|
+
TargetOption = Annotated[Target, typer.Option("--target", help="Which site to act on.")]
|
|
172
|
+
CompanyOption = Annotated[str | None, typer.Option("--company", help="Company name; defaults to the site's only one.")]
|
|
173
|
+
YesOption = Annotated[bool, typer.Option("--yes", help="Skip the confirmation prompt.")]
|
|
174
|
+
|
|
175
|
+
# Tests patch these names on this module.
|
|
176
|
+
_Context = Context
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@dataclasses.dataclass
|
|
180
|
+
class _BooksChoice:
|
|
181
|
+
"""What `--books` said, and the books it named once something asked for them."""
|
|
182
|
+
|
|
183
|
+
flag: pathlib.Path | None
|
|
184
|
+
loaded: Books | None = None
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@app.callback()
|
|
188
|
+
def main(
|
|
189
|
+
ctx: typer.Context,
|
|
190
|
+
books: Annotated[
|
|
191
|
+
pathlib.Path | None,
|
|
192
|
+
typer.Option(
|
|
193
|
+
"--books",
|
|
194
|
+
help=(
|
|
195
|
+
"The books directory: chart.coa, mercury.toml, forecast.toml, qbo.toml and books.toml. "
|
|
196
|
+
"Or set RUNNINGTAB_BOOKS. There is no default."
|
|
197
|
+
),
|
|
198
|
+
show_default=False,
|
|
199
|
+
),
|
|
200
|
+
] = None,
|
|
201
|
+
) -> None:
|
|
202
|
+
"""Keep a company's books in ERPNext: the chart, the bank feed, the forecast, the quarter close."""
|
|
203
|
+
ctx.obj = _BooksChoice(books)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _books(ctx: typer.Context) -> Books:
|
|
207
|
+
"""The books this command acts on, loaded once per invocation; refuses, naming the flag and variable, if none.
|
|
208
|
+
|
|
209
|
+
Every command calls this before doing anything else, so a command never
|
|
210
|
+
gets halfway before discovering it does not know whose books it is
|
|
211
|
+
keeping.
|
|
212
|
+
"""
|
|
213
|
+
choice = ctx.find_root().obj
|
|
214
|
+
if not isinstance(choice, _BooksChoice):
|
|
215
|
+
raise TypeError("runningtab's commands run under its `main` callback, which records --books")
|
|
216
|
+
if choice.loaded is None:
|
|
217
|
+
try:
|
|
218
|
+
choice.loaded = resolve_books(choice.flag)
|
|
219
|
+
except BooksError as exc:
|
|
220
|
+
raise typer.BadParameter(str(exc), param_hint="--books") from exc
|
|
221
|
+
return choice.loaded
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _sole_company(client: Client) -> str:
|
|
225
|
+
try:
|
|
226
|
+
return coa.sole_company(client)
|
|
227
|
+
except coa.NoSoleCompanyError as exc:
|
|
228
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _plan_against(books: Books, target: Target, company: str | None, *, readonly: bool = False) -> Context:
|
|
232
|
+
config = site_config(books.site, target, readonly=readonly)
|
|
233
|
+
typer.echo(f"site: {config.url}")
|
|
234
|
+
chart_text = books.chart_path.read_text(encoding="utf-8")
|
|
235
|
+
try:
|
|
236
|
+
return coa.plan_against(config, company, parse_chart(chart_text), parse_defaults(chart_text))
|
|
237
|
+
except coa.NoSoleCompanyError as exc:
|
|
238
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _confirm(yes: bool, prompt: str) -> None:
|
|
242
|
+
if not (yes or typer.confirm(prompt)):
|
|
243
|
+
typer.echo("aborted")
|
|
244
|
+
raise typer.Exit(code=1)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
# --- coa ------------------------------------------------------------------
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _print_plan(plan: ChartPlan) -> None:
|
|
251
|
+
typer.echo(f"create {len(plan.creates):4}")
|
|
252
|
+
for change in plan.creates:
|
|
253
|
+
typer.echo(f" + {change.desired.number} {change.desired.name}")
|
|
254
|
+
# Split for the reader, not for the tool: both are one ERPNext rename, but
|
|
255
|
+
# a moved number moves the GL code the bank shows and a moved name does not.
|
|
256
|
+
renumbers = [c for c in plan.renames if c.changes_number]
|
|
257
|
+
renames = [c for c in plan.renames if not c.changes_number]
|
|
258
|
+
typer.echo(f"renumber {len(renumbers):4} (the account's GL code moves)")
|
|
259
|
+
for change in renumbers:
|
|
260
|
+
typer.echo(f" # {change.live.document_name} -> {change.desired.number} {change.desired.name}")
|
|
261
|
+
typer.echo(f"rename {len(renames):4} (same number, new label)")
|
|
262
|
+
for change in renames:
|
|
263
|
+
typer.echo(f" ~ {change.live.document_name} -> {change.desired.name}")
|
|
264
|
+
typer.echo(f"retype {len(plan.retypes):4}")
|
|
265
|
+
for change in plan.retypes:
|
|
266
|
+
typer.echo(f" ~ {change.live.document_name} account_type={change.desired.account_type}")
|
|
267
|
+
typer.echo(f"enable {len(plan.enables):4} (chart wants this account; the site has it disabled)")
|
|
268
|
+
for change in plan.enables:
|
|
269
|
+
typer.echo(f" ~ {change.live.document_name}")
|
|
270
|
+
typer.echo(f"extra {len(plan.extras):4} (on the site, not in the chart; left alone)")
|
|
271
|
+
for extra in plan.extras:
|
|
272
|
+
typer.echo(f" ? {extra.live.document_name}")
|
|
273
|
+
if plan.refusals:
|
|
274
|
+
typer.echo(f"REFUSED {len(plan.refusals):4}")
|
|
275
|
+
for refusal in plan.refusals:
|
|
276
|
+
name, parent = refusal.key
|
|
277
|
+
where = f"{name!r} under {parent!r}" if parent else f"{name!r} (root)"
|
|
278
|
+
typer.echo(f" ! {where} {refusal.reason}: {refusal.detail}")
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _print_prune_plan(plan: PrunePlan) -> None:
|
|
282
|
+
typer.echo(f"delete {len(plan.deletes):4}")
|
|
283
|
+
for change in plan.deletes:
|
|
284
|
+
typer.echo(f" - {change.live.document_name}")
|
|
285
|
+
if plan.refusals:
|
|
286
|
+
typer.echo(f"REFUSED {len(plan.refusals):4}")
|
|
287
|
+
for refusal in plan.refusals:
|
|
288
|
+
typer.echo(f" ! {refusal.reason}: {refusal.detail}")
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _print_prune_report(report: PruneReport) -> None:
|
|
292
|
+
summary = f"deleted {len(report.deleted)}, disabled {len(report.disabled)}, failed {len(report.failed)}"
|
|
293
|
+
if report.already_disabled:
|
|
294
|
+
summary += f", already-disabled {len(report.already_disabled)}"
|
|
295
|
+
typer.echo(summary)
|
|
296
|
+
for name, message in report.failed:
|
|
297
|
+
typer.echo(f" ! {name}: {message}")
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _print_prune_outcome(outcome: PruneOutcome) -> None:
|
|
301
|
+
if outcome.cleared_defaults:
|
|
302
|
+
typer.echo(f"cleared Company defaults: {', '.join(outcome.cleared_defaults)}")
|
|
303
|
+
_print_prune_report(outcome.report)
|
|
304
|
+
if outcome.parking is None:
|
|
305
|
+
return
|
|
306
|
+
for name in outcome.parking.gained_entries:
|
|
307
|
+
typer.echo(f"not parking {name}: gained ledger entries since planning")
|
|
308
|
+
for p in outcome.parking.parked:
|
|
309
|
+
typer.echo(f"parked {p.old_document_name} -> {p.parked_number}")
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
@coa_app.command("plan")
|
|
313
|
+
def coa_plan(
|
|
314
|
+
ctx: typer.Context,
|
|
315
|
+
target: TargetOption,
|
|
316
|
+
company: CompanyOption = None,
|
|
317
|
+
require_empty: Annotated[
|
|
318
|
+
bool,
|
|
319
|
+
typer.Option(
|
|
320
|
+
"--require-empty",
|
|
321
|
+
help="Exit non-zero unless the site already matches the chart. For CI, after an apply.",
|
|
322
|
+
),
|
|
323
|
+
] = False,
|
|
324
|
+
) -> None:
|
|
325
|
+
"""Show what applying the chart would change. Writes nothing.
|
|
326
|
+
|
|
327
|
+
Runs as the reader, whose credential the server refuses every write for.
|
|
328
|
+
That is what lets this run on a merge request, where the branch under
|
|
329
|
+
review decides what code the job executes: "plan does not write" is a
|
|
330
|
+
property of this command, and a credential that cannot write is a
|
|
331
|
+
property the server enforces.
|
|
332
|
+
"""
|
|
333
|
+
context = _plan_against(_books(ctx), target, company, readonly=True)
|
|
334
|
+
with context.client:
|
|
335
|
+
_print_plan(context.plan)
|
|
336
|
+
if not context.plan.is_applyable:
|
|
337
|
+
raise typer.Exit(code=1)
|
|
338
|
+
if not context.plan.is_empty:
|
|
339
|
+
if require_empty:
|
|
340
|
+
typer.echo("site does not match the chart")
|
|
341
|
+
raise typer.Exit(code=1)
|
|
342
|
+
return
|
|
343
|
+
typer.echo("site already matches the chart")
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
@coa_app.command("apply")
|
|
347
|
+
def coa_apply(ctx: typer.Context, target: TargetOption, company: CompanyOption = None, yes: YesOption = False) -> None:
|
|
348
|
+
"""Apply the chart, then point Company's defaults at it.
|
|
349
|
+
|
|
350
|
+
Confirmation gates the whole command, since the defaults write reaches
|
|
351
|
+
the ledger even when no account changes. The plan is re-made after the
|
|
352
|
+
prompt and refused if it changed while the prompt sat open.
|
|
353
|
+
"""
|
|
354
|
+
books = _books(ctx)
|
|
355
|
+
context = _plan_against(books, target, company)
|
|
356
|
+
with context.client:
|
|
357
|
+
_print_plan(context.plan)
|
|
358
|
+
if not context.plan.is_applyable:
|
|
359
|
+
typer.echo("refusing to apply: resolve the refusals above first")
|
|
360
|
+
raise typer.Exit(code=1)
|
|
361
|
+
_confirm(yes, f"apply to {context.company}?")
|
|
362
|
+
|
|
363
|
+
fresh = _plan_against(books, target, company)
|
|
364
|
+
fresh.client.close()
|
|
365
|
+
try:
|
|
366
|
+
outcome = coa.apply_confirmed(context, fresh.plan)
|
|
367
|
+
except PlanChangedError:
|
|
368
|
+
typer.echo("refusing to apply: the chart plan changed since planning; re-run `coa apply` to review it")
|
|
369
|
+
_print_plan(fresh.plan)
|
|
370
|
+
raise typer.Exit(code=1) from None
|
|
371
|
+
except UnknownCompanyFieldError as exc:
|
|
372
|
+
typer.echo(f"refusing to apply: {exc}")
|
|
373
|
+
raise typer.Exit(code=1) from exc
|
|
374
|
+
|
|
375
|
+
if outcome.report is None:
|
|
376
|
+
typer.echo("nothing to apply")
|
|
377
|
+
else:
|
|
378
|
+
report = outcome.report
|
|
379
|
+
summary = (
|
|
380
|
+
f"created {len(report.created)}, renamed {len(report.renamed)}, retyped {len(report.retyped)}, "
|
|
381
|
+
f"enabled {len(report.enabled)}"
|
|
382
|
+
)
|
|
383
|
+
if report.parked:
|
|
384
|
+
summary += f", parked {len(report.parked)}"
|
|
385
|
+
typer.echo(summary)
|
|
386
|
+
typer.echo(f"company defaults set: {', '.join(outcome.defaults_written)}")
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _guard_unapplied_chart(plan: ChartPlan, allow_unapplied: bool) -> None:
|
|
390
|
+
if plan.is_empty:
|
|
391
|
+
return
|
|
392
|
+
if not allow_unapplied:
|
|
393
|
+
typer.echo("refusing to prune: the chart above is not yet applied; run `runningtab coa apply` first")
|
|
394
|
+
raise typer.Exit(code=1)
|
|
395
|
+
typer.echo(
|
|
396
|
+
"WARNING: pruning with the chart unapplied. Company defaults pointing at accounts "
|
|
397
|
+
"this run deletes are cleared first; running `runningtab coa apply` afterwards is what "
|
|
398
|
+
"sets them to their replacements."
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _recheck_prune_plan_after_confirmation(
|
|
403
|
+
books: Books, target: Target, company: str | None, context: Context, prune_plan: PrunePlan
|
|
404
|
+
) -> PrunePlan:
|
|
405
|
+
"""Re-plan after the prompt; refuse if the chart or the prune plan changed meanwhile."""
|
|
406
|
+
fresh = _plan_against(books, target, company)
|
|
407
|
+
fresh_prune_plan = coa.prune_plan_for(fresh)
|
|
408
|
+
if fresh.plan == context.plan and fresh_prune_plan == prune_plan:
|
|
409
|
+
return prune_plan
|
|
410
|
+
typer.echo("refusing to prune: the chart or prune plan changed since planning; re-run `coa prune` to review it")
|
|
411
|
+
_print_plan(fresh.plan)
|
|
412
|
+
_print_prune_plan(fresh_prune_plan)
|
|
413
|
+
raise typer.Exit(code=1)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
@coa_app.command("prune")
|
|
417
|
+
def coa_prune(
|
|
418
|
+
ctx: typer.Context,
|
|
419
|
+
target: TargetOption,
|
|
420
|
+
company: CompanyOption = None,
|
|
421
|
+
yes: YesOption = False,
|
|
422
|
+
allow_unapplied: Annotated[
|
|
423
|
+
bool,
|
|
424
|
+
typer.Option(
|
|
425
|
+
"--allow-unapplied",
|
|
426
|
+
help=(
|
|
427
|
+
"Prune even though `coa apply` has not been run. For retiring an existing chart "
|
|
428
|
+
"(e.g. ERPNext's sample chart) before applying ours, when the two collide on "
|
|
429
|
+
"account numbers and apply-first is impossible. Does not bypass the blocking-"
|
|
430
|
+
"refusal check (ROOT_NOT_UNIQUE, AMBIGUOUS_MATCH)."
|
|
431
|
+
),
|
|
432
|
+
),
|
|
433
|
+
] = False,
|
|
434
|
+
) -> None:
|
|
435
|
+
"""Delete accounts the chart does not describe -- the only destructive command.
|
|
436
|
+
|
|
437
|
+
Refuses when the chart plan carries a refusal that could make the extras
|
|
438
|
+
set wrong, or (without `--allow-unapplied`) when the chart is not yet
|
|
439
|
+
applied. Under the flag it also clears Company defaults pointing at
|
|
440
|
+
accounts it deletes and parks survivors off the chart's numbers, since
|
|
441
|
+
ERPNext keeps some sample accounts alive by name until apply repoints
|
|
442
|
+
the defaults. Parking is a renumber, so it needs the prompt too.
|
|
443
|
+
"""
|
|
444
|
+
books = _books(ctx)
|
|
445
|
+
context = _plan_against(books, target, company)
|
|
446
|
+
with context.client:
|
|
447
|
+
_print_plan(context.plan)
|
|
448
|
+
if coa.blocking_prune_refusals(context.plan.refusals):
|
|
449
|
+
typer.echo("refusing to prune: resolve the refusals above first")
|
|
450
|
+
raise typer.Exit(code=1)
|
|
451
|
+
if context.plan.refusals:
|
|
452
|
+
typer.echo(f"{len(context.plan.refusals)} refusal(s) above do not affect prune's extras set; proceeding")
|
|
453
|
+
_guard_unapplied_chart(context.plan, allow_unapplied)
|
|
454
|
+
|
|
455
|
+
prune_plan = coa.prune_plan_for(context)
|
|
456
|
+
_print_prune_plan(prune_plan)
|
|
457
|
+
if prune_plan.is_empty:
|
|
458
|
+
typer.echo("nothing to prune")
|
|
459
|
+
return
|
|
460
|
+
parkable = allow_unapplied and bool(prune_plan.refusals)
|
|
461
|
+
if not prune_plan.deletes and not parkable:
|
|
462
|
+
typer.echo("nothing to delete; see the refusals above")
|
|
463
|
+
return
|
|
464
|
+
if prune_plan.deletes:
|
|
465
|
+
prompt = f"delete {len(prune_plan.deletes)} account(s) from {context.company}?"
|
|
466
|
+
else:
|
|
467
|
+
typer.echo("nothing to delete; see the refusals above")
|
|
468
|
+
prompt = f"attempt to park colliding survivors among the refusals above, from {context.company}?"
|
|
469
|
+
_confirm(yes, prompt)
|
|
470
|
+
|
|
471
|
+
prune_plan = _recheck_prune_plan_after_confirmation(books, target, company, context, prune_plan)
|
|
472
|
+
outcome = coa.prune_confirmed(context, prune_plan, allow_unapplied=allow_unapplied)
|
|
473
|
+
_print_prune_outcome(outcome)
|
|
474
|
+
if outcome.failed:
|
|
475
|
+
raise typer.Exit(code=1)
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
@coa_app.command("export-gl-codes")
|
|
479
|
+
def coa_export_gl_codes(
|
|
480
|
+
ctx: typer.Context,
|
|
481
|
+
output: Annotated[pathlib.Path | None, typer.Option("--output", help="Write here instead of stdout.")] = None,
|
|
482
|
+
) -> None:
|
|
483
|
+
"""Emit the single-column CSV the bank accepts. The upload is manual."""
|
|
484
|
+
csv = gl_code_csv(load_chart(_books(ctx).chart_path))
|
|
485
|
+
if output is None:
|
|
486
|
+
sys.stdout.write(csv)
|
|
487
|
+
return
|
|
488
|
+
output.write_text(csv)
|
|
489
|
+
typer.echo(f"wrote {output}")
|
|
490
|
+
typer.echo("upload it through the bank's dashboard; there is no API for this")
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
# --- chart ----------------------------------------------------------------
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
@chart_app.command("render-doc")
|
|
497
|
+
def chart_render_doc(
|
|
498
|
+
ctx: typer.Context,
|
|
499
|
+
check: Annotated[
|
|
500
|
+
bool,
|
|
501
|
+
typer.Option("--check", help="Exit 1 and print a diff instead of writing, for a CI-style gate."),
|
|
502
|
+
] = False,
|
|
503
|
+
) -> None:
|
|
504
|
+
"""Regenerate the chart doc's account tables (books.toml's chart_doc) from chart.coa."""
|
|
505
|
+
books = _books(ctx)
|
|
506
|
+
doc_path = books.chart_doc
|
|
507
|
+
if not doc_path.is_file():
|
|
508
|
+
raise typer.BadParameter(f"{doc_path} (books.toml's chart_doc) does not exist", param_hint="--books")
|
|
509
|
+
try:
|
|
510
|
+
chart = load_chart_document(books.chart_path)
|
|
511
|
+
except ChartError as exc:
|
|
512
|
+
typer.echo(f"chart.coa: {exc}")
|
|
513
|
+
raise typer.Exit(code=1) from exc
|
|
514
|
+
original = doc_path.read_text(encoding="utf-8")
|
|
515
|
+
try:
|
|
516
|
+
rendered = render_doc(original, chart)
|
|
517
|
+
except ChartError as exc:
|
|
518
|
+
typer.echo(f"{doc_path}: {exc}")
|
|
519
|
+
raise typer.Exit(code=1) from exc
|
|
520
|
+
if rendered == original:
|
|
521
|
+
typer.echo(f"{doc_path} is already up to date")
|
|
522
|
+
return
|
|
523
|
+
if check:
|
|
524
|
+
diff_lines = difflib.unified_diff(
|
|
525
|
+
original.splitlines(keepends=True),
|
|
526
|
+
rendered.splitlines(keepends=True),
|
|
527
|
+
fromfile=str(doc_path),
|
|
528
|
+
tofile=f"{doc_path} (generated)",
|
|
529
|
+
)
|
|
530
|
+
sys.stdout.writelines(diff_lines)
|
|
531
|
+
raise typer.Exit(code=1)
|
|
532
|
+
doc_path.write_text(rendered, encoding="utf-8")
|
|
533
|
+
typer.echo(f"{doc_path} updated")
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
# --- access ---------------------------------------------------------------
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _format_flag(value: bool | None) -> str:
|
|
540
|
+
if value is None:
|
|
541
|
+
return "unknown"
|
|
542
|
+
return "allowed" if value else "denied"
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def _report_refused(probe: SubmitProbe) -> None:
|
|
546
|
+
typer.echo("submit refused by the server")
|
|
547
|
+
typer.echo(f" server-evaluated: create={_format_flag(probe.can_create)} submit={_format_flag(probe.submit_flag)}")
|
|
548
|
+
if probe.error is not None:
|
|
549
|
+
typer.echo(f" submit attempt: {probe.error.message}")
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _report_cannot_submit(probe: SubmitProbe) -> None:
|
|
553
|
+
if probe.can_create is False:
|
|
554
|
+
typer.echo(
|
|
555
|
+
f" the credential cannot create {PROBE_DOCTYPE} drafts, so the submit refusal proves nothing; not storing"
|
|
556
|
+
)
|
|
557
|
+
elif probe.document_name is not None:
|
|
558
|
+
typer.echo(f" the probe document actually posted: {probe.document_name} -- this is an incident")
|
|
559
|
+
elif isinstance(probe.error, ValidationError):
|
|
560
|
+
typer.echo(f" submit permission passed; validation stopped it instead: {probe.error.message}")
|
|
561
|
+
elif isinstance(probe.error, PermissionError_):
|
|
562
|
+
typer.echo(
|
|
563
|
+
f" the submit-flagged insert was refused ({probe.error.message}), but the doctype-level check said "
|
|
564
|
+
f"submit={_format_flag(probe.submit_flag)} -- the two disagree; not storing on an inconsistent answer"
|
|
565
|
+
)
|
|
566
|
+
elif probe.error is not None:
|
|
567
|
+
typer.echo(
|
|
568
|
+
f" the submit-flagged insert failed inconclusively, neither a permission nor a validation "
|
|
569
|
+
f"answer: {probe.error.message}"
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def _report_flags(flags: dict[str, list[str]], verdict: str) -> None:
|
|
574
|
+
typer.echo(" server-evaluated, per doctype:")
|
|
575
|
+
for doctype, ptypes in sorted(flags.items()):
|
|
576
|
+
typer.echo(f" {doctype}: {', '.join(ptypes)}={verdict}")
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _report_provision_result(report: ProvisionReport) -> None:
|
|
580
|
+
typer.echo(f"role created: {report.role_created}")
|
|
581
|
+
if report.role_desk_access_repaired:
|
|
582
|
+
typer.echo(" desk access repaired")
|
|
583
|
+
if report.role_disabled_repaired:
|
|
584
|
+
typer.echo(" role re-enabled")
|
|
585
|
+
for doctype, sent in report.grants.items():
|
|
586
|
+
typer.echo(f" {doctype}: " + ", ".join(f"{ptype}={value}" for ptype, value in sent))
|
|
587
|
+
if report.revoked:
|
|
588
|
+
typer.echo(f"grants revoked: {', '.join(sorted(report.revoked))}")
|
|
589
|
+
typer.echo(f"user created: {report.user.created}")
|
|
590
|
+
if report.user.added_roles:
|
|
591
|
+
typer.echo(f" roles added: {', '.join(sorted(report.user.added_roles))}")
|
|
592
|
+
if report.user.removed_roles:
|
|
593
|
+
typer.echo(f" roles removed: {', '.join(sorted(report.user.removed_roles))}")
|
|
594
|
+
if report.user.enabled_repaired:
|
|
595
|
+
typer.echo(" user re-enabled")
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
def _report_fence_install(results: list[tuple[str, str]]) -> None:
|
|
599
|
+
typer.echo("fence:")
|
|
600
|
+
for label, outcome in results:
|
|
601
|
+
typer.echo(f" {outcome:9} {label}")
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def _report_fence_proof(proof: FenceProof) -> None:
|
|
605
|
+
typer.echo("fence proof (the bot's key attempted each write; an administrator read it back):")
|
|
606
|
+
for field, refused, how in (
|
|
607
|
+
(LOCK_FIELDS[0], proof.date_refused, proof.date_how),
|
|
608
|
+
(LOCK_FIELDS[1], proof.role_refused, proof.role_how),
|
|
609
|
+
):
|
|
610
|
+
typer.echo(f" {field}: {'refused' if refused else 'NOT REFUSED'} ({how})")
|
|
611
|
+
if proof.restored:
|
|
612
|
+
typer.echo(f" restored by the administrator: {', '.join(proof.restored)}")
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def _warn_if_rotation_left_books_broken(target: Target, profile: Profile = Profile.BOT) -> None:
|
|
616
|
+
# generate_keys rotates the live secret unconditionally; only the books
|
|
617
|
+
# target persists a credential that is now stale. Naming the profile
|
|
618
|
+
# matters: the recovery command differs, and re-running the default would
|
|
619
|
+
# rotate the bot while leaving the reader broken.
|
|
620
|
+
if target is not Target.BOOKS:
|
|
621
|
+
return
|
|
622
|
+
flag = " --profile reader" if profile is Profile.READER else ""
|
|
623
|
+
typer.echo(
|
|
624
|
+
f"the {target.value} {profile.value}'s live secret has already rotated and no longer matches "
|
|
625
|
+
f"the stored credential; re-run `runningtab access provision --target {target.value}{flag}` now to "
|
|
626
|
+
"restore a working one"
|
|
627
|
+
)
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _fail_after_rotation(target: Target, reason: str, profile: Profile = Profile.BOT) -> NoReturn:
|
|
631
|
+
typer.echo(f"FAILED: {reason}")
|
|
632
|
+
_warn_if_rotation_left_books_broken(target, profile)
|
|
633
|
+
raise typer.Exit(code=1)
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
_CHECK_FAILURE = {
|
|
637
|
+
"probe": "the submit probe itself failed",
|
|
638
|
+
"prohibited": "checking prohibited permissions on other doctypes failed",
|
|
639
|
+
"grants": "checking granted permissions failed",
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
def _report_inherited(check: CredentialCheck) -> None:
|
|
644
|
+
"""Say what the reader can still write that we never granted it.
|
|
645
|
+
|
|
646
|
+
Printed on a pass as well as a failure: it is the one thing about this
|
|
647
|
+
credential that "read-only" does not cover, and it should not need a
|
|
648
|
+
failure to become visible.
|
|
649
|
+
"""
|
|
650
|
+
if not check.inherited:
|
|
651
|
+
return
|
|
652
|
+
typer.echo("inherited (not granted by us, not removable from here):")
|
|
653
|
+
for doctype, flags in sorted(check.inherited.items()):
|
|
654
|
+
typer.echo(f" {doctype}: {', '.join(flags)}=allowed")
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def _report_check_failure(check: CredentialCheck, *, storing: bool) -> None:
|
|
658
|
+
"""Explain a failed credential check; `storing` selects the provision wording."""
|
|
659
|
+
if not check.probe.refused:
|
|
660
|
+
if storing:
|
|
661
|
+
typer.echo("REFUSING to store: the bot's fresh credential was NOT refused when it tried to submit")
|
|
662
|
+
elif check.probe.document_name is not None:
|
|
663
|
+
typer.echo("the current credential CAN submit")
|
|
664
|
+
else:
|
|
665
|
+
typer.echo("submit denial could not be proven")
|
|
666
|
+
_report_cannot_submit(check.probe)
|
|
667
|
+
elif check.leaked:
|
|
668
|
+
typer.echo(
|
|
669
|
+
"REFUSING to store: submit was refused on Journal Entry, but the fresh credential still "
|
|
670
|
+
"allows a prohibited flag on another granted doctype"
|
|
671
|
+
if storing
|
|
672
|
+
else "submit refused on Journal Entry, but a prohibited flag is allowed on another granted doctype"
|
|
673
|
+
)
|
|
674
|
+
_report_flags(check.leaked, "allowed")
|
|
675
|
+
elif check.missing:
|
|
676
|
+
typer.echo(
|
|
677
|
+
"REFUSING to store: submit was refused and no prohibited flag leaked, but a positive grant "
|
|
678
|
+
"the bot needs is not actually effective on the server"
|
|
679
|
+
if storing
|
|
680
|
+
else "submit refused, but a positive grant the bot needs is not effective on the server"
|
|
681
|
+
)
|
|
682
|
+
_report_flags(check.missing, "not effective")
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
ProfileOption = Annotated[
|
|
686
|
+
Profile, typer.Option("--profile", help="Which identity to provision: the writing bot, or the reader.")
|
|
687
|
+
]
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def _report_skipped_grants(policy: Policy) -> None:
|
|
691
|
+
"""Name any grant above permlevel 0 that this command's own check cannot prove.
|
|
692
|
+
|
|
693
|
+
`missing_grants_report` leaves these out rather than fake-check them with
|
|
694
|
+
`has_permission`, which cannot see permlevel at all -- see its docstring.
|
|
695
|
+
Printed unconditionally, not just on a failure, so a person reading a
|
|
696
|
+
passing run still learns the pass did not cover everything.
|
|
697
|
+
"""
|
|
698
|
+
labels = skipped_grants(policy.grants)
|
|
699
|
+
if not labels:
|
|
700
|
+
return
|
|
701
|
+
typer.echo("not checked here (has_permission cannot see permlevel > 0; see tests/test_live_fence.py):")
|
|
702
|
+
for label in labels:
|
|
703
|
+
typer.echo(f" {label}")
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def _announce_policy(policy: Policy, profile: Profile) -> None:
|
|
707
|
+
typer.echo(f"role: {policy.role}")
|
|
708
|
+
typer.echo(f"user: {policy.user}")
|
|
709
|
+
for g in policy.grants:
|
|
710
|
+
typer.echo(f" {g.doctype}: {', '.join(sorted(g.ptypes))}")
|
|
711
|
+
if profile is Profile.READER:
|
|
712
|
+
typer.echo(" never: write, create, delete, submit (on any doctype)")
|
|
713
|
+
else:
|
|
714
|
+
typer.echo(f" submit only on {', '.join(sorted(BOT_SUBMITS))} (posts nothing until a person reconciles it)")
|
|
715
|
+
typer.echo(" never: cancel, amend (on any doctype); never submit anything else")
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def _verify_minted(profile: Profile, config: SiteConfig, company: str, target: Target) -> CredentialCheck:
|
|
719
|
+
"""Ask the server what the fresh key can actually do, and report the headline."""
|
|
720
|
+
reading = profile is Profile.READER
|
|
721
|
+
try:
|
|
722
|
+
check = check_reader_credential(config) if reading else check_credential(config, company)
|
|
723
|
+
except OrphanProbeDocumentError as exc:
|
|
724
|
+
_fail_after_rotation(
|
|
725
|
+
target,
|
|
726
|
+
f"the create-access probe created a real {exc.doctype} document ({exc.name}) that could not be "
|
|
727
|
+
f"cleaned up ({exc.cleanup_error}) -- a human must delete or investigate it on the live site",
|
|
728
|
+
profile,
|
|
729
|
+
)
|
|
730
|
+
except CredentialCheckError as exc:
|
|
731
|
+
_fail_after_rotation(target, f"{_CHECK_FAILURE[exc.stage]}: {exc.cause}", profile)
|
|
732
|
+
|
|
733
|
+
if profile is Profile.READER:
|
|
734
|
+
_report_inherited(check)
|
|
735
|
+
typer.echo("refused: every mutating flag on the ledger denied by the server")
|
|
736
|
+
elif check.probe.refused:
|
|
737
|
+
_report_refused(check.probe)
|
|
738
|
+
return check
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def _install_and_check_fence(client: Client, company: str, probe_role: str) -> list[tuple[str, str]]:
|
|
742
|
+
"""Fence the lock fields and refuse an unprovable probe -- both before the bot's key is minted.
|
|
743
|
+
|
|
744
|
+
Idempotent and reversible, so -- like provision()'s own grants -- this
|
|
745
|
+
runs before the one irreversible step, the key rotation: a failure here
|
|
746
|
+
must never look like the live secret already rotated. The proof after
|
|
747
|
+
minting repeats the probe check; running it here first keeps an
|
|
748
|
+
unprovable probe from costing a freshly rotated key.
|
|
749
|
+
"""
|
|
750
|
+
results = install_fence(client, probe_role)
|
|
751
|
+
try:
|
|
752
|
+
check_fence_probe(client, company, probe_role)
|
|
753
|
+
except PeriodError as exc:
|
|
754
|
+
typer.echo(f"refused: {exc} (no key was minted; the stored credential is unchanged)", err=True)
|
|
755
|
+
raise typer.Exit(code=1) from exc
|
|
756
|
+
return results
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def _refuse_missing_doctypes(client: Client, policy: Policy, site_url: str, target: Target) -> None:
|
|
760
|
+
"""Refuse a policy that grants a doctype the site lacks.
|
|
761
|
+
|
|
762
|
+
Runs before anything is written: after the confirmation prompt for
|
|
763
|
+
`access provision`, and before the checks for `access verify`, which has
|
|
764
|
+
no prompt.
|
|
765
|
+
|
|
766
|
+
Granting permission on a doctype that does not exist fails midway, after
|
|
767
|
+
earlier grants were written; a plain sentence up front leaves the site
|
|
768
|
+
untouched.
|
|
769
|
+
"""
|
|
770
|
+
if any(g.doctype == CASH_FORECAST for g in policy.grants) and not is_installed(client):
|
|
771
|
+
typer.echo(_doctype_missing_sentence(site_url, target), err=True)
|
|
772
|
+
raise typer.Exit(code=1)
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
def _doctype_missing_sentence(site_url: str, target: Target) -> str:
|
|
776
|
+
return (
|
|
777
|
+
f"the {CASH_FORECAST} doctype is not installed on {site_url.rstrip('/')}; "
|
|
778
|
+
f"run `runningtab forecast install-doctype --target {target.value}` as an administrator first"
|
|
779
|
+
)
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
@access_app.command("provision")
|
|
783
|
+
def access_provision(
|
|
784
|
+
ctx: typer.Context, target: TargetOption, profile: ProfileOption = Profile.BOT, yes: YesOption = False
|
|
785
|
+
) -> None:
|
|
786
|
+
"""Create/repair a role and user, mint a fresh key, and prove the server scopes it as policy says.
|
|
787
|
+
|
|
788
|
+
Runs as an administrator (`RUNNINGTAB_ADMIN_TOKEN`, your own `key:secret`, for
|
|
789
|
+
books). For the bot, also fences Company's two close-lock fields
|
|
790
|
+
(`accounts_frozen_till_date`, `role_allowed_for_frozen_entries`) at
|
|
791
|
+
permission level 1 and proves with a real write that the freshly minted
|
|
792
|
+
key cannot move either one -- Frappe resets an over-permlevel write
|
|
793
|
+
silently rather than refusing it (`validate_higher_perm_levels`), so that
|
|
794
|
+
proof is the only thing worth trusting. Stores the key, for the books
|
|
795
|
+
target only, once every check has passed: for the bot, that submit was
|
|
796
|
+
refused, no prohibited flag leaked, and neither lock field moved; for the
|
|
797
|
+
reader, that no write flag is allowed at all. The live secret is rotated
|
|
798
|
+
the moment the key is minted, so every later failure says so.
|
|
799
|
+
"""
|
|
800
|
+
books = _books(ctx)
|
|
801
|
+
config = admin_config(books.site, target)
|
|
802
|
+
policy = books.reader if profile is Profile.READER else books.bot
|
|
803
|
+
secret_id = books.site.books_readonly_secret if profile is Profile.READER else books.site.books_secret
|
|
804
|
+
_announce_policy(policy, profile)
|
|
805
|
+
_report_skipped_grants(policy)
|
|
806
|
+
_confirm(yes, f"provision {profile.value} on {target.value}?")
|
|
807
|
+
|
|
808
|
+
with Client(config) as client:
|
|
809
|
+
company = _sole_company(client) # before the key rotates, so its errors are ordinary ones
|
|
810
|
+
_refuse_missing_doctypes(client, policy, config.url, target)
|
|
811
|
+
fence_results = (
|
|
812
|
+
_install_and_check_fence(client, company, books.fence_probe_role) if profile is Profile.BOT else []
|
|
813
|
+
)
|
|
814
|
+
try:
|
|
815
|
+
report = provision(client, policy)
|
|
816
|
+
except KeyRotationFailedError as exc:
|
|
817
|
+
_fail_after_rotation(target, f"minting a fresh key failed: {exc}", profile)
|
|
818
|
+
_report_provision_result(report)
|
|
819
|
+
if fence_results:
|
|
820
|
+
_report_fence_install(fence_results)
|
|
821
|
+
|
|
822
|
+
probe_config = dataclasses.replace(config, api_key=report.keys.api_key, api_secret=report.keys.api_secret)
|
|
823
|
+
check = _verify_minted(profile, probe_config, company, target)
|
|
824
|
+
if not check.passed:
|
|
825
|
+
_report_check_failure(check, storing=True)
|
|
826
|
+
_warn_if_rotation_left_books_broken(target, profile)
|
|
827
|
+
raise typer.Exit(code=1)
|
|
828
|
+
|
|
829
|
+
if profile is Profile.BOT:
|
|
830
|
+
# The one proof that means anything: a real write with the fresh key,
|
|
831
|
+
# read back by an administrator. See runningtab.period.fence.prove_fence.
|
|
832
|
+
# Wrapped the same way `_verify_minted` is: a failure here means we do
|
|
833
|
+
# not actually know whether the fence held, and the secret has
|
|
834
|
+
# already rotated by the time we find out, so it goes through
|
|
835
|
+
# `_fail_after_rotation` like every other post-mint failure.
|
|
836
|
+
with Client(config) as admin, Client(probe_config) as bot_client:
|
|
837
|
+
try:
|
|
838
|
+
proof = prove_fence(admin, bot_client, company, books.fence_probe_role)
|
|
839
|
+
except Exception as exc:
|
|
840
|
+
_fail_after_rotation(
|
|
841
|
+
target,
|
|
842
|
+
f"proving the fence held failed: {exc}; a lock field may have been left moved -- "
|
|
843
|
+
"check Company in ERPNext now",
|
|
844
|
+
profile,
|
|
845
|
+
)
|
|
846
|
+
_report_fence_proof(proof)
|
|
847
|
+
if not (proof.date_refused and proof.role_refused):
|
|
848
|
+
moved = [
|
|
849
|
+
field
|
|
850
|
+
for field, refused in zip(LOCK_FIELDS, (proof.date_refused, proof.role_refused), strict=True)
|
|
851
|
+
if not refused
|
|
852
|
+
]
|
|
853
|
+
typer.echo(f"REFUSING to store: the bot's key moved {', '.join(moved)} (now restored by the admin)")
|
|
854
|
+
_warn_if_rotation_left_books_broken(target, profile)
|
|
855
|
+
raise typer.Exit(code=1)
|
|
856
|
+
|
|
857
|
+
if target is Target.BOOKS:
|
|
858
|
+
try:
|
|
859
|
+
version = store_books_credential(books.site, report.keys.api_key, report.keys.api_secret, secret_id)
|
|
860
|
+
except Exception as exc:
|
|
861
|
+
_fail_after_rotation(target, f"writing the new credential to Secret Manager failed: {exc}", profile)
|
|
862
|
+
typer.echo(f"credential stored: {version}")
|
|
863
|
+
else:
|
|
864
|
+
typer.echo(f"target {target.value!r}: credential not stored; it keeps its credential in the environment")
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
@access_app.command("verify")
|
|
868
|
+
def access_verify(
|
|
869
|
+
ctx: typer.Context, target: TargetOption, profile: ProfileOption = Profile.BOT, company: CompanyOption = None
|
|
870
|
+
) -> None:
|
|
871
|
+
"""Ask the server what the stored credential can do; exit 0 only on a full pass.
|
|
872
|
+
|
|
873
|
+
Rotates nothing, so it is the way to re-check an identity after the policy
|
|
874
|
+
it is held to changes. `--target test` is refused: its bot key is never
|
|
875
|
+
persisted, so there is nothing standalone to verify -- `access provision
|
|
876
|
+
--target test` probes the key it mints.
|
|
877
|
+
"""
|
|
878
|
+
books = _books(ctx)
|
|
879
|
+
if target is Target.TEST:
|
|
880
|
+
raise typer.BadParameter(
|
|
881
|
+
"not supported for --target test: its bot credential is never persisted, so there is nothing "
|
|
882
|
+
"standalone to verify. Run `runningtab access provision --target test` instead -- it mints, probes, "
|
|
883
|
+
"and reports in one step."
|
|
884
|
+
)
|
|
885
|
+
reading = profile is Profile.READER
|
|
886
|
+
config = site_config(books.site, target, readonly=reading)
|
|
887
|
+
policy = books.reader if reading else books.bot
|
|
888
|
+
_report_skipped_grants(policy)
|
|
889
|
+
with Client(config) as client:
|
|
890
|
+
_refuse_missing_doctypes(client, policy, config.url, target)
|
|
891
|
+
|
|
892
|
+
if reading:
|
|
893
|
+
check = check_reader_credential(config)
|
|
894
|
+
_report_inherited(check)
|
|
895
|
+
if check.passed:
|
|
896
|
+
typer.echo("refused: every mutating flag on the ledger denied by the server")
|
|
897
|
+
return
|
|
898
|
+
_report_check_failure(check, storing=False)
|
|
899
|
+
raise typer.Exit(code=1)
|
|
900
|
+
|
|
901
|
+
resolved = company
|
|
902
|
+
if resolved is None:
|
|
903
|
+
with Client(config) as client:
|
|
904
|
+
resolved = _sole_company(client)
|
|
905
|
+
|
|
906
|
+
check = check_credential(config, resolved)
|
|
907
|
+
if check.passed:
|
|
908
|
+
_report_refused(check.probe)
|
|
909
|
+
return
|
|
910
|
+
_report_check_failure(check, storing=False)
|
|
911
|
+
raise typer.Exit(code=1)
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
# --- health ---------------------------------------------------------------
|
|
915
|
+
|
|
916
|
+
# Which Mercury deployment goes with which ERPNext site. The pairing is the
|
|
917
|
+
# point: the disposable site pairs with the sandbox's seeded data, and the
|
|
918
|
+
# real ledger pairs with the company's actual bank. Nothing should be able to
|
|
919
|
+
# read one and report on the other. The ingest commands take `--env` instead,
|
|
920
|
+
# because posting sandbox data into the books is a refusal rather than a
|
|
921
|
+
# pairing (`_refuse_sandbox_into_the_ledger`).
|
|
922
|
+
MERCURY_FOR_TARGET: dict[Target, MercuryEnv] = {
|
|
923
|
+
Target.TEST: MercuryEnv.SANDBOX,
|
|
924
|
+
Target.BOOKS: MercuryEnv.PRODUCTION,
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
def _balance_invariant_check(books: Books, target: Target) -> health.Check:
|
|
929
|
+
config = mercury_config(books.site, MERCURY_FOR_TARGET[target])
|
|
930
|
+
with MercuryClient(config) as client:
|
|
931
|
+
return health.balance_invariant(client)
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
def _suspense_check(books: Books, target: Target, company: str | None) -> health.Check:
|
|
935
|
+
"""Resolve the company, derive the suspense account's document name, and read it.
|
|
936
|
+
|
|
937
|
+
`coa.sole_company`, not `_sole_company`: the latter re-raises as a typer
|
|
938
|
+
usage error, which exits before the report prints. Here the refusal to
|
|
939
|
+
guess is one section's outcome, not a complaint about the command line.
|
|
940
|
+
"""
|
|
941
|
+
with Client(site_config(books.site, target, readonly=True)) as client:
|
|
942
|
+
resolved = company or coa.sole_company(client)
|
|
943
|
+
wanted = suspense_account(load_chart(books.chart_path))
|
|
944
|
+
account = health.suspense_account_name(wanted.number, wanted.name, company_abbr(client, resolved))
|
|
945
|
+
# Local date: this is a report a person runs and reads, and the ageing
|
|
946
|
+
# it prints is in days.
|
|
947
|
+
return health.suspense_contents(client, resolved, account, dt.date.today())
|
|
948
|
+
|
|
949
|
+
|
|
950
|
+
@app.command("health")
|
|
951
|
+
def health_report(
|
|
952
|
+
ctx: typer.Context,
|
|
953
|
+
target: TargetOption,
|
|
954
|
+
company: CompanyOption = None,
|
|
955
|
+
strict: Annotated[
|
|
956
|
+
bool,
|
|
957
|
+
typer.Option(
|
|
958
|
+
"--strict",
|
|
959
|
+
help="Fail on a check that is not available yet, not only on one that is out of bounds.",
|
|
960
|
+
),
|
|
961
|
+
] = False,
|
|
962
|
+
mercury_mapping: Annotated[
|
|
963
|
+
pathlib.Path | None,
|
|
964
|
+
typer.Option(
|
|
965
|
+
"--mercury-mapping",
|
|
966
|
+
help=(
|
|
967
|
+
"Path to a mercury.toml-shaped mapping file; defaults to the books directory's mercury.toml, which "
|
|
968
|
+
"names only the real Mercury accounts. --target test reads the Mercury sandbox, whose "
|
|
969
|
+
"account ids are not in that file, so the closed-periods section's rebuild needs a "
|
|
970
|
+
"mapping of its own here, the same as `period status --target test` does."
|
|
971
|
+
),
|
|
972
|
+
),
|
|
973
|
+
] = None,
|
|
974
|
+
) -> None:
|
|
975
|
+
"""Say whether the books are actually current, and exit non-zero when they are not.
|
|
976
|
+
|
|
977
|
+
Six sections. Three run today. The other three need the Mercury ingest
|
|
978
|
+
and a token for a real Mercury account, and report that rather than
|
|
979
|
+
passing vacuously -- a green report from a check that never ran is the
|
|
980
|
+
failure mode this command exists to prevent. They count as failures only
|
|
981
|
+
under `--strict`, which is what to wire up once the ingest lands.
|
|
982
|
+
|
|
983
|
+
Reads only. Runs as the reader on the ERPNext side, whose credential the
|
|
984
|
+
server refuses every write for.
|
|
985
|
+
"""
|
|
986
|
+
books = _books(ctx)
|
|
987
|
+
typer.echo("runningtab health")
|
|
988
|
+
typer.echo(f" erpnext: {target.value}")
|
|
989
|
+
typer.echo(f" mercury: {MERCURY_FOR_TARGET[target].value}")
|
|
990
|
+
typer.echo("")
|
|
991
|
+
|
|
992
|
+
checks = health.report(
|
|
993
|
+
health.guarded("balance-invariant", "balance invariant", lambda: _balance_invariant_check(books, target)),
|
|
994
|
+
health.guarded("suspense-contents", "suspense contents", lambda: _suspense_check(books, target, company)),
|
|
995
|
+
health.guarded(
|
|
996
|
+
"closed-periods",
|
|
997
|
+
"closed periods",
|
|
998
|
+
lambda: closed_periods_check(_period_status_report(books, target, company, mercury_mapping)),
|
|
999
|
+
),
|
|
1000
|
+
)
|
|
1001
|
+
for line in health.render(checks):
|
|
1002
|
+
typer.echo(line)
|
|
1003
|
+
|
|
1004
|
+
code = health.exit_code(checks, strict=strict)
|
|
1005
|
+
if code:
|
|
1006
|
+
raise typer.Exit(code=code)
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
# --- period ---------------------------------------------------------------
|
|
1010
|
+
|
|
1011
|
+
period_app = typer.Typer(help="A quarter's books as a package for an accountant or auditor.", no_args_is_help=True)
|
|
1012
|
+
app.add_typer(period_app, name="period")
|
|
1013
|
+
|
|
1014
|
+
|
|
1015
|
+
@dataclasses.dataclass(frozen=True)
|
|
1016
|
+
class PeriodReads:
|
|
1017
|
+
"""Everything `period export` reads from the two outside systems, read once."""
|
|
1018
|
+
|
|
1019
|
+
site_url: str
|
|
1020
|
+
company: str
|
|
1021
|
+
abbr: str
|
|
1022
|
+
fiscal_years: list[FiscalYear]
|
|
1023
|
+
rows: list[GLRow]
|
|
1024
|
+
ledgers: list[health.AccountLedger]
|
|
1025
|
+
documents: dict[str, bytes]
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
def _refuse_chart_drift(plan: ChartPlan, target: Target) -> None:
|
|
1029
|
+
"""Refuse, naming each, unless the live chart already matches `chart.coa`.
|
|
1030
|
+
|
|
1031
|
+
The export classifies every GL row from `chart.coa` without ever
|
|
1032
|
+
checking the live site. If the chart was never applied, or has drifted
|
|
1033
|
+
since, every statement here still balances -- an account classified
|
|
1034
|
+
under the wrong number or type looks fine in isolation and wrong only
|
|
1035
|
+
against the real books. `plan` is the same diff `runningtab coa plan` runs;
|
|
1036
|
+
`is_empty` is exactly what that command calls "the site already matches
|
|
1037
|
+
the chart" (extras excluded -- they never block), and `is_applyable`
|
|
1038
|
+
exactly what it calls a plan with nothing refused.
|
|
1039
|
+
"""
|
|
1040
|
+
if plan.is_applyable and plan.is_empty:
|
|
1041
|
+
return
|
|
1042
|
+
parts: list[str] = []
|
|
1043
|
+
parts.extend(f"create {c.desired.number} {c.desired.name}" for c in plan.creates)
|
|
1044
|
+
parts.extend(
|
|
1045
|
+
f"{'renumber' if c.changes_number else 'rename'} {c.live.document_name} -> {c.desired.number} {c.desired.name}"
|
|
1046
|
+
for c in plan.renames
|
|
1047
|
+
)
|
|
1048
|
+
parts.extend(f"retype {c.live.document_name} to {c.desired.account_type}" for c in plan.retypes)
|
|
1049
|
+
parts.extend(f"enable {c.live.document_name}" for c in plan.enables)
|
|
1050
|
+
for refusal in plan.refusals:
|
|
1051
|
+
name, parent = refusal.key
|
|
1052
|
+
where = f"{name!r} under {parent!r}" if parent else f"{name!r} (root)"
|
|
1053
|
+
parts.append(f"refused {where}: {refusal.reason} ({refusal.detail})")
|
|
1054
|
+
raise PeriodError(
|
|
1055
|
+
"the live chart does not match chart.coa: "
|
|
1056
|
+
+ "; ".join(parts)
|
|
1057
|
+
+ f"; run `runningtab coa plan --target {target.value}` to review, `runningtab coa apply` to bring it current"
|
|
1058
|
+
)
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def _period_reads(
|
|
1062
|
+
books: Books,
|
|
1063
|
+
target: Target,
|
|
1064
|
+
company: str | None,
|
|
1065
|
+
quarter: Quarter,
|
|
1066
|
+
chart: list[DesiredAccount],
|
|
1067
|
+
defaults: dict[str, str],
|
|
1068
|
+
config: SiteConfig,
|
|
1069
|
+
*,
|
|
1070
|
+
check_reader_grants: bool,
|
|
1071
|
+
) -> PeriodReads:
|
|
1072
|
+
"""Read the quarter's GL, fiscal years, documents, and bank ledgers with `config`'s credential.
|
|
1073
|
+
|
|
1074
|
+
`config` is the caller's to choose: `period export` reads as the reader
|
|
1075
|
+
(`site_config(books.site, target, readonly=True)`), `period close` as an
|
|
1076
|
+
administrator (`admin_config(books.site, target)`) -- an administrator's grants are
|
|
1077
|
+
not the reader's, so `check_reader_grants` gates the one check below that
|
|
1078
|
+
is specifically about the reader's own provisioning.
|
|
1079
|
+
|
|
1080
|
+
`refuse_unreadable_doctypes` below checks the *intended* grants
|
|
1081
|
+
(`READER_GRANTS`) against the voucher types this quarter actually needs --
|
|
1082
|
+
it catches a voucher type nobody ever granted. It says nothing about
|
|
1083
|
+
whether the credential in hand actually holds what `READER_GRANTS`
|
|
1084
|
+
describes: if the reader was never (re-)provisioned after a grant was
|
|
1085
|
+
added, the server silently omits Files on a doctype it cannot read rather
|
|
1086
|
+
than refusing the query, and this command would ship a package silently
|
|
1087
|
+
short of documents. Asking the server what it actually honours, before
|
|
1088
|
+
any read runs, is what `missing_grants_report` is for -- the same check
|
|
1089
|
+
`access verify` uses. It is only meaningful against the reader's own
|
|
1090
|
+
config, so `check_reader_grants=False` (close) skips it.
|
|
1091
|
+
|
|
1092
|
+
`refuse_missing_bank_transactions` runs last, after both the Frappe and
|
|
1093
|
+
the Mercury reads: it is the only check here that needs both sides at
|
|
1094
|
+
once, to prove every settled Mercury transaction in the quarter has a
|
|
1095
|
+
Bank Transaction the reader could actually see (and so query Files on).
|
|
1096
|
+
|
|
1097
|
+
`chart` and `defaults` are `period_export`'s single parse of `chart.coa`, handed in
|
|
1098
|
+
rather than read here again: the live-chart comparison below runs
|
|
1099
|
+
against the exact same object `period_export` then builds the package
|
|
1100
|
+
from and hashes into the manifest, so a mid-export edit to the file
|
|
1101
|
+
cannot make one of those three see different content than another.
|
|
1102
|
+
"""
|
|
1103
|
+
if check_reader_grants:
|
|
1104
|
+
missing = missing_grants_report(config, READER_GRANTS)
|
|
1105
|
+
if missing:
|
|
1106
|
+
named = "; ".join(f"{doctype} ({', '.join(flags)})" for doctype, flags in sorted(missing.items()))
|
|
1107
|
+
raise PeriodError(
|
|
1108
|
+
f"the reader credential does not hold: {named}; run "
|
|
1109
|
+
f"`runningtab access provision --target {target.value} --profile reader` with RUNNINGTAB_ADMIN_TOKEN set"
|
|
1110
|
+
)
|
|
1111
|
+
chart_plan = coa.plan_against(config, company, chart, defaults)
|
|
1112
|
+
chart_plan.client.close()
|
|
1113
|
+
_refuse_chart_drift(chart_plan.plan, target)
|
|
1114
|
+
with Client(config) as client:
|
|
1115
|
+
resolved = company or coa.sole_company(client)
|
|
1116
|
+
rows = read_gl(client, resolved, quarter.end)
|
|
1117
|
+
vouchers = quarter_vouchers(rows, quarter)
|
|
1118
|
+
refuse_unreadable_doctypes(vouchers, (g.doctype for g in READER_GRANTS))
|
|
1119
|
+
refuse_unreadable_vouchers(client, vouchers)
|
|
1120
|
+
bank_transactions = read_bank_transactions(client, resolved, quarter)
|
|
1121
|
+
documents = read_documents(client, vouchers, [bt.name for bt in bank_transactions])
|
|
1122
|
+
reads = PeriodReads(
|
|
1123
|
+
config.url,
|
|
1124
|
+
resolved,
|
|
1125
|
+
company_abbr(client, resolved),
|
|
1126
|
+
read_fiscal_years(client, resolved),
|
|
1127
|
+
rows,
|
|
1128
|
+
[],
|
|
1129
|
+
documents,
|
|
1130
|
+
)
|
|
1131
|
+
with MercuryClient(mercury_config(books.site, MERCURY_FOR_TARGET[target])) as bank:
|
|
1132
|
+
reads = dataclasses.replace(reads, ledgers=read_mercury_ledgers(bank))
|
|
1133
|
+
refuse_missing_bank_transactions(reads.ledgers, bank_transactions, quarter)
|
|
1134
|
+
return reads
|
|
1135
|
+
|
|
1136
|
+
|
|
1137
|
+
def _period_status_report(
|
|
1138
|
+
books: Books, target: Target, company: str | None, mercury_mapping: pathlib.Path | None = None
|
|
1139
|
+
) -> StatusReport:
|
|
1140
|
+
"""Build the closed-periods status report, reading as the reader -- the same credential `period export` uses.
|
|
1141
|
+
|
|
1142
|
+
Runs `period export`'s own reader effective-grant check first, before
|
|
1143
|
+
comparing anything: a reader credential missing a grant is a named
|
|
1144
|
+
refusal, never reported as drift in a quarter nobody touched (that check
|
|
1145
|
+
is server-side, not merely a static read of `READER_GRANTS`; see
|
|
1146
|
+
`_period_reads`'s own docstring on why). Each stored quarter is rebuilt
|
|
1147
|
+
through `_period_reads` and `build_package` -- the exact path `period
|
|
1148
|
+
export` uses -- so a mismatch `period_status` reports is never an
|
|
1149
|
+
artefact of a different code path disagreeing with itself. Package
|
|
1150
|
+
bytes only: nothing here writes anywhere.
|
|
1151
|
+
|
|
1152
|
+
`mercury_mapping` is `period export`'s own option, needed for the same
|
|
1153
|
+
reason there: `--target test` reads the Mercury sandbox, whose account
|
|
1154
|
+
ids are not in the books directory's `mercury.toml`.
|
|
1155
|
+
"""
|
|
1156
|
+
config = site_config(books.site, target, readonly=True)
|
|
1157
|
+
missing = missing_grants_report(config, READER_GRANTS)
|
|
1158
|
+
if missing:
|
|
1159
|
+
named = "; ".join(f"{doctype} ({', '.join(flags)})" for doctype, flags in sorted(missing.items()))
|
|
1160
|
+
raise PeriodError(
|
|
1161
|
+
f"the reader credential does not hold: {named}; run "
|
|
1162
|
+
f"`runningtab access provision --target {target.value} --profile reader` with RUNNINGTAB_ADMIN_TOKEN set"
|
|
1163
|
+
)
|
|
1164
|
+
|
|
1165
|
+
chart_text = books.chart_path.read_bytes().decode("utf-8")
|
|
1166
|
+
chart = parse_chart(chart_text)
|
|
1167
|
+
defaults = parse_defaults(chart_text)
|
|
1168
|
+
qmap = load_qbo_map(books.qbo_path)
|
|
1169
|
+
mapping = load_mapping(mercury_mapping or books.mercury_path)
|
|
1170
|
+
record = GcsRecord(books.record_buckets[target])
|
|
1171
|
+
|
|
1172
|
+
with Client(config) as client:
|
|
1173
|
+
resolved = company or coa.sole_company(client)
|
|
1174
|
+
abbr = company_abbr(client, resolved)
|
|
1175
|
+
lock = read_visible_lock(client, resolved, Profile.READER)
|
|
1176
|
+
history = lock_history(client, resolved)
|
|
1177
|
+
|
|
1178
|
+
stored_latest = latest(stored_packages(record, abbr))
|
|
1179
|
+
|
|
1180
|
+
def rebuild(quarter: Quarter) -> Package:
|
|
1181
|
+
reads = _period_reads(books, target, resolved, quarter, chart, defaults, config, check_reader_grants=False)
|
|
1182
|
+
fiscal_year = fiscal_year_for(quarter, reads.fiscal_years)
|
|
1183
|
+
return build_package(
|
|
1184
|
+
PackageInputs(
|
|
1185
|
+
quarter=quarter,
|
|
1186
|
+
fiscal_year=fiscal_year,
|
|
1187
|
+
company=reads.company,
|
|
1188
|
+
site_url=reads.site_url,
|
|
1189
|
+
chart=tuple(chart),
|
|
1190
|
+
abbr=reads.abbr,
|
|
1191
|
+
rows=tuple(reads.rows),
|
|
1192
|
+
ledgers=tuple(reads.ledgers),
|
|
1193
|
+
mapping=mapping,
|
|
1194
|
+
qmap=qmap,
|
|
1195
|
+
documents=reads.documents,
|
|
1196
|
+
)
|
|
1197
|
+
)
|
|
1198
|
+
|
|
1199
|
+
return period_status(
|
|
1200
|
+
lock=lock,
|
|
1201
|
+
history=history,
|
|
1202
|
+
stored_latest=stored_latest,
|
|
1203
|
+
sums=lambda stored: read_sums(record, stored),
|
|
1204
|
+
rebuild=rebuild,
|
|
1205
|
+
lock_record=lambda stored: read_lock_version(record, stored),
|
|
1206
|
+
)
|
|
1207
|
+
|
|
1208
|
+
|
|
1209
|
+
@period_app.command("status")
|
|
1210
|
+
def period_status_cmd(
|
|
1211
|
+
ctx: typer.Context,
|
|
1212
|
+
target: TargetOption,
|
|
1213
|
+
company: CompanyOption = None,
|
|
1214
|
+
mercury_mapping: Annotated[
|
|
1215
|
+
pathlib.Path | None,
|
|
1216
|
+
typer.Option(
|
|
1217
|
+
"--mercury-mapping",
|
|
1218
|
+
help=(
|
|
1219
|
+
"Path to a mercury.toml-shaped mapping file; defaults to the books directory's mercury.toml, which "
|
|
1220
|
+
"names only the real Mercury accounts. --target test reads the Mercury sandbox, whose "
|
|
1221
|
+
"account ids are not in that file, so checking it needs a mapping of its own here."
|
|
1222
|
+
),
|
|
1223
|
+
),
|
|
1224
|
+
] = None,
|
|
1225
|
+
) -> None:
|
|
1226
|
+
"""Report which quarters are closed, whether their stored packages still match the ledger, and whether the
|
|
1227
|
+
lock itself is still intact.
|
|
1228
|
+
|
|
1229
|
+
Reads only, as the reader -- the same credential `period export` uses.
|
|
1230
|
+
Every closed quarter's latest revision is rebuilt through that exact same
|
|
1231
|
+
path and compared, byte for byte, against its stored `SHA256SUMS`; a
|
|
1232
|
+
changed or missing file is named, not merely flagged. Also reports a
|
|
1233
|
+
reopened quarter (Books Closed Through moved backward, naming who and
|
|
1234
|
+
when from the Company Version history), a quarter locked with no stored
|
|
1235
|
+
package (Books Closed Through past every stored quarter), and
|
|
1236
|
+
`role_allowed_for_frozen_entries` left set. Exits 1 unless every one of
|
|
1237
|
+
those comes back clean.
|
|
1238
|
+
"""
|
|
1239
|
+
try:
|
|
1240
|
+
report = _period_status_report(_books(ctx), target, company, mercury_mapping)
|
|
1241
|
+
except _CLOSE_REFUSALS as exc:
|
|
1242
|
+
typer.echo(f"refused: {exc}", err=True)
|
|
1243
|
+
raise typer.Exit(code=1) from exc
|
|
1244
|
+
|
|
1245
|
+
shown = report.lock.closed_through.isoformat() if report.lock.closed_through else "not set"
|
|
1246
|
+
typer.echo(f"Books Closed Through: {shown}")
|
|
1247
|
+
if not report.quarters:
|
|
1248
|
+
typer.echo(" nothing closed yet")
|
|
1249
|
+
for quarter_status in report.quarters:
|
|
1250
|
+
if quarter_status.changed:
|
|
1251
|
+
typer.echo(
|
|
1252
|
+
f" {quarter_status.quarter.label} rev{quarter_status.revision}: changed "
|
|
1253
|
+
+ ", ".join(quarter_status.changed)
|
|
1254
|
+
)
|
|
1255
|
+
else:
|
|
1256
|
+
typer.echo(f" {quarter_status.quarter.label} rev{quarter_status.revision}: ok")
|
|
1257
|
+
for problem in report.problems:
|
|
1258
|
+
typer.echo(f" problem: {problem}")
|
|
1259
|
+
|
|
1260
|
+
if not report.ok:
|
|
1261
|
+
raise typer.Exit(code=1)
|
|
1262
|
+
|
|
1263
|
+
|
|
1264
|
+
@period_app.command("export")
|
|
1265
|
+
def period_export(
|
|
1266
|
+
ctx: typer.Context,
|
|
1267
|
+
target: TargetOption,
|
|
1268
|
+
quarter: Annotated[str, typer.Option("--quarter", help="The quarter, as YYYYQn (e.g. 2026Q3).")],
|
|
1269
|
+
company: CompanyOption = None,
|
|
1270
|
+
output: Annotated[
|
|
1271
|
+
pathlib.Path | None, typer.Option("--output", help="Directory to create; defaults to ./<quarter>-DRAFT.")
|
|
1272
|
+
] = None,
|
|
1273
|
+
mercury_mapping: Annotated[
|
|
1274
|
+
pathlib.Path | None,
|
|
1275
|
+
typer.Option(
|
|
1276
|
+
"--mercury-mapping",
|
|
1277
|
+
help=(
|
|
1278
|
+
"Path to a mercury.toml-shaped mapping file; defaults to the books directory's mercury.toml, which "
|
|
1279
|
+
"names only the real Mercury accounts. --target test reads the Mercury sandbox, whose "
|
|
1280
|
+
"account ids are not in that file, so it needs a mapping of its own here."
|
|
1281
|
+
),
|
|
1282
|
+
),
|
|
1283
|
+
] = None,
|
|
1284
|
+
) -> None:
|
|
1285
|
+
"""Build a draft package for one quarter. Reads only; writes a new local directory and nothing else.
|
|
1286
|
+
|
|
1287
|
+
Exits 1 when the bank and the ledger disagree at the quarter's end -- the
|
|
1288
|
+
draft is still written, because it is what a person investigates with --
|
|
1289
|
+
and 1 without writing anything when the package cannot be built at all.
|
|
1290
|
+
"""
|
|
1291
|
+
try:
|
|
1292
|
+
parsed = Quarter.parse(quarter)
|
|
1293
|
+
except PeriodError as exc:
|
|
1294
|
+
raise typer.BadParameter(str(exc), param_hint="--quarter") from exc
|
|
1295
|
+
books = _books(ctx)
|
|
1296
|
+
directory = output or pathlib.Path(f"{parsed.label}-DRAFT")
|
|
1297
|
+
try:
|
|
1298
|
+
# Read chart.coa once and parse it once, here: the live-chart check
|
|
1299
|
+
# inside _period_reads, the package below, and the manifest's SHA-256
|
|
1300
|
+
# all use this same snapshot, so an edit landing mid-export cannot
|
|
1301
|
+
# make one of the three see different content than another.
|
|
1302
|
+
chart_bytes = books.chart_path.read_bytes()
|
|
1303
|
+
chart = parse_chart(chart_bytes.decode("utf-8"))
|
|
1304
|
+
defaults = parse_defaults(chart_bytes.decode("utf-8"))
|
|
1305
|
+
reader_config = site_config(books.site, target, readonly=True)
|
|
1306
|
+
reads = _period_reads(books, target, company, parsed, chart, defaults, reader_config, check_reader_grants=True)
|
|
1307
|
+
fiscal_year = fiscal_year_for(parsed, reads.fiscal_years)
|
|
1308
|
+
qmap = load_qbo_map(books.qbo_path)
|
|
1309
|
+
package = build_package(
|
|
1310
|
+
PackageInputs(
|
|
1311
|
+
quarter=parsed,
|
|
1312
|
+
fiscal_year=fiscal_year,
|
|
1313
|
+
company=reads.company,
|
|
1314
|
+
site_url=reads.site_url,
|
|
1315
|
+
chart=tuple(chart),
|
|
1316
|
+
abbr=reads.abbr,
|
|
1317
|
+
rows=tuple(reads.rows),
|
|
1318
|
+
ledgers=tuple(reads.ledgers),
|
|
1319
|
+
mapping=load_mapping(mercury_mapping or books.mercury_path),
|
|
1320
|
+
qmap=qmap,
|
|
1321
|
+
documents=reads.documents,
|
|
1322
|
+
)
|
|
1323
|
+
)
|
|
1324
|
+
facts = ManifestFacts(
|
|
1325
|
+
quarter=parsed,
|
|
1326
|
+
fiscal_year=fiscal_year,
|
|
1327
|
+
company=reads.company,
|
|
1328
|
+
books_name=books.name,
|
|
1329
|
+
site_url=reads.site_url,
|
|
1330
|
+
chart_sha256=hashlib.sha256(chart_bytes).hexdigest(),
|
|
1331
|
+
qmap=qmap,
|
|
1332
|
+
generated_at=dt.datetime.now(dt.UTC),
|
|
1333
|
+
generated_by=getpass.getuser(),
|
|
1334
|
+
draft=True,
|
|
1335
|
+
today=dt.date.today(),
|
|
1336
|
+
bank=package.bank,
|
|
1337
|
+
split_vouchers=package.split_vouchers,
|
|
1338
|
+
has_journals=any(p.startswith("qbo/journal-entries-") for p in package.files),
|
|
1339
|
+
)
|
|
1340
|
+
write_package(directory, package, render_manifest(facts, package.files))
|
|
1341
|
+
except (
|
|
1342
|
+
PeriodError,
|
|
1343
|
+
ChartError,
|
|
1344
|
+
SiteCredentialError,
|
|
1345
|
+
MercuryError,
|
|
1346
|
+
# Every doctype this command reads is one the reader might not (yet)
|
|
1347
|
+
# hold; the same set `runningtab.health.guarded` catches, for the same
|
|
1348
|
+
# reason -- these are all "could not read", not a crash.
|
|
1349
|
+
FrappeError,
|
|
1350
|
+
coa.NoSoleCompanyError,
|
|
1351
|
+
# `company_abbr` raises LookupError for a company the site does not
|
|
1352
|
+
# have, or one with no abbreviation.
|
|
1353
|
+
LookupError,
|
|
1354
|
+
httpx2.HTTPError,
|
|
1355
|
+
# A hand-edited mercury.toml or qbo.toml that does not parse, or does
|
|
1356
|
+
# not describe a mapping this command can post with: refused, not
|
|
1357
|
+
# crashed -- the person editing the file gets the same "could not
|
|
1358
|
+
# build the package" report as every other input problem here.
|
|
1359
|
+
MappingError,
|
|
1360
|
+
tomllib.TOMLDecodeError,
|
|
1361
|
+
# write_package's own filesystem writes (a directory it cannot
|
|
1362
|
+
# create, a parent that is not a directory): a bad --output is a
|
|
1363
|
+
# refusal, not a traceback.
|
|
1364
|
+
OSError,
|
|
1365
|
+
) as exc:
|
|
1366
|
+
typer.echo(f"refused: {exc}", err=True)
|
|
1367
|
+
if isinstance(exc, PermissionError_):
|
|
1368
|
+
typer.echo(
|
|
1369
|
+
f"the reader may be missing a grant this command needs; run "
|
|
1370
|
+
f"`runningtab access provision --target {target.value} --profile reader` with RUNNINGTAB_ADMIN_TOKEN set",
|
|
1371
|
+
err=True,
|
|
1372
|
+
)
|
|
1373
|
+
raise typer.Exit(code=1) from exc
|
|
1374
|
+
typer.echo(f"wrote {directory}/ ({len(package.files)} files)")
|
|
1375
|
+
for voucher in package.split_vouchers:
|
|
1376
|
+
typer.echo(f" split through QBO Import Clearing: {voucher}")
|
|
1377
|
+
for line in package.bank:
|
|
1378
|
+
typer.echo(
|
|
1379
|
+
f" bank {line.number} {line.label}: bank {line.bank}, ledger {line.ledger}, difference {line.difference}"
|
|
1380
|
+
)
|
|
1381
|
+
code = export_exit_code(package)
|
|
1382
|
+
if code:
|
|
1383
|
+
typer.echo("the bank and the ledger disagree at the quarter's end; see bank-reconciliation.csv", err=True)
|
|
1384
|
+
raise typer.Exit(code=code)
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
def export_exit_code(package: Package) -> int:
|
|
1388
|
+
"""1 when any Mercury account disagrees with the ledger at the quarter's end: the draft wants a person."""
|
|
1389
|
+
return 1 if any(line.difference for line in package.bank) else 0
|
|
1390
|
+
|
|
1391
|
+
|
|
1392
|
+
def _today() -> dt.date:
|
|
1393
|
+
"""The close ceremony's clock, as its own name so a test can patch it instead of the real one."""
|
|
1394
|
+
return dt.date.today()
|
|
1395
|
+
|
|
1396
|
+
|
|
1397
|
+
_CLOSE_REFUSALS: tuple[type[Exception], ...] = (
|
|
1398
|
+
PeriodError,
|
|
1399
|
+
ChartError,
|
|
1400
|
+
SiteCredentialError,
|
|
1401
|
+
MercuryError,
|
|
1402
|
+
FrappeError,
|
|
1403
|
+
coa.NoSoleCompanyError,
|
|
1404
|
+
LookupError,
|
|
1405
|
+
httpx2.HTTPError,
|
|
1406
|
+
MappingError,
|
|
1407
|
+
tomllib.TOMLDecodeError,
|
|
1408
|
+
OSError,
|
|
1409
|
+
GoogleAPIError,
|
|
1410
|
+
# ADC expiring or missing mid-command (GcsRecord's storage.Client(), or a
|
|
1411
|
+
# revoked/expired token discovered on upload) raises from google.auth,
|
|
1412
|
+
# not google.api_core -- GoogleAuthError is not a GoogleAPIError subclass,
|
|
1413
|
+
# so it needs naming here too or it crashes instead of refusing.
|
|
1414
|
+
GoogleAuthError,
|
|
1415
|
+
)
|
|
1416
|
+
|
|
1417
|
+
|
|
1418
|
+
# Post-lock failures whose own message already names the next step, so the
|
|
1419
|
+
# lock line beside them adds no retry advice: a rebuild mismatch says to
|
|
1420
|
+
# re-run or reopen, and a suspect stored package says to run `period status`
|
|
1421
|
+
# (a retry there would only be refused as "already has a stored package").
|
|
1422
|
+
_NAMES_ITS_OWN_NEXT_STEP: tuple[type[Exception], ...] = (HeldForReview, SuspectPackage)
|
|
1423
|
+
|
|
1424
|
+
|
|
1425
|
+
def _report_held_lock(held: Held, quarter: Quarter, exc: Exception, *, refusal: bool) -> None:
|
|
1426
|
+
"""Say the quarter is locked, before a failure after the lock is reported, so it never reads as "nothing happened".
|
|
1427
|
+
|
|
1428
|
+
A named refusal that is not one of `_NAMES_ITS_OWN_NEXT_STEP` (a
|
|
1429
|
+
dropped connection, a bucket 503) gets its next step. On a first close
|
|
1430
|
+
the re-run takes the recovery path and stores the package -- after
|
|
1431
|
+
`period status` shows it is still missing, since an error can come back
|
|
1432
|
+
after the store completed and a re-run would then only be refused as
|
|
1433
|
+
"already has a stored package". On a re-close
|
|
1434
|
+
of a stored quarter the re-run would be refused as "already has a stored
|
|
1435
|
+
package", so the step is to reopen the quarter and close it again -- but
|
|
1436
|
+
only after `period status` shows the new revision did not land (an error
|
|
1437
|
+
can come back after the store completed, and every revision is
|
|
1438
|
+
permanent), and followed by every stored quarter after it, in order. An
|
|
1439
|
+
unexpected exception gets the lock line alone -- whether a retry helps is
|
|
1440
|
+
unknown -- and is re-raised.
|
|
1441
|
+
"""
|
|
1442
|
+
change = held.change
|
|
1443
|
+
when = change.new.isoformat() if change.new else "unset"
|
|
1444
|
+
line = f"Books Closed Through is now {when} (Company Version {change.version}, by {change.by})"
|
|
1445
|
+
if refusal and not isinstance(exc, _NAMES_ITS_OWN_NEXT_STEP):
|
|
1446
|
+
if held.reclosing:
|
|
1447
|
+
then = f", then {reclose_steps(list(held.later))}" if held.later else ""
|
|
1448
|
+
line += (
|
|
1449
|
+
f"; the new revision may not have been stored -- run `runningtab period status` first. If it still "
|
|
1450
|
+
f"shows {quarter.label}'s earlier revision, {reopen_step(quarter)} and run `runningtab period close` "
|
|
1451
|
+
f"again for {quarter.label}{then}. A re-run as it stands is refused because {quarter.label} "
|
|
1452
|
+
"already has a stored package."
|
|
1453
|
+
)
|
|
1454
|
+
else:
|
|
1455
|
+
line += (
|
|
1456
|
+
f"; the package may not have been stored -- run `runningtab period status` first. If it shows "
|
|
1457
|
+
f"{quarter.label} locked with no stored package, run `runningtab period close` again for "
|
|
1458
|
+
f"{quarter.label} to store it."
|
|
1459
|
+
)
|
|
1460
|
+
else:
|
|
1461
|
+
line += "; the quarter stays locked."
|
|
1462
|
+
typer.echo(line, err=True)
|
|
1463
|
+
|
|
1464
|
+
|
|
1465
|
+
@period_app.command("close")
|
|
1466
|
+
def period_close(
|
|
1467
|
+
ctx: typer.Context,
|
|
1468
|
+
target: TargetOption,
|
|
1469
|
+
quarter: Annotated[str, typer.Option("--quarter", help="The quarter, as YYYYQn (e.g. 2026Q3).")],
|
|
1470
|
+
company: CompanyOption = None,
|
|
1471
|
+
mercury_mapping: Annotated[
|
|
1472
|
+
pathlib.Path | None,
|
|
1473
|
+
typer.Option(
|
|
1474
|
+
"--mercury-mapping",
|
|
1475
|
+
help=(
|
|
1476
|
+
"Path to a mercury.toml-shaped mapping file; defaults to the books directory's mercury.toml, which "
|
|
1477
|
+
"names only the real Mercury accounts. --target test reads the Mercury sandbox, whose "
|
|
1478
|
+
"account ids are not in that file, so it needs a mapping of its own here."
|
|
1479
|
+
),
|
|
1480
|
+
),
|
|
1481
|
+
] = None,
|
|
1482
|
+
) -> None:
|
|
1483
|
+
"""Run the close ceremony (`runningtab.period.close.close_quarter`) for real: lock the quarter and store its package.
|
|
1484
|
+
|
|
1485
|
+
Runs as an administrator -- `RUNNINGTAB_ADMIN_TOKEN`, your own `key:secret`,
|
|
1486
|
+
for `--target books` -- the credential that alone can move Books Closed
|
|
1487
|
+
Through; refused before any request when that variable is unset. Every
|
|
1488
|
+
ERPNext read, the lock, and the cross-check use `admin_config(books.site, target)`,
|
|
1489
|
+
never the reader's; the lock, the readiness read, and the cross-check
|
|
1490
|
+
share one `Client(admin_config(books.site, target))` for the whole command, while
|
|
1491
|
+
each build (the initial one and the ceremony's own rebuild-and-compare)
|
|
1492
|
+
opens its own fresh `Client`/Mercury connection through the same config,
|
|
1493
|
+
the same way `period export` reads with the reader's -- a cached read
|
|
1494
|
+
here would make the rebuild compare a snapshot against itself and never
|
|
1495
|
+
catch a ledger change between the check and the lock. The finished
|
|
1496
|
+
package lands in the record bucket `books.toml` names for the target.
|
|
1497
|
+
See `runningtab.period.close` for the ceremony's own order and its refusals;
|
|
1498
|
+
this only wires it to the real site, the real Mercury account, and the
|
|
1499
|
+
real record bucket.
|
|
1500
|
+
"""
|
|
1501
|
+
try:
|
|
1502
|
+
parsed = Quarter.parse(quarter)
|
|
1503
|
+
except PeriodError as exc:
|
|
1504
|
+
raise typer.BadParameter(str(exc), param_hint="--quarter") from exc
|
|
1505
|
+
# Filled by `close_quarter` (through `CloseSteps.locked`) once the
|
|
1506
|
+
# quarter is locked -- by this run, or by an earlier one this run is
|
|
1507
|
+
# recovering -- so a failure caught below knows the difference between
|
|
1508
|
+
# "nothing happened" and "the quarter is locked but nothing was stored".
|
|
1509
|
+
held: list[Held] = []
|
|
1510
|
+
books = _books(ctx)
|
|
1511
|
+
try:
|
|
1512
|
+
# admin_config raises MissingSiteCredentialError from RUNNINGTAB_ADMIN_TOKEN
|
|
1513
|
+
# alone, reading nothing else, before anything below ever opens a
|
|
1514
|
+
# connection -- so an unset token refuses here, with no request made.
|
|
1515
|
+
config = admin_config(books.site, target)
|
|
1516
|
+
chart_bytes = books.chart_path.read_bytes()
|
|
1517
|
+
chart = parse_chart(chart_bytes.decode("utf-8"))
|
|
1518
|
+
defaults = parse_defaults(chart_bytes.decode("utf-8"))
|
|
1519
|
+
chart_sha256 = hashlib.sha256(chart_bytes).hexdigest()
|
|
1520
|
+
qmap = load_qbo_map(books.qbo_path)
|
|
1521
|
+
record = GcsRecord(books.record_buckets[target])
|
|
1522
|
+
|
|
1523
|
+
with Client(config) as client:
|
|
1524
|
+
resolved_company = company or coa.sole_company(client)
|
|
1525
|
+
abbr = company_abbr(client, resolved_company)
|
|
1526
|
+
index = index_accounts(chart, abbr)
|
|
1527
|
+
crosscheck = bind_crosscheck(client, resolved_company, index)
|
|
1528
|
+
wanted = suspense_account(chart)
|
|
1529
|
+
suspense_name = health.suspense_account_name(wanted.number, wanted.name, abbr)
|
|
1530
|
+
|
|
1531
|
+
def build_step() -> Built:
|
|
1532
|
+
# A fresh read every call -- including the ceremony's own
|
|
1533
|
+
# rebuild-and-compare call -- through the admin config, the
|
|
1534
|
+
# same way `period export` reads with the reader's; a cached
|
|
1535
|
+
# read here would make the rebuild compare a snapshot against
|
|
1536
|
+
# itself and never catch a ledger change between the check
|
|
1537
|
+
# and the lock.
|
|
1538
|
+
reads = _period_reads(
|
|
1539
|
+
books, target, resolved_company, parsed, chart, defaults, config, check_reader_grants=False
|
|
1540
|
+
)
|
|
1541
|
+
fiscal_year = fiscal_year_for(parsed, reads.fiscal_years)
|
|
1542
|
+
step_index = index_accounts(chart, reads.abbr)
|
|
1543
|
+
rows = tuple(reads.rows)
|
|
1544
|
+
tb = trial_balance(rows, step_index, parsed, fiscal_year)
|
|
1545
|
+
receivables = tuple(open_items(rows, step_index, "Receivable", parsed.end))
|
|
1546
|
+
payables = tuple(open_items(rows, step_index, "Payable", parsed.end))
|
|
1547
|
+
package = build_package(
|
|
1548
|
+
PackageInputs(
|
|
1549
|
+
quarter=parsed,
|
|
1550
|
+
fiscal_year=fiscal_year,
|
|
1551
|
+
company=reads.company,
|
|
1552
|
+
site_url=reads.site_url,
|
|
1553
|
+
chart=tuple(chart),
|
|
1554
|
+
abbr=reads.abbr,
|
|
1555
|
+
rows=rows,
|
|
1556
|
+
ledgers=tuple(reads.ledgers),
|
|
1557
|
+
mapping=load_mapping(mercury_mapping or books.mercury_path),
|
|
1558
|
+
qmap=qmap,
|
|
1559
|
+
documents=reads.documents,
|
|
1560
|
+
)
|
|
1561
|
+
)
|
|
1562
|
+
facts = ManifestFacts(
|
|
1563
|
+
quarter=parsed,
|
|
1564
|
+
fiscal_year=fiscal_year,
|
|
1565
|
+
company=reads.company,
|
|
1566
|
+
books_name=books.name,
|
|
1567
|
+
site_url=reads.site_url,
|
|
1568
|
+
chart_sha256=chart_sha256,
|
|
1569
|
+
qmap=qmap,
|
|
1570
|
+
generated_at=dt.datetime.now(dt.UTC),
|
|
1571
|
+
generated_by=getpass.getuser(),
|
|
1572
|
+
draft=True,
|
|
1573
|
+
today=_today(),
|
|
1574
|
+
bank=package.bank,
|
|
1575
|
+
split_vouchers=package.split_vouchers,
|
|
1576
|
+
has_journals=any(p.startswith("qbo/journal-entries-") for p in package.files),
|
|
1577
|
+
)
|
|
1578
|
+
return Built(
|
|
1579
|
+
package=package, trial_balance=tb, receivables=receivables, payables=payables, manifest_facts=facts
|
|
1580
|
+
)
|
|
1581
|
+
|
|
1582
|
+
def read_blockers_step() -> list[Blocker]:
|
|
1583
|
+
# Its own GL read, through the same admin client: stage 1's
|
|
1584
|
+
# preconditions must refuse (if they are going to) before any
|
|
1585
|
+
# heavy read runs, so this cannot borrow `build`'s rows --
|
|
1586
|
+
# `build` has not been called yet when this does.
|
|
1587
|
+
rows = read_gl(client, resolved_company, parsed.end)
|
|
1588
|
+
return read_blockers(client, resolved_company, parsed, suspense_name, rows)
|
|
1589
|
+
|
|
1590
|
+
def upload_step(files: dict[str, bytes], build_manifest: Callable[[int], bytes]) -> str:
|
|
1591
|
+
prefix = next_prefix(record, abbr, parsed)
|
|
1592
|
+
revision = revision_of(prefix, abbr, parsed)
|
|
1593
|
+
manifest = build_manifest(revision)
|
|
1594
|
+
all_files = {**files, "MANIFEST.md": manifest, "SHA256SUMS": sha256sums(files)}
|
|
1595
|
+
upload_package(record, prefix, all_files)
|
|
1596
|
+
return prefix
|
|
1597
|
+
|
|
1598
|
+
steps = CloseSteps(
|
|
1599
|
+
read_lock=lambda: read_lock(client, resolved_company),
|
|
1600
|
+
read_blockers=read_blockers_step,
|
|
1601
|
+
build=build_step,
|
|
1602
|
+
crosscheck=crosscheck,
|
|
1603
|
+
set_closed_through=lambda date: set_closed_through(client, resolved_company, date),
|
|
1604
|
+
lock_history=lambda: lock_history(client, resolved_company),
|
|
1605
|
+
locked=held.append,
|
|
1606
|
+
stored_packages=lambda: stored_packages(record, abbr),
|
|
1607
|
+
upload=upload_step,
|
|
1608
|
+
today=_today,
|
|
1609
|
+
)
|
|
1610
|
+
outcome = close_quarter(steps, parsed, abbr)
|
|
1611
|
+
except Exception as exc:
|
|
1612
|
+
refusal = isinstance(exc, _CLOSE_REFUSALS)
|
|
1613
|
+
if held:
|
|
1614
|
+
_report_held_lock(held[-1], parsed, exc, refusal=refusal)
|
|
1615
|
+
if not refusal:
|
|
1616
|
+
raise
|
|
1617
|
+
typer.echo(f"refused: {exc}", err=True)
|
|
1618
|
+
raise typer.Exit(code=1) from exc
|
|
1619
|
+
|
|
1620
|
+
typer.echo(f"closed {parsed.label}: stored at {outcome.prefix}")
|
|
1621
|
+
typer.echo(f"lock: Company Version {outcome.lock.version}, by {outcome.lock.by}")
|
|
1622
|
+
typer.echo(f"files: {len(outcome.package.files)}")
|
|
1623
|
+
|
|
1624
|
+
|
|
1625
|
+
# --- mercury --------------------------------------------------------------
|
|
1626
|
+
|
|
1627
|
+
EnvOption = Annotated[MercuryEnv, typer.Option("--env", help="Which Mercury deployment to read.")]
|
|
1628
|
+
PruneOption = Annotated[
|
|
1629
|
+
bool,
|
|
1630
|
+
typer.Option(
|
|
1631
|
+
"--prune",
|
|
1632
|
+
help=(
|
|
1633
|
+
"Also delete unmanaged categories -- but only ones no transaction carries. "
|
|
1634
|
+
"Reads every transaction on every account to find out."
|
|
1635
|
+
),
|
|
1636
|
+
),
|
|
1637
|
+
]
|
|
1638
|
+
StateOption = Annotated[
|
|
1639
|
+
pathlib.Path | None, typer.Option("--state", help="Ingest state file; defaults to the XDG state path.")
|
|
1640
|
+
]
|
|
1641
|
+
DryRunOption = Annotated[bool, typer.Option("--dry-run", help="Report what would post; write nothing.")]
|
|
1642
|
+
|
|
1643
|
+
|
|
1644
|
+
def _open_state(books: Books, env: MercuryEnv, target: Target, state: pathlib.Path | None) -> IngestState:
|
|
1645
|
+
"""The state file for one Mercury deployment and one ledger, and no other.
|
|
1646
|
+
|
|
1647
|
+
The default path keeps them apart by name and the recorded identity keeps
|
|
1648
|
+
them apart when `--state` names a file directly. The single place either is
|
|
1649
|
+
decided, so no command can reach for the other's cursor.
|
|
1650
|
+
"""
|
|
1651
|
+
identity = f"{env.value}/{target.value}"
|
|
1652
|
+
try:
|
|
1653
|
+
return IngestState(state or default_state_path(books.name, env.value, target.value), identity=identity)
|
|
1654
|
+
except StateIdentityError as exc:
|
|
1655
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
1656
|
+
|
|
1657
|
+
|
|
1658
|
+
def _refuse_sandbox_into_the_ledger(env: MercuryEnv, target: Target) -> None:
|
|
1659
|
+
"""The one combination that is never wanted: seeded sandbox data in the real books."""
|
|
1660
|
+
if env is MercuryEnv.SANDBOX and target is Target.BOOKS:
|
|
1661
|
+
raise typer.BadParameter(
|
|
1662
|
+
"refusing to post sandbox transactions into the books ledger; "
|
|
1663
|
+
"the sandbox is seeded data and the ledger is the company's"
|
|
1664
|
+
)
|
|
1665
|
+
|
|
1666
|
+
|
|
1667
|
+
def _print_ledger_outcome(report: IngestReport) -> None:
|
|
1668
|
+
"""What reached the ledger, and what did not."""
|
|
1669
|
+
typer.echo(f"submitted {len(report.submitted):4} (new Bank Transactions, ready for reconciliation)")
|
|
1670
|
+
typer.echo(f"drafted {len(report.drafted):4} (draft Journal Entries for coded rows; a person submits them)")
|
|
1671
|
+
typer.echo(f"updated {len(report.updated):4} (Bank Transactions and draft vouchers brought up to date)")
|
|
1672
|
+
typer.echo(f"adopted {len(report.adopted):4} (hand-imported rows recognised by their trace number)")
|
|
1673
|
+
typer.echo(f"attached {len(report.attached):4} (Mercury receipts now on their Bank Transaction)")
|
|
1674
|
+
for line in report.attached:
|
|
1675
|
+
typer.echo(f" . {line}")
|
|
1676
|
+
if report.planned:
|
|
1677
|
+
typer.echo(f"planned {len(report.planned):4} (dry run; nothing was written)")
|
|
1678
|
+
for line in report.planned:
|
|
1679
|
+
typer.echo(f" . {line}")
|
|
1680
|
+
typer.echo(f"skipped {len(report.skipped):4} (not settled)")
|
|
1681
|
+
|
|
1682
|
+
|
|
1683
|
+
def _print_coverage(report: IngestReport) -> None:
|
|
1684
|
+
"""The two ways the ledger can look fine while being incomplete."""
|
|
1685
|
+
if report.uncoded:
|
|
1686
|
+
# Rows the person reconciling gets no hint for; somebody codes each one
|
|
1687
|
+
# in Mercury or by hand.
|
|
1688
|
+
typer.echo(f"uncoded {len(report.uncoded):4} (settled, non-transfer, no GL code)")
|
|
1689
|
+
for transaction_id, age in report.uncoded[:10]:
|
|
1690
|
+
typer.echo(f" ! {transaction_id} {age}d")
|
|
1691
|
+
for check in report.balances:
|
|
1692
|
+
verdict = "ok" if check.ok else f"OFF BY {check.difference}"
|
|
1693
|
+
typer.echo(f"balance {check.label}: reported {check.reported} computed {check.computed} {verdict}")
|
|
1694
|
+
|
|
1695
|
+
|
|
1696
|
+
def _print_closed(report: IngestReport) -> None:
|
|
1697
|
+
"""Rows dated in a closed quarter: posted to the bank, drafted to nobody."""
|
|
1698
|
+
if not report.closed:
|
|
1699
|
+
return
|
|
1700
|
+
typer.echo(f"closed {len(report.closed):4} (closed period; no draft voucher -- reopen the quarter to book it)")
|
|
1701
|
+
for line in report.closed:
|
|
1702
|
+
typer.echo(f" ! {line}")
|
|
1703
|
+
|
|
1704
|
+
|
|
1705
|
+
def _desk_url(site_url: str, doctype: str, name: str) -> str:
|
|
1706
|
+
"""The document's page in the ERPNext desk: `/app/<doctype slug>/<name>`."""
|
|
1707
|
+
slug = doctype.lower().replace(" ", "-")
|
|
1708
|
+
return f"{site_url.rstrip('/')}/app/{slug}/{urllib.parse.quote(name)}"
|
|
1709
|
+
|
|
1710
|
+
|
|
1711
|
+
def _print_unneeded(report: IngestReport, site_url: str) -> None:
|
|
1712
|
+
"""Drafts of ours nothing needs any more, each linked so a person can open and delete it."""
|
|
1713
|
+
if not report.unneeded:
|
|
1714
|
+
return
|
|
1715
|
+
typer.echo(f"delete {len(report.unneeded):4} (draft Journal Entries no longer needed; a person deletes them)")
|
|
1716
|
+
for draft in report.unneeded:
|
|
1717
|
+
typer.echo(f" ! {draft.journal_entry} for {draft.transaction_id}: {draft.why}")
|
|
1718
|
+
typer.echo(f" {_desk_url(site_url, JOURNAL_ENTRY, draft.journal_entry)}")
|
|
1719
|
+
|
|
1720
|
+
|
|
1721
|
+
def _print_report(report: IngestReport, site_url: str) -> None:
|
|
1722
|
+
_print_ledger_outcome(report)
|
|
1723
|
+
_print_coverage(report)
|
|
1724
|
+
_print_closed(report)
|
|
1725
|
+
_print_unneeded(report, site_url)
|
|
1726
|
+
if report.attention:
|
|
1727
|
+
typer.echo(f"ATTENTION {len(report.attention):4}")
|
|
1728
|
+
for transaction_id, reason in report.attention:
|
|
1729
|
+
typer.echo(f" ! {transaction_id} {reason}")
|
|
1730
|
+
typer.echo(
|
|
1731
|
+
f"events read {report.events_read} applied {report.events_applied} "
|
|
1732
|
+
f"deduplicated {report.events_deduplicated} out-of-order {report.events_out_of_order}"
|
|
1733
|
+
)
|
|
1734
|
+
typer.echo(f"cursor {report.cursor}")
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
def _run_ingest(
|
|
1738
|
+
books: Books,
|
|
1739
|
+
env: MercuryEnv,
|
|
1740
|
+
target: Target,
|
|
1741
|
+
company: str | None,
|
|
1742
|
+
state: pathlib.Path | None,
|
|
1743
|
+
dry_run: bool,
|
|
1744
|
+
*,
|
|
1745
|
+
snapshot: bool,
|
|
1746
|
+
) -> None:
|
|
1747
|
+
_refuse_sandbox_into_the_ledger(env, target)
|
|
1748
|
+
# The one client in this CLI built with submit capability; see ingest_config.
|
|
1749
|
+
config = ingest_config(books.site, target)
|
|
1750
|
+
typer.echo(f"site: {config.url}")
|
|
1751
|
+
typer.echo(f"mercury: {BASE_URLS[env]}")
|
|
1752
|
+
with (
|
|
1753
|
+
MercuryClient(mercury_config(books.site, env)) as mercury,
|
|
1754
|
+
Client(config) as ledger,
|
|
1755
|
+
# Its own pool: a receipt comes from S3 on a presigned URL, not from
|
|
1756
|
+
# Mercury's API host, and it is closed with the run that opened it.
|
|
1757
|
+
ReceiptFetcher() as receipts,
|
|
1758
|
+
_open_state(books, env, target, state) as ingest_state,
|
|
1759
|
+
):
|
|
1760
|
+
try:
|
|
1761
|
+
ctx = posting_context(
|
|
1762
|
+
ledger,
|
|
1763
|
+
mapping=load_mapping(books.mercury_path),
|
|
1764
|
+
chart=load_chart(books.chart_path),
|
|
1765
|
+
bot_user=books.bot.user,
|
|
1766
|
+
mercury=mercury,
|
|
1767
|
+
company=company,
|
|
1768
|
+
)
|
|
1769
|
+
except PeriodError as exc:
|
|
1770
|
+
# A bot that cannot see Books Closed Through: refused before
|
|
1771
|
+
# anything posts, rather than drafting into a closed quarter.
|
|
1772
|
+
typer.echo(f"refused: {exc}", err=True)
|
|
1773
|
+
raise typer.Exit(code=1) from exc
|
|
1774
|
+
run = run_snapshot if snapshot else run_sync
|
|
1775
|
+
report = run(
|
|
1776
|
+
mercury,
|
|
1777
|
+
ledger,
|
|
1778
|
+
ctx,
|
|
1779
|
+
ingest_state,
|
|
1780
|
+
now=dt.datetime.now(dt.UTC),
|
|
1781
|
+
dry_run=dry_run,
|
|
1782
|
+
receipts=receipts,
|
|
1783
|
+
)
|
|
1784
|
+
_print_report(report, config.url)
|
|
1785
|
+
if report.needs_attention:
|
|
1786
|
+
raise typer.Exit(code=1)
|
|
1787
|
+
|
|
1788
|
+
|
|
1789
|
+
@mercury_app.command("snapshot")
|
|
1790
|
+
def mercury_snapshot(
|
|
1791
|
+
ctx: typer.Context,
|
|
1792
|
+
env: EnvOption,
|
|
1793
|
+
target: TargetOption,
|
|
1794
|
+
company: CompanyOption = None,
|
|
1795
|
+
state: StateOption = None,
|
|
1796
|
+
dry_run: DryRunOption = False,
|
|
1797
|
+
) -> None:
|
|
1798
|
+
"""Establish initial state from transaction lists, and record where in the event log it was true.
|
|
1799
|
+
|
|
1800
|
+
Also the recovery path: events are retained 90 days, so a gap wider than
|
|
1801
|
+
that is closed by re-snapshotting rather than by resuming a cursor into a
|
|
1802
|
+
hole, which `runningtab mercury sync` refuses to do.
|
|
1803
|
+
"""
|
|
1804
|
+
_run_ingest(_books(ctx), env, target, company, state, dry_run, snapshot=True)
|
|
1805
|
+
|
|
1806
|
+
|
|
1807
|
+
@mercury_app.command("sync")
|
|
1808
|
+
def mercury_sync(
|
|
1809
|
+
ctx: typer.Context,
|
|
1810
|
+
env: EnvOption,
|
|
1811
|
+
target: TargetOption,
|
|
1812
|
+
company: CompanyOption = None,
|
|
1813
|
+
state: StateOption = None,
|
|
1814
|
+
dry_run: DryRunOption = False,
|
|
1815
|
+
) -> None:
|
|
1816
|
+
"""Apply every event since the cursor. The same applier a webhook will feed."""
|
|
1817
|
+
try:
|
|
1818
|
+
_run_ingest(_books(ctx), env, target, company, state, dry_run, snapshot=False)
|
|
1819
|
+
except NoCursorError as exc:
|
|
1820
|
+
typer.echo(str(exc))
|
|
1821
|
+
raise typer.Exit(code=1) from exc
|
|
1822
|
+
except StaleCursorError as exc:
|
|
1823
|
+
typer.echo(str(exc))
|
|
1824
|
+
typer.echo("run `runningtab mercury snapshot` to re-establish state")
|
|
1825
|
+
raise typer.Exit(code=1) from exc
|
|
1826
|
+
|
|
1827
|
+
|
|
1828
|
+
@mercury_app.command("status")
|
|
1829
|
+
def mercury_status(ctx: typer.Context, env: EnvOption, target: TargetOption, state: StateOption = None) -> None:
|
|
1830
|
+
"""Where the ingest thinks it is: the cursor, its age, and how many events it has seen.
|
|
1831
|
+
|
|
1832
|
+
Named the same way a run is, because the cursor it reports belongs to one
|
|
1833
|
+
deployment and one ledger. A `--state` file written by a different pair is
|
|
1834
|
+
refused here rather than reported as though it answered the question asked.
|
|
1835
|
+
"""
|
|
1836
|
+
books = _books(ctx)
|
|
1837
|
+
path = state or default_state_path(books.name, env.value, target.value)
|
|
1838
|
+
if not path.exists():
|
|
1839
|
+
typer.echo(f"no state at {path}; run `runningtab mercury snapshot` first")
|
|
1840
|
+
raise typer.Exit(code=1)
|
|
1841
|
+
with _open_state(books, env, target, path) as ingest_state:
|
|
1842
|
+
cursor = ingest_state.cursor()
|
|
1843
|
+
typer.echo(f"state {path}")
|
|
1844
|
+
typer.echo(f"events {ingest_state.seen_count()} applied")
|
|
1845
|
+
if cursor is None:
|
|
1846
|
+
typer.echo("cursor none; nothing has been snapshotted")
|
|
1847
|
+
raise typer.Exit(code=1)
|
|
1848
|
+
age = dt.datetime.now(dt.UTC) - cursor.synced_at
|
|
1849
|
+
typer.echo(f"cursor {cursor.event_id} as of {cursor.synced_at.isoformat()} ({age.days}d ago)")
|
|
1850
|
+
if age > RETENTION:
|
|
1851
|
+
typer.echo(f"STALE: past Mercury's {RETENTION.days}-day retention; re-snapshot rather than resuming")
|
|
1852
|
+
raise typer.Exit(code=1)
|
|
1853
|
+
|
|
1854
|
+
|
|
1855
|
+
@mercury_app.command("install-fields")
|
|
1856
|
+
def mercury_install_fields(ctx: typer.Context, target: TargetOption, yes: YesOption = False) -> None:
|
|
1857
|
+
"""Create the custom fields the ingest writes, as an administrator.
|
|
1858
|
+
|
|
1859
|
+
The books bot cannot do this: its role carries no Custom Field permission,
|
|
1860
|
+
deliberately. `mercury_transaction_id` is unique on Bank Transaction, so
|
|
1861
|
+
once these exist the server itself refuses a second row for the same
|
|
1862
|
+
transaction. On Journal Entry it is not unique, so a person can amend a
|
|
1863
|
+
voucher; the ingest refuses a second live voucher itself. Re-running
|
|
1864
|
+
repairs a field installed with the old flags. The GL code and note are
|
|
1865
|
+
writable so a statement export imported by hand can map its columns onto
|
|
1866
|
+
them.
|
|
1867
|
+
"""
|
|
1868
|
+
config = admin_config(_books(ctx).site, target)
|
|
1869
|
+
typer.echo(f"site: {config.url}")
|
|
1870
|
+
_confirm(yes, "create the Mercury custom fields on Bank Transaction and Journal Entry?")
|
|
1871
|
+
with Client(config) as client:
|
|
1872
|
+
for label, outcome in install_custom_fields(client):
|
|
1873
|
+
typer.echo(f" {outcome:9} {label}")
|
|
1874
|
+
|
|
1875
|
+
|
|
1876
|
+
@mercury_app.command("install-bank-accounts")
|
|
1877
|
+
def mercury_install_bank_accounts(ctx: typer.Context, target: TargetOption, yes: YesOption = False) -> None:
|
|
1878
|
+
"""Create the Bank and one Bank Account per mapped Mercury account, as an administrator.
|
|
1879
|
+
|
|
1880
|
+
The books bot cannot do this: its role carries no Bank or Bank Account
|
|
1881
|
+
permission. Which accounts, and what they are called, comes from the
|
|
1882
|
+
bank_accounts table in mercury.toml; each links the chart account the
|
|
1883
|
+
accounts table gives the same Mercury account. Re-running checks and
|
|
1884
|
+
repairs rather than duplicating.
|
|
1885
|
+
"""
|
|
1886
|
+
books = _books(ctx)
|
|
1887
|
+
config = admin_config(books.site, target)
|
|
1888
|
+
typer.echo(f"site: {config.url}")
|
|
1889
|
+
with Client(config) as client:
|
|
1890
|
+
context = posting_context(
|
|
1891
|
+
client,
|
|
1892
|
+
mapping=load_mapping(books.mercury_path),
|
|
1893
|
+
chart=load_chart(books.chart_path),
|
|
1894
|
+
bot_user=books.bot.user,
|
|
1895
|
+
)
|
|
1896
|
+
try:
|
|
1897
|
+
planned = bank_account_documents(context)
|
|
1898
|
+
except (PostingError, MappingError) as exc:
|
|
1899
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
1900
|
+
_confirm(yes, f"create the Bank '{BANK}' and {len(planned)} Bank Account document(s) on {config.url}?")
|
|
1901
|
+
for label, outcome in install_bank_accounts(client, context):
|
|
1902
|
+
typer.echo(f" {outcome:9} {label}")
|
|
1903
|
+
|
|
1904
|
+
|
|
1905
|
+
# --- mercury categories ---------------------------------------------------
|
|
1906
|
+
|
|
1907
|
+
|
|
1908
|
+
def _category_plan(books: Books, env: MercuryEnv, *, prune: bool = False) -> CategoryPlan:
|
|
1909
|
+
"""Read the chart and Mercury's categories, and reconcile them. Uses the read-only token.
|
|
1910
|
+
|
|
1911
|
+
Pruning costs a second, much larger read: every transaction on every
|
|
1912
|
+
account, through the same `read_everything` the ingest trusts, so that
|
|
1913
|
+
"nothing carries this category" is answered from the whole bank rather than
|
|
1914
|
+
from a recent window. It refuses a short read, which is the point -- a
|
|
1915
|
+
category that looks unused because the read stopped early is a category
|
|
1916
|
+
somebody's coding is about to be deleted off.
|
|
1917
|
+
"""
|
|
1918
|
+
typer.echo(f"mercury: {BASE_URLS[env]}")
|
|
1919
|
+
with MercuryClient(mercury_config(books.site, env)) as mercury:
|
|
1920
|
+
live = mercury.categories()
|
|
1921
|
+
carried = None
|
|
1922
|
+
if prune:
|
|
1923
|
+
transactions, _, _ = read_everything(mercury)
|
|
1924
|
+
typer.echo(f"read {len(transactions)} transactions to find which categories are in use")
|
|
1925
|
+
carried = carried_categories(transactions)
|
|
1926
|
+
return plan_categories(desired_categories(load_chart_entries(books.chart_path)), live, carried=carried)
|
|
1927
|
+
|
|
1928
|
+
|
|
1929
|
+
def _unmanaged_flags(unmanaged: UnmanagedCategory) -> str:
|
|
1930
|
+
"""The same `CRO` column for a category the chart does not name."""
|
|
1931
|
+
return "".join(
|
|
1932
|
+
letter if value else "-" for letter, value in zip("CRO", live_flags(unmanaged.category).values(), strict=True)
|
|
1933
|
+
)
|
|
1934
|
+
|
|
1935
|
+
|
|
1936
|
+
def _print_category_plan(plan: CategoryPlan, *, prune: bool) -> None:
|
|
1937
|
+
"""One line per action, each carrying what a person would have to type in the dashboard.
|
|
1938
|
+
|
|
1939
|
+
Mercury's write endpoints need a token this repo refuses to store, so the
|
|
1940
|
+
plan has to stand on its own as a checklist somebody can work through by
|
|
1941
|
+
hand: the exact category name and the three flags, in chart order.
|
|
1942
|
+
|
|
1943
|
+
Under `--prune` each unmanaged category is reported as what it is -- a
|
|
1944
|
+
`delete`, or `in use` with the number of transactions that keep it -- so
|
|
1945
|
+
the list a person is asked to approve names the deletions individually
|
|
1946
|
+
rather than as a count.
|
|
1947
|
+
"""
|
|
1948
|
+
typer.echo("flags: C card spend, R reimbursements, O other; - off")
|
|
1949
|
+
typer.echo("")
|
|
1950
|
+
for create in plan.creates:
|
|
1951
|
+
typer.echo(f"create {create.desired.flag_label} {create.desired.name}")
|
|
1952
|
+
for edit in plan.edits:
|
|
1953
|
+
changes = "; ".join(str(change) for change in edit.changes)
|
|
1954
|
+
typer.echo(f"edit {edit.desired.flag_label} {edit.desired.name} {changes}")
|
|
1955
|
+
if prune:
|
|
1956
|
+
for delete in plan.deletes:
|
|
1957
|
+
typer.echo(f"delete {delete.category.name}")
|
|
1958
|
+
for one in plan.in_use:
|
|
1959
|
+
typer.echo(f"in use {one}")
|
|
1960
|
+
else:
|
|
1961
|
+
for unmanaged in plan.unmanaged:
|
|
1962
|
+
typer.echo(f"unmanaged {_unmanaged_flags(unmanaged)} {unmanaged.category.name}")
|
|
1963
|
+
for refusal in plan.refusals:
|
|
1964
|
+
typer.echo(f"REFUSED {refusal}")
|
|
1965
|
+
typer.echo("")
|
|
1966
|
+
typer.echo(f"matches {len(plan.matches):4}")
|
|
1967
|
+
typer.echo(f"create {len(plan.creates):4}")
|
|
1968
|
+
typer.echo(f"edit {len(plan.edits):4}")
|
|
1969
|
+
if prune:
|
|
1970
|
+
typer.echo(f"delete {len(plan.deletes):4} (unmanaged, and no transaction carries them)")
|
|
1971
|
+
typer.echo(f"in use {len(plan.in_use):4} (re-code these in Mercury, or leave them)")
|
|
1972
|
+
else:
|
|
1973
|
+
typer.echo(f"unmanaged {len(plan.unmanaged):4} (Mercury's own, or somebody's; never touched)")
|
|
1974
|
+
if plan.refusals:
|
|
1975
|
+
typer.echo(f"REFUSED {len(plan.refusals):4}")
|
|
1976
|
+
|
|
1977
|
+
|
|
1978
|
+
@categories_app.command("plan")
|
|
1979
|
+
def mercury_categories_plan(ctx: typer.Context, env: EnvOption, prune: PruneOption = False) -> None:
|
|
1980
|
+
"""Report how Mercury's categories differ from the chart. Writes nothing; exits 1 on any drift.
|
|
1981
|
+
|
|
1982
|
+
Cheap enough to be a standing check, which is the point: a category whose
|
|
1983
|
+
name no longer matches a GL code derives nothing through Mercury's rules,
|
|
1984
|
+
so the transaction arrives uncoded and nobody finds out for weeks.
|
|
1985
|
+
|
|
1986
|
+
`--prune` shows what `apply --prune` would delete. It is still read-only,
|
|
1987
|
+
and it is the only way to see that list before agreeing to it.
|
|
1988
|
+
"""
|
|
1989
|
+
try:
|
|
1990
|
+
plan = _category_plan(_books(ctx), env, prune=prune)
|
|
1991
|
+
except MercuryError as exc:
|
|
1992
|
+
typer.echo(f"REFUSED {exc}")
|
|
1993
|
+
raise typer.Exit(code=1) from exc
|
|
1994
|
+
_print_category_plan(plan, prune=prune)
|
|
1995
|
+
if plan.has_drift:
|
|
1996
|
+
raise typer.Exit(code=1)
|
|
1997
|
+
|
|
1998
|
+
|
|
1999
|
+
@categories_app.command("apply")
|
|
2000
|
+
def mercury_categories_apply(
|
|
2001
|
+
ctx: typer.Context, env: EnvOption, yes: YesOption = False, prune: PruneOption = False
|
|
2002
|
+
) -> None:
|
|
2003
|
+
"""Create the categories the chart wants and fix the flags on the ones that exist.
|
|
2004
|
+
|
|
2005
|
+
Never renames: the name is what every transaction already carrying a
|
|
2006
|
+
category appears to say, and the match key besides.
|
|
2007
|
+
|
|
2008
|
+
Deletes only under `--prune`, and then only a category the bank says no
|
|
2009
|
+
transaction carries -- a deletion strips somebody's coding off their rows
|
|
2010
|
+
and nothing reproduces it. An unmanaged category that is in use is reported
|
|
2011
|
+
with its count and left alone; re-code those rows in Mercury and run again.
|
|
2012
|
+
|
|
2013
|
+
The token comes from `MERCURY_ADMIN_TOKEN` (or `MERCURY_SANDBOX_TOKEN`) and
|
|
2014
|
+
from nowhere else. Mercury issues no category-only write scope, so that is
|
|
2015
|
+
a Read and Write token which can also send money -- mint it, run this,
|
|
2016
|
+
revoke it. The refusal for an unset variable lands before any request.
|
|
2017
|
+
"""
|
|
2018
|
+
books = _books(ctx)
|
|
2019
|
+
try:
|
|
2020
|
+
config = mercury_write_config(env)
|
|
2021
|
+
except SiteCredentialError as exc:
|
|
2022
|
+
typer.echo(str(exc))
|
|
2023
|
+
raise typer.Exit(code=1) from exc
|
|
2024
|
+
try:
|
|
2025
|
+
plan = _category_plan(books, env, prune=prune)
|
|
2026
|
+
except MercuryError as exc:
|
|
2027
|
+
typer.echo(f"REFUSED {exc}")
|
|
2028
|
+
raise typer.Exit(code=1) from exc
|
|
2029
|
+
_print_category_plan(plan, prune=prune)
|
|
2030
|
+
if not plan.has_work:
|
|
2031
|
+
typer.echo("nothing to write")
|
|
2032
|
+
raise typer.Exit(code=1 if plan.refusals or plan.in_use else 0)
|
|
2033
|
+
work = f"create {len(plan.creates)} and edit {len(plan.edits)}"
|
|
2034
|
+
if prune:
|
|
2035
|
+
work += f" and DELETE {len(plan.deletes)}"
|
|
2036
|
+
_confirm(yes, f"{work} Mercury categories?")
|
|
2037
|
+
failed = False
|
|
2038
|
+
with MercuryCategoryWriter(config) as writer:
|
|
2039
|
+
try:
|
|
2040
|
+
for applied in apply_plan(writer, plan):
|
|
2041
|
+
if applied.error is not None:
|
|
2042
|
+
typer.echo(f" REFUSED {applied.category_id} {applied.name} {applied.error}")
|
|
2043
|
+
failed = True
|
|
2044
|
+
else:
|
|
2045
|
+
typer.echo(f" {applied.action:8} {applied.category_id} {applied.name}")
|
|
2046
|
+
except MercuryError as exc:
|
|
2047
|
+
# Everything before this has already printed. A run that wrote four
|
|
2048
|
+
# things and then failed must not read as a run that wrote nothing.
|
|
2049
|
+
typer.echo(f"FAILED {exc}")
|
|
2050
|
+
failed = True
|
|
2051
|
+
if failed or plan.refusals or plan.in_use:
|
|
2052
|
+
raise typer.Exit(code=1)
|
|
2053
|
+
|
|
2054
|
+
|
|
2055
|
+
# --- forecast -------------------------------------------------------------
|
|
2056
|
+
|
|
2057
|
+
forecast_app = typer.Typer(help="The cash forecast and the capital call it asks of members.", no_args_is_help=True)
|
|
2058
|
+
app.add_typer(forecast_app, name="forecast")
|
|
2059
|
+
|
|
2060
|
+
|
|
2061
|
+
def _now() -> dt.datetime:
|
|
2062
|
+
"""Local time with its offset, as its own name so a test can patch it.
|
|
2063
|
+
|
|
2064
|
+
Local, not UTC: an evening run in the US is already tomorrow in UTC, and
|
|
2065
|
+
the forecast's window starts today.
|
|
2066
|
+
"""
|
|
2067
|
+
return dt.datetime.now().astimezone()
|
|
2068
|
+
|
|
2069
|
+
|
|
2070
|
+
def _usd(value: Decimal) -> str:
|
|
2071
|
+
return f"-${-value:,.2f}" if value < 0 else f"${value:,.2f}"
|
|
2072
|
+
|
|
2073
|
+
|
|
2074
|
+
@contextmanager
|
|
2075
|
+
def _forecast_refusals(provision: str | None = None) -> Generator[None]:
|
|
2076
|
+
"""Turn every failure a person can act on into one sentence and exit 1.
|
|
2077
|
+
|
|
2078
|
+
`provision` is the command that grants this credential what the forecast
|
|
2079
|
+
reads; a permission refusal names it, since re-provisioning is the fix.
|
|
2080
|
+
Anything else is a bug and keeps its traceback.
|
|
2081
|
+
"""
|
|
2082
|
+
try:
|
|
2083
|
+
yield
|
|
2084
|
+
except PermissionError_ as exc:
|
|
2085
|
+
if provision is None:
|
|
2086
|
+
raise
|
|
2087
|
+
refusal = re.sub(r"<[^>]+>", "", exc.message)
|
|
2088
|
+
typer.echo(f"refused: {refusal}; run `{provision}` as an administrator", err=True)
|
|
2089
|
+
raise typer.Exit(code=1) from exc
|
|
2090
|
+
except (SourceError, ForecastFileError, MercuryError, SiteCredentialError) as exc:
|
|
2091
|
+
typer.echo(f"refused: {exc}", err=True)
|
|
2092
|
+
raise typer.Exit(code=1) from exc
|
|
2093
|
+
except FrappeError as exc:
|
|
2094
|
+
sentence = maintenance_message(exc)
|
|
2095
|
+
if sentence is None:
|
|
2096
|
+
raise
|
|
2097
|
+
typer.echo(f"refused: {sentence}", err=True)
|
|
2098
|
+
raise typer.Exit(code=1) from exc
|
|
2099
|
+
|
|
2100
|
+
|
|
2101
|
+
def _print_forecast(forecast: Forecast, name_of: Callable[[str], str]) -> None:
|
|
2102
|
+
typer.echo(f"window: {forecast.window_start} to {forecast.window_end}")
|
|
2103
|
+
typer.echo(f"opening cash: {_usd(forecast.opening_cash)}")
|
|
2104
|
+
typer.echo(f"ledger through: {forecast.ledger_through}")
|
|
2105
|
+
typer.echo("")
|
|
2106
|
+
typer.echo(f"{'month':<10} {'opening':>12} {'outflows':>12} {'inflows':>12} {'closing':>12}")
|
|
2107
|
+
for m in forecast.months:
|
|
2108
|
+
typer.echo(
|
|
2109
|
+
f"{m.month:%Y-%m} {_usd(m.opening):>12} {_usd(m.outflows):>12} {_usd(m.inflows):>12} {_usd(m.closing):>12}"
|
|
2110
|
+
)
|
|
2111
|
+
typer.echo("")
|
|
2112
|
+
typer.echo("dues:")
|
|
2113
|
+
for d in forecast.dues:
|
|
2114
|
+
typer.echo(f" {d.due} {d.line.name:<30} {_usd(d.amount):>12} {d.status}")
|
|
2115
|
+
typer.echo("")
|
|
2116
|
+
typer.echo(f"floor: {_usd(forecast.floor)} (from {forecast.floor_from})")
|
|
2117
|
+
typer.echo(f"total call: {_usd(forecast.total_call)}")
|
|
2118
|
+
typer.echo("per member:")
|
|
2119
|
+
for number, amount in forecast.members:
|
|
2120
|
+
typer.echo(f" {name_of(number)}: {_usd(amount)}")
|
|
2121
|
+
|
|
2122
|
+
|
|
2123
|
+
def _refuse_sandbox_forecast(env: MercuryEnv, target: Target) -> None:
|
|
2124
|
+
"""Refuse, before any request, a capital call computed from seeded sandbox data."""
|
|
2125
|
+
if env is MercuryEnv.SANDBOX and target is Target.BOOKS:
|
|
2126
|
+
raise typer.BadParameter(
|
|
2127
|
+
"refusing to forecast the books from sandbox data; the sandbox is seeded and the capital call is real money"
|
|
2128
|
+
)
|
|
2129
|
+
|
|
2130
|
+
|
|
2131
|
+
def _compute_forecast(books: Books, env: MercuryEnv, ledger: Client) -> Forecast:
|
|
2132
|
+
forecast_file = load_forecast(books.forecast_path, Chart.load(load_chart(books.chart_path)))
|
|
2133
|
+
now = _now()
|
|
2134
|
+
with MercuryClient(mercury_config(books.site, env)) as mercury:
|
|
2135
|
+
transactions, balances, _ = read_everything(mercury)
|
|
2136
|
+
return compute(
|
|
2137
|
+
forecast_file,
|
|
2138
|
+
now=now,
|
|
2139
|
+
opening_cash=opening_cash(balances),
|
|
2140
|
+
ledger_through=ledger_through(ledger, now.date()),
|
|
2141
|
+
transactions=transactions,
|
|
2142
|
+
)
|
|
2143
|
+
|
|
2144
|
+
|
|
2145
|
+
def _account_names(books: Books) -> dict[str, str]:
|
|
2146
|
+
return Chart.load(load_chart(books.chart_path)).postable
|
|
2147
|
+
|
|
2148
|
+
|
|
2149
|
+
@forecast_app.command("plan")
|
|
2150
|
+
def forecast_plan(ctx: typer.Context, env: EnvOption, target: TargetOption) -> None:
|
|
2151
|
+
"""Compute the forecast and print it. Reads Mercury and the ledger; writes nothing."""
|
|
2152
|
+
books = _books(ctx)
|
|
2153
|
+
_refuse_sandbox_forecast(env, target)
|
|
2154
|
+
with _forecast_refusals():
|
|
2155
|
+
config = site_config(books.site, target, readonly=True)
|
|
2156
|
+
typer.echo(f"site: {config.url}")
|
|
2157
|
+
names = _account_names(books)
|
|
2158
|
+
with (
|
|
2159
|
+
_forecast_refusals(f"runningtab access provision --target {target.value} --profile reader"),
|
|
2160
|
+
Client(config) as ledger,
|
|
2161
|
+
):
|
|
2162
|
+
forecast = _compute_forecast(books, env, ledger)
|
|
2163
|
+
_print_forecast(forecast, lambda number: f"{number} - {names[number]}")
|
|
2164
|
+
|
|
2165
|
+
|
|
2166
|
+
@forecast_app.command("apply")
|
|
2167
|
+
def forecast_apply(ctx: typer.Context, env: EnvOption, target: TargetOption) -> None:
|
|
2168
|
+
"""Compute the forecast, print it, and record it as a new Cash Forecast document.
|
|
2169
|
+
|
|
2170
|
+
Each run adds a document and never edits an earlier one, so what a past
|
|
2171
|
+
run told the members stays on record.
|
|
2172
|
+
"""
|
|
2173
|
+
books = _books(ctx)
|
|
2174
|
+
_refuse_sandbox_forecast(env, target)
|
|
2175
|
+
with _forecast_refusals():
|
|
2176
|
+
config = site_config(books.site, target)
|
|
2177
|
+
typer.echo(f"site: {config.url}")
|
|
2178
|
+
names = _account_names(books)
|
|
2179
|
+
with _forecast_refusals(f"runningtab access provision --target {target.value}"), Client(config) as ledger:
|
|
2180
|
+
if not is_installed(ledger):
|
|
2181
|
+
typer.echo(_doctype_missing_sentence(config.url, target), err=True)
|
|
2182
|
+
raise typer.Exit(code=1)
|
|
2183
|
+
forecast = _compute_forecast(books, env, ledger)
|
|
2184
|
+
_print_forecast(forecast, lambda number: f"{number} - {names[number]}")
|
|
2185
|
+
try:
|
|
2186
|
+
abbr = company_abbr(ledger, _sole_company(ledger))
|
|
2187
|
+
except LookupError as exc:
|
|
2188
|
+
raise SourceError(str(exc)) from exc
|
|
2189
|
+
try:
|
|
2190
|
+
name = write_forecast(ledger, forecast, link=lambda number: document_name(names[number], abbr, number))
|
|
2191
|
+
except PermissionError_ as exc:
|
|
2192
|
+
typer.echo(
|
|
2193
|
+
f"the books bot may not create {CASH_FORECAST} documents on {config.url.rstrip('/')}; "
|
|
2194
|
+
f"run `runningtab access provision --target {target.value}` as an administrator first",
|
|
2195
|
+
err=True,
|
|
2196
|
+
)
|
|
2197
|
+
raise typer.Exit(code=1) from exc
|
|
2198
|
+
typer.echo("")
|
|
2199
|
+
typer.echo(f"wrote {name}")
|
|
2200
|
+
typer.echo(_desk_url(config.url, CASH_FORECAST, name))
|
|
2201
|
+
|
|
2202
|
+
|
|
2203
|
+
@forecast_app.command("install-doctype")
|
|
2204
|
+
def forecast_install_doctype(ctx: typer.Context, target: TargetOption, yes: YesOption = False) -> None:
|
|
2205
|
+
"""Create the Cash Forecast doctypes, as an administrator.
|
|
2206
|
+
|
|
2207
|
+
The books bot cannot do this: it may write forecasts, not reshape
|
|
2208
|
+
doctypes. Run it before `runningtab access provision`, which grants the bot the
|
|
2209
|
+
doctype. Re-running repairs a doctype whose fields drifted.
|
|
2210
|
+
"""
|
|
2211
|
+
config = admin_config(_books(ctx).site, target)
|
|
2212
|
+
typer.echo(f"site: {config.url}")
|
|
2213
|
+
_confirm(yes, f"create the {CASH_FORECAST} doctypes on {config.url}?")
|
|
2214
|
+
with _forecast_refusals(), Client(config) as client:
|
|
2215
|
+
for label, outcome in install_doctypes(client):
|
|
2216
|
+
typer.echo(f" {outcome:9} {label}")
|