formist-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- formist_cli/__init__.py +1 -0
- formist_cli/__main__.py +3 -0
- formist_cli/airtable.py +238 -0
- formist_cli/alerts.py +207 -0
- formist_cli/api.py +1027 -0
- formist_cli/chat.py +618 -0
- formist_cli/chats.py +517 -0
- formist_cli/cli.py +1367 -0
- formist_cli/cli_support.py +221 -0
- formist_cli/connectors.py +358 -0
- formist_cli/context.py +164 -0
- formist_cli/diff.py +943 -0
- formist_cli/doctor.py +128 -0
- formist_cli/export.py +427 -0
- formist_cli/files.py +670 -0
- formist_cli/importer.py +425 -0
- formist_cli/kb.py +185 -0
- formist_cli/login.py +615 -0
- formist_cli/logs.py +781 -0
- formist_cli/meetings.py +200 -0
- formist_cli/messages.py +414 -0
- formist_cli/participants.py +402 -0
- formist_cli/prefs.py +328 -0
- formist_cli/privileges.py +375 -0
- formist_cli/projects.py +127 -0
- formist_cli/resolve.py +443 -0
- formist_cli/settings.py +199 -0
- formist_cli/skill_library.py +246 -0
- formist_cli/skills.py +308 -0
- formist_cli/space_plan.py +411 -0
- formist_cli/spaces.py +693 -0
- formist_cli/targets.py +153 -0
- formist_cli/textnorm.py +26 -0
- formist_cli/tokens.py +339 -0
- formist_cli/update.py +433 -0
- formist_cli/uploads.py +928 -0
- formist_cli/usage_tokens.py +152 -0
- formist_cli/users.py +384 -0
- formist_cli/whatsapp.py +451 -0
- formist_cli-0.1.0.dist-info/METADATA +176 -0
- formist_cli-0.1.0.dist-info/RECORD +44 -0
- formist_cli-0.1.0.dist-info/WHEEL +4 -0
- formist_cli-0.1.0.dist-info/entry_points.txt +2 -0
- formist_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
formist_cli/diff.py
ADDED
|
@@ -0,0 +1,943 @@
|
|
|
1
|
+
"""``fm diff``: what to write. ``fm update`` already answers how.
|
|
2
|
+
|
|
3
|
+
This compares a source of truth against a target environment and
|
|
4
|
+
emits a payload ``fm update`` consumes. The two compose: ``fm diff``,
|
|
5
|
+
read it, ``fm update``.
|
|
6
|
+
|
|
7
|
+
The output names rows by name, never by id. Ops carry ``$name``,
|
|
8
|
+
which ``fm update`` re-resolves against the target seconds before the
|
|
9
|
+
write and copies into ``$expect`` so the row itself refuses a
|
|
10
|
+
mis-resolution. Emitting ids here would move a transcribed-id failure
|
|
11
|
+
one file earlier and drop that copy-back. A target id never appears
|
|
12
|
+
in the output at all.
|
|
13
|
+
|
|
14
|
+
The comparison itself is where every defect in this kind of tool
|
|
15
|
+
tends to live, so its rules are deliberate and stated up front:
|
|
16
|
+
|
|
17
|
+
- Trailing and non-breaking whitespace in the match key. An untrimmed
|
|
18
|
+
name is a phantom new row, not a visible duplicate: it creates
|
|
19
|
+
silently. See :func:`normalise_key`.
|
|
20
|
+
- Curly vs straight quotes, en-dash vs hyphen, present in real names
|
|
21
|
+
on both sides.
|
|
22
|
+
- The same column holding two shapes: a value can be a bare string in
|
|
23
|
+
one environment and an ``{"en": ...}`` map in another, and the two
|
|
24
|
+
are not coalesced, since a shape change can hide a real difference.
|
|
25
|
+
Nothing is rendered through ``str()``: ``str({"en": "FAO"})`` is a
|
|
26
|
+
dict repr and matches nothing at all.
|
|
27
|
+
- Ids differ between environments. Matching is by name, and 0 or more
|
|
28
|
+
than 1 matches is a refusal rather than a choice.
|
|
29
|
+
- An id prefix is not a key: a short UUID prefix is not safe as a
|
|
30
|
+
primary key over a table where a prefix collision is possible.
|
|
31
|
+
- A row missing from the source is never authority to delete.
|
|
32
|
+
Additive and updating only; what would have been deleted is
|
|
33
|
+
reported.
|
|
34
|
+
- The equality rule used is printed, per field, on every run, so two
|
|
35
|
+
people reading the same report cannot silently disagree about what
|
|
36
|
+
"already equal" means.
|
|
37
|
+
|
|
38
|
+
Stdlib only: this tool must not add a runtime dependency, which is
|
|
39
|
+
also why xlsx is refused with a conversion hint rather than parsed by
|
|
40
|
+
hand.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
from __future__ import annotations
|
|
44
|
+
|
|
45
|
+
import argparse
|
|
46
|
+
import csv
|
|
47
|
+
import hashlib
|
|
48
|
+
import json
|
|
49
|
+
import re
|
|
50
|
+
import unicodedata
|
|
51
|
+
from dataclasses import dataclass, field
|
|
52
|
+
from typing import Any
|
|
53
|
+
|
|
54
|
+
from formist_cli import resolve as resolve_mod
|
|
55
|
+
from formist_cli import textnorm
|
|
56
|
+
from formist_cli.targets import Target
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class DiffError(Exception):
|
|
60
|
+
"""A comparison that will not be made, or a payload that will not be written."""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
#: The resolver's remedy for a truncated page is "narrow the search". A diff
|
|
64
|
+
#: has no such option: narrowing is what makes the unseen rows look absent,
|
|
65
|
+
#: and absent is what this tool reports as a would-delete. It is a whole
|
|
66
|
+
#: table or nothing, which is why these callers paginate, and why the only
|
|
67
|
+
#: honest remedy left once a paginated walk has still come up short is to
|
|
68
|
+
#: stop.
|
|
69
|
+
_PARTIAL_REMEDY = (
|
|
70
|
+
"A diff over a partial table is not a partial diff: every row the walk did not "
|
|
71
|
+
"reach would be reported as a would-delete, so there is no narrowing that helps "
|
|
72
|
+
"and nothing was compared.\n"
|
|
73
|
+
" This kind is larger than the walk's ceiling "
|
|
74
|
+
f"({resolve_mod.MAX_PAGES} x {resolve_mod.MAX_PAGE} rows), or the server stopped "
|
|
75
|
+
"returning rows before it reached the total it reported."
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# ===========================================================================
|
|
80
|
+
# The two rules. They are different on purpose, and both are printed.
|
|
81
|
+
# ===========================================================================
|
|
82
|
+
# The three character tables (which characters are a space / a quote / a
|
|
83
|
+
# dash) live in textnorm.py, byte-identical, since `fm ai-benchmark` needs
|
|
84
|
+
# the same fact under a different rule. What stays here is the rule, and it
|
|
85
|
+
# stays case-sensitive. The tables are used through `textnorm.` below rather
|
|
86
|
+
# than re-bound to local names, so there is one definition.
|
|
87
|
+
|
|
88
|
+
KEY_RULE = (
|
|
89
|
+
"NFKC -> Unicode spaces (incl. U+00A0 and the zero-width U+200B / U+FEFF, which "
|
|
90
|
+
"no trim reaches) -> U+0020 -> collapse runs -> trim -> curly quotes -> straight, "
|
|
91
|
+
"en/em/non-breaking dashes -> '-'; CASE-SENSITIVE"
|
|
92
|
+
)
|
|
93
|
+
VALUE_RULE = (
|
|
94
|
+
"exact and SHAPE-SENSITIVE: a bare string never equals an {en,...} map, and no "
|
|
95
|
+
"value is rendered through str()"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def normalise_key(value: str) -> str:
|
|
100
|
+
"""The match key. Lenient, because its job is to find the row.
|
|
101
|
+
|
|
102
|
+
The ``textnorm.SPACES`` pass runs before the trim, and that
|
|
103
|
+
ordering is the only part of this function that is load-bearing
|
|
104
|
+
rather than belt-and-braces: ``.strip()`` already removes U+00A0
|
|
105
|
+
(it is ``isspace()``), and NFKC maps it to U+0020 besides. What no
|
|
106
|
+
generic step reaches is U+200B and U+FEFF: zero-width,
|
|
107
|
+
``isspace()``-false, unmatched by ``\\s``, unchanged by NFKC. They
|
|
108
|
+
are also invisible on screen, so the phantom row they create is
|
|
109
|
+
named identically to the row it duplicates.
|
|
110
|
+
|
|
111
|
+
Case is not folded: folding case is how a resolver can pick the
|
|
112
|
+
wrong row, and this must agree with the resolver that
|
|
113
|
+
``fm update`` will use seconds later.
|
|
114
|
+
"""
|
|
115
|
+
text = unicodedata.normalize("NFKC", value)
|
|
116
|
+
for space in textnorm.SPACES:
|
|
117
|
+
text = text.replace(space, " ")
|
|
118
|
+
for src, dst in {**textnorm.QUOTES, **textnorm.DASHES}.items():
|
|
119
|
+
text = text.replace(src, dst)
|
|
120
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def values_equal(left: Any, right: Any) -> tuple[bool, str]:
|
|
124
|
+
"""Are two column values the same value? ``(equal, why)``.
|
|
125
|
+
|
|
126
|
+
Stricter than the key rule, and deliberately not the same
|
|
127
|
+
function. A key is matched so the row can be found; a value
|
|
128
|
+
decides whether to write. Shape is part of a value: ``"FAO"`` and
|
|
129
|
+
``{"en": "FAO"}`` are different, and the endpoint's own
|
|
130
|
+
``$expect`` check refuses to coalesce them.
|
|
131
|
+
"""
|
|
132
|
+
if type(left) is not type(right):
|
|
133
|
+
if _shape_of(left) != _shape_of(right):
|
|
134
|
+
# NULL is absence, not a shape. Filling an empty column is the
|
|
135
|
+
# most ordinary update there is, and blocking it would block
|
|
136
|
+
# most of the real work. The reverse is NOT symmetric and stays
|
|
137
|
+
# blocked: a source null over a populated column is a
|
|
138
|
+
# field-level delete, and "a row missing from the source is
|
|
139
|
+
# never authority to delete" is this module's rule one level
|
|
140
|
+
# down. Clearing a value takes --allow-shape-change,
|
|
141
|
+
# deliberately.
|
|
142
|
+
if left is None:
|
|
143
|
+
return False, "target holds no value"
|
|
144
|
+
return False, f"shape differs: {_shape_of(left)} vs {_shape_of(right)}"
|
|
145
|
+
if isinstance(left, str) and isinstance(right, str):
|
|
146
|
+
if left == right:
|
|
147
|
+
return True, "identical"
|
|
148
|
+
if normalise_key(left) == normalise_key(right):
|
|
149
|
+
return False, "differs only by whitespace/quote/dash normalisation"
|
|
150
|
+
return False, "differs"
|
|
151
|
+
return (True, "identical") if left == right else (False, "differs")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _shape_of(value: Any) -> str:
|
|
155
|
+
if value is None:
|
|
156
|
+
return "null"
|
|
157
|
+
if isinstance(value, str):
|
|
158
|
+
return "string"
|
|
159
|
+
if isinstance(value, dict):
|
|
160
|
+
return "map"
|
|
161
|
+
if isinstance(value, list):
|
|
162
|
+
return "list"
|
|
163
|
+
return type(value).__name__
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def key_strings(value: Any) -> list[str]:
|
|
167
|
+
"""Every normalised string a column value could be matched by.
|
|
168
|
+
|
|
169
|
+
Mirrors the resolver's own string extraction, so that a row this
|
|
170
|
+
module matches is the row ``fm update`` will resolve. Two
|
|
171
|
+
different notions of "the name of this row" in one pipeline is
|
|
172
|
+
how a diff proposes an op that then refuses.
|
|
173
|
+
"""
|
|
174
|
+
if isinstance(value, str):
|
|
175
|
+
return [normalise_key(value)]
|
|
176
|
+
if isinstance(value, dict):
|
|
177
|
+
return [normalise_key(v) for v in value.values() if isinstance(v, str)]
|
|
178
|
+
return []
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
# ===========================================================================
|
|
182
|
+
# Sources of truth
|
|
183
|
+
# ===========================================================================
|
|
184
|
+
@dataclass(frozen=True)
|
|
185
|
+
class Source:
|
|
186
|
+
rows: list[dict]
|
|
187
|
+
origin: str
|
|
188
|
+
#: sha256 of the bytes compared, so a report can be tied to an input.
|
|
189
|
+
digest: str
|
|
190
|
+
fmt: str
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def load_source(path: str) -> Source:
|
|
194
|
+
"""A JSON array / ``{"rows": [...]}`` or a CSV with a header row.
|
|
195
|
+
|
|
196
|
+
xlsx is refused rather than parsed. This tool is stdlib-only by
|
|
197
|
+
contract, and a hand-rolled xlsx reader (shared strings, date
|
|
198
|
+
serials, inline vs shared cells) is precisely the class of thing
|
|
199
|
+
that reads almost right and is believed. The refusal names the
|
|
200
|
+
conversion instead.
|
|
201
|
+
"""
|
|
202
|
+
lower = path.lower()
|
|
203
|
+
if lower.endswith((".xlsx", ".xls", ".xlsm")):
|
|
204
|
+
raise DiffError(
|
|
205
|
+
f"{path} is a spreadsheet, and fm refuses to parse one.\n"
|
|
206
|
+
"`fm` is stdlib-only so an xlsx reader here would be "
|
|
207
|
+
"hand-rolled: shared strings, date serials and inline-vs-shared cells "
|
|
208
|
+
"are where such a reader is subtly wrong, and a subtly wrong source of "
|
|
209
|
+
"truth is worse than none.\n"
|
|
210
|
+
"\n"
|
|
211
|
+
"Convert it first, then pass the result:\n"
|
|
212
|
+
" python -c \"import openpyxl,json,sys; "
|
|
213
|
+
"wb=openpyxl.load_workbook(sys.argv[1]); ws=wb.active; "
|
|
214
|
+
"rows=list(ws.values); hdr=[str(h).strip() for h in rows[0]]; "
|
|
215
|
+
"json.dump([dict(zip(hdr,r)) for r in rows[1:]], open(sys.argv[2],'w'), "
|
|
216
|
+
'default=str)" book.xlsx rows.json'
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
try:
|
|
220
|
+
raw = open(path, "rb").read()
|
|
221
|
+
except FileNotFoundError:
|
|
222
|
+
raise DiffError(f"no such source file: {path}") from None
|
|
223
|
+
except OSError as exc:
|
|
224
|
+
raise DiffError(f"could not read {path}: {exc}") from None
|
|
225
|
+
|
|
226
|
+
digest = hashlib.sha256(raw).hexdigest()
|
|
227
|
+
text = raw.decode("utf-8-sig")
|
|
228
|
+
|
|
229
|
+
if lower.endswith(".csv"):
|
|
230
|
+
reader = csv.DictReader(text.splitlines())
|
|
231
|
+
rows = [dict(r) for r in reader]
|
|
232
|
+
fmt = "csv"
|
|
233
|
+
else:
|
|
234
|
+
try:
|
|
235
|
+
payload = json.loads(text)
|
|
236
|
+
except json.JSONDecodeError as exc:
|
|
237
|
+
raise DiffError(f"{path} is not valid JSON: {exc}") from None
|
|
238
|
+
if isinstance(payload, dict):
|
|
239
|
+
payload = payload.get("rows")
|
|
240
|
+
if not isinstance(payload, list):
|
|
241
|
+
raise DiffError(
|
|
242
|
+
f"{path} must hold a JSON array of row objects, or "
|
|
243
|
+
'{"rows": [...]}, got a ' + type(payload).__name__ + "."
|
|
244
|
+
)
|
|
245
|
+
rows = [r for r in payload if isinstance(r, dict)]
|
|
246
|
+
fmt = "json"
|
|
247
|
+
|
|
248
|
+
if not rows:
|
|
249
|
+
raise DiffError(f"{path} has no rows; nothing to compare.")
|
|
250
|
+
return Source(rows=rows, origin=path, digest=digest, fmt=fmt)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def source_from_env(
|
|
254
|
+
*, kind: str, target: Target, space: str, token: str | None, fetch=None
|
|
255
|
+
) -> Source:
|
|
256
|
+
"""Another environment as the source of truth.
|
|
257
|
+
|
|
258
|
+
The rows come back exactly as the server sent them, ids included,
|
|
259
|
+
and the ids are then never used for anything. They are carried so
|
|
260
|
+
the report can say which source row a change came from, in the
|
|
261
|
+
source environment's own vocabulary.
|
|
262
|
+
"""
|
|
263
|
+
rows = resolve_mod.list_rows(
|
|
264
|
+
kind=kind, target=target, space=space, token=token, fetch=fetch,
|
|
265
|
+
subject=f"{kind} in {target.name} cannot be used as a source of truth",
|
|
266
|
+
remedy=_PARTIAL_REMEDY,
|
|
267
|
+
paginate=True,
|
|
268
|
+
)
|
|
269
|
+
blob = json.dumps(rows, sort_keys=True, ensure_ascii=False, default=str)
|
|
270
|
+
return Source(
|
|
271
|
+
rows=rows,
|
|
272
|
+
origin=f"{target.name}:{space}",
|
|
273
|
+
digest=hashlib.sha256(blob.encode()).hexdigest(),
|
|
274
|
+
fmt="env",
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
# ===========================================================================
|
|
279
|
+
# The comparison
|
|
280
|
+
# ===========================================================================
|
|
281
|
+
@dataclass(frozen=True)
|
|
282
|
+
class Change:
|
|
283
|
+
"""One op, already reduced to the exact ``$set``/``$expect`` it will carry.
|
|
284
|
+
|
|
285
|
+
The payloads are computed here rather than in
|
|
286
|
+
:meth:`DiffResult.ops`, so exactly one place knows that a
|
|
287
|
+
jsonb-backed kind writes a whole column while a column-backed kind
|
|
288
|
+
writes fields. Two representations of the same op is how a diff
|
|
289
|
+
and the payload it emits come to disagree.
|
|
290
|
+
"""
|
|
291
|
+
|
|
292
|
+
action: str # "add" | "update"
|
|
293
|
+
key: str
|
|
294
|
+
#: Column -> (target value, source value, why they differ). For display.
|
|
295
|
+
fields: dict[str, tuple[Any, Any, str]] = field(default_factory=dict)
|
|
296
|
+
#: Fields the target holds that the source did not mention. Carried forward,
|
|
297
|
+
#: never dropped, same rule as rows.
|
|
298
|
+
preserved: tuple[str, ...] = ()
|
|
299
|
+
shape_changes: tuple[str, ...] = ()
|
|
300
|
+
set_payload: dict = field(default_factory=dict)
|
|
301
|
+
expect_payload: dict = field(default_factory=dict)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
@dataclass(frozen=True)
|
|
305
|
+
class DiffResult:
|
|
306
|
+
kind: str
|
|
307
|
+
env: str
|
|
308
|
+
space: str
|
|
309
|
+
key_field: str
|
|
310
|
+
source: Source
|
|
311
|
+
creates: list[Change]
|
|
312
|
+
updates: list[Change]
|
|
313
|
+
unchanged: list[str]
|
|
314
|
+
would_delete: list[str]
|
|
315
|
+
#: Source keys whose text changed under normalisation. A silent trim is
|
|
316
|
+
#: how a phantom row is born, so every one is named.
|
|
317
|
+
normalised: list[tuple[str, str]]
|
|
318
|
+
blocked_shape: list[Change]
|
|
319
|
+
data_field: str | None
|
|
320
|
+
#: How many target rows were actually read. Not derived from the other
|
|
321
|
+
#: buckets, so this count can catch a bucket that silently dropped a row
|
|
322
|
+
#: instead of agreeing with it by construction.
|
|
323
|
+
target_count: int = 0
|
|
324
|
+
#: Target names that identify more than one row, which the source does
|
|
325
|
+
#: not mention. Reported with ids: the no-ids rule governs
|
|
326
|
+
#: :meth:`ops`, the payload that gets written; a human report of rows
|
|
327
|
+
#: that need de-duplicating is actionable only by id, and nothing here
|
|
328
|
+
#: is ever fed back to a write.
|
|
329
|
+
ambiguous: list[tuple[str, list[str]]] = field(default_factory=list)
|
|
330
|
+
|
|
331
|
+
def ops(self) -> list[dict]:
|
|
332
|
+
"""The payload ``fm update`` consumes.
|
|
333
|
+
|
|
334
|
+
Ops only: the endpoint forbids extra keys, so the deletion
|
|
335
|
+
list and the equality rules live in the ``--report`` sidecar
|
|
336
|
+
rather than in here.
|
|
337
|
+
|
|
338
|
+
``$expect`` carries the values this diff read. That is
|
|
339
|
+
deliberate even though auto-filling a before-value from a
|
|
340
|
+
read milliseconds earlier in the same command would convert a
|
|
341
|
+
staleness check into one that cannot fail; here the read and
|
|
342
|
+
the write are two commands with a human review between them,
|
|
343
|
+
which is the gap ``$expect`` exists to cover: a plan is valid
|
|
344
|
+
only against the state it was generated from. The key field is
|
|
345
|
+
left out so that ``fm update``'s own copy-back can fill it
|
|
346
|
+
from its own resolution.
|
|
347
|
+
"""
|
|
348
|
+
ops: list[dict] = []
|
|
349
|
+
for change in self.creates:
|
|
350
|
+
ops.append({"$action": "add", "$kind": self.kind, "$set": change.set_payload})
|
|
351
|
+
for change in self.updates:
|
|
352
|
+
op = {
|
|
353
|
+
"$action": "update",
|
|
354
|
+
"$kind": self.kind,
|
|
355
|
+
"$name": change.key,
|
|
356
|
+
"$set": change.set_payload,
|
|
357
|
+
}
|
|
358
|
+
if change.expect_payload:
|
|
359
|
+
op["$expect"] = change.expect_payload
|
|
360
|
+
ops.append(op)
|
|
361
|
+
return ops
|
|
362
|
+
|
|
363
|
+
def report(self) -> dict:
|
|
364
|
+
return {
|
|
365
|
+
"kind": self.kind,
|
|
366
|
+
"target": {"env": self.env, "space": self.space},
|
|
367
|
+
"source": {
|
|
368
|
+
"origin": self.source.origin,
|
|
369
|
+
"format": self.source.fmt,
|
|
370
|
+
"sha256": self.source.digest,
|
|
371
|
+
"rows": len(self.source.rows),
|
|
372
|
+
},
|
|
373
|
+
"equality_rules": {
|
|
374
|
+
"match_key": {"field": self.key_field, "rule": KEY_RULE},
|
|
375
|
+
"values": {"rule": VALUE_RULE},
|
|
376
|
+
"written_column": (
|
|
377
|
+
f"{self.data_field} (whole jsonb column; unmentioned fields "
|
|
378
|
+
"merged forward, never dropped)"
|
|
379
|
+
if self.data_field
|
|
380
|
+
else "per-column"
|
|
381
|
+
),
|
|
382
|
+
},
|
|
383
|
+
"counts": {
|
|
384
|
+
"create": len(self.creates),
|
|
385
|
+
"update": len(self.updates),
|
|
386
|
+
"unchanged": len(self.unchanged),
|
|
387
|
+
"would_delete": len(self.would_delete),
|
|
388
|
+
"blocked_on_shape": len(self.blocked_shape),
|
|
389
|
+
"ambiguous_in_target": len(self.ambiguous),
|
|
390
|
+
"target_rows_read": self.target_count,
|
|
391
|
+
},
|
|
392
|
+
"normalised_keys": [
|
|
393
|
+
{"as_written": raw, "matched_as": norm} for raw, norm in self.normalised
|
|
394
|
+
],
|
|
395
|
+
"creates": [c.key for c in self.creates],
|
|
396
|
+
"updates": [
|
|
397
|
+
{
|
|
398
|
+
"name": c.key,
|
|
399
|
+
"fields": {
|
|
400
|
+
col: {"target": tv, "source": sv, "why": why}
|
|
401
|
+
for col, (tv, sv, why) in c.fields.items()
|
|
402
|
+
},
|
|
403
|
+
"preserved": list(c.preserved),
|
|
404
|
+
}
|
|
405
|
+
for c in self.updates
|
|
406
|
+
],
|
|
407
|
+
"blocked_on_shape": [
|
|
408
|
+
{"name": c.key, "reasons": list(c.shape_changes)}
|
|
409
|
+
for c in self.blocked_shape
|
|
410
|
+
],
|
|
411
|
+
"ambiguous_in_target": [
|
|
412
|
+
{"name": key, "ids": ids} for key, ids in self.ambiguous
|
|
413
|
+
],
|
|
414
|
+
"would_delete": self.would_delete,
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _get(row: dict, column: str, data_field: str | None) -> Any:
|
|
419
|
+
if data_field is None:
|
|
420
|
+
return row.get(column)
|
|
421
|
+
blob = row.get(data_field)
|
|
422
|
+
return blob.get(column) if isinstance(blob, dict) else None
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _reconcile(
|
|
426
|
+
kind: str, read: int, unchanged: list, updates: list, blocked: list,
|
|
427
|
+
would_delete: list,
|
|
428
|
+
) -> None:
|
|
429
|
+
"""Every target row must land in exactly one bucket, or nothing is emitted.
|
|
430
|
+
|
|
431
|
+
A function rather than inline, because inline it could not be
|
|
432
|
+
tested on its own: the guards above it mean nothing can currently
|
|
433
|
+
reach this in normal operation, so it is a backstop that needs to
|
|
434
|
+
be tested at its own level to prove it still fires if one of those
|
|
435
|
+
guards is ever loosened.
|
|
436
|
+
"""
|
|
437
|
+
accounted = len(unchanged) + len(updates) + len(blocked) + len(would_delete)
|
|
438
|
+
if accounted != read:
|
|
439
|
+
raise DiffError(
|
|
440
|
+
f"internal: {read} {kind} rows were read but {accounted} were "
|
|
441
|
+
"accounted for (unchanged + update + blocked + would-delete).\n"
|
|
442
|
+
"A row that is read and then neither compared nor reported has gone "
|
|
443
|
+
"missing, and the only symptom would have been a total slightly too "
|
|
444
|
+
"small. Refusing to emit a payload computed from an incomplete pass."
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def build_diff(
|
|
449
|
+
*,
|
|
450
|
+
kind: str,
|
|
451
|
+
key_field: str,
|
|
452
|
+
fields: list[str] | None,
|
|
453
|
+
source: Source,
|
|
454
|
+
target_rows: list[dict],
|
|
455
|
+
env: str,
|
|
456
|
+
space: str,
|
|
457
|
+
data_field: str | None,
|
|
458
|
+
allow_shape_change: bool = False,
|
|
459
|
+
extra_set: dict | None = None,
|
|
460
|
+
) -> DiffResult:
|
|
461
|
+
"""Compare, or refuse. Never guess which of two rows was meant."""
|
|
462
|
+
# A name stored as an {en, ar} map indexes the same row under two keys,
|
|
463
|
+
# which is the resolver's own "match across shapes" rule applied here
|
|
464
|
+
# too. But it means "which target rows were matched" is a question
|
|
465
|
+
# about rows, not about keys: matching the English key leaves the
|
|
466
|
+
# Arabic one unmatched, and counting keys would report a live row as a
|
|
467
|
+
# would-delete. Rows are tracked by identity.
|
|
468
|
+
index: dict[str, list[dict]] = {}
|
|
469
|
+
row_keys: dict[int, list[str]] = {}
|
|
470
|
+
for row in target_rows:
|
|
471
|
+
# dict.fromkeys, because a row whose {en, ar} halves hold the same
|
|
472
|
+
# text yields that key twice and would be indexed against itself,
|
|
473
|
+
# which would report it as ambiguous with its own id printed twice.
|
|
474
|
+
# A row is never ambiguous with itself; only distinct rows can be.
|
|
475
|
+
keys = list(dict.fromkeys(
|
|
476
|
+
k for k in key_strings(_get(row, key_field, data_field)) if k
|
|
477
|
+
))
|
|
478
|
+
row_keys[id(row)] = keys
|
|
479
|
+
for key in keys:
|
|
480
|
+
index.setdefault(key, []).append(row)
|
|
481
|
+
|
|
482
|
+
ambiguous = {k: v for k, v in index.items() if len(v) > 1}
|
|
483
|
+
|
|
484
|
+
# The source is normalised and validated before anything is compared,
|
|
485
|
+
# both so a source defect is reported as a source defect, and because
|
|
486
|
+
# which ambiguities actually matter is a question about what the source
|
|
487
|
+
# references.
|
|
488
|
+
seen_source: dict[str, str] = {}
|
|
489
|
+
normalised: list[tuple[str, str]] = []
|
|
490
|
+
source_keys: list[str] = []
|
|
491
|
+
for position, row in enumerate(source.rows):
|
|
492
|
+
raw_key = row.get(key_field)
|
|
493
|
+
if not isinstance(raw_key, str) or not raw_key.strip():
|
|
494
|
+
raise DiffError(
|
|
495
|
+
f"source row {position} has no usable {key_field!r}: {raw_key!r}.\n"
|
|
496
|
+
"Every row must be identifiable by the match key; a row that is not "
|
|
497
|
+
"cannot be compared and must not be created blind."
|
|
498
|
+
)
|
|
499
|
+
key = normalise_key(raw_key)
|
|
500
|
+
if not key:
|
|
501
|
+
raise DiffError(f"source row {position}: {key_field!r} normalises to empty.")
|
|
502
|
+
if key != raw_key:
|
|
503
|
+
normalised.append((raw_key, key))
|
|
504
|
+
if key in seen_source:
|
|
505
|
+
raise DiffError(
|
|
506
|
+
f"refusing to diff: source rows {seen_source[key]} and {position} "
|
|
507
|
+
f"both normalise to {key!r}.\n"
|
|
508
|
+
"Two source rows for one target row is a source-of-truth defect, and "
|
|
509
|
+
"applying them in file order would silently let the last one win.\n"
|
|
510
|
+
f" as written: {raw_key!r}\n"
|
|
511
|
+
"Nothing was compared."
|
|
512
|
+
)
|
|
513
|
+
seen_source[key] = str(position)
|
|
514
|
+
source_keys.append(key)
|
|
515
|
+
|
|
516
|
+
# Only an ambiguity the source references can cause a wrong write, and
|
|
517
|
+
# that one is fatal. An ambiguity elsewhere in the target is a data
|
|
518
|
+
# defect this tool did not create and cannot resolve; refusing the
|
|
519
|
+
# whole run over it would make the command unusable against a real
|
|
520
|
+
# space that holds such pairs. They are reported, loudly, and their
|
|
521
|
+
# rows are never written to.
|
|
522
|
+
blocking = {k: v for k, v in ambiguous.items() if k in seen_source}
|
|
523
|
+
if blocking:
|
|
524
|
+
listing = "\n".join(
|
|
525
|
+
f" {key!r} -> " + ", ".join(str(r.get("id")) for r in rows)
|
|
526
|
+
for key, rows in sorted(blocking.items())
|
|
527
|
+
)
|
|
528
|
+
raise DiffError(
|
|
529
|
+
f"refusing to diff: {len(blocking)} name(s) in the SOURCE identify more "
|
|
530
|
+
f"than one {kind} row in {env}:\n{listing}\n"
|
|
531
|
+
"\n"
|
|
532
|
+
"A name that identifies two rows identifies neither, and picking one is "
|
|
533
|
+
"how a wrong row gets written. Nothing was compared.\n"
|
|
534
|
+
"De-duplicate the target, or address those rows by id in a payload "
|
|
535
|
+
"you write yourself."
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
matched_rows: set[int] = set()
|
|
539
|
+
creates: list[Change] = []
|
|
540
|
+
updates: list[Change] = []
|
|
541
|
+
blocked: list[Change] = []
|
|
542
|
+
unchanged: list[str] = []
|
|
543
|
+
matched: set[str] = set()
|
|
544
|
+
|
|
545
|
+
# strict=True: source_keys is built from source.rows one-for-one, and if
|
|
546
|
+
# that ever stops being true the pairs would silently shift and every
|
|
547
|
+
# row after the gap would be compared against the wrong key.
|
|
548
|
+
for row, key in zip(source.rows, source_keys, strict=True):
|
|
549
|
+
columns = fields or [c for c in row if c != key_field]
|
|
550
|
+
target = index.get(key, [None])[0]
|
|
551
|
+
|
|
552
|
+
if target is None:
|
|
553
|
+
payload = {c: row[c] for c in columns if c in row}
|
|
554
|
+
# The normalised key is what gets written, not the raw text.
|
|
555
|
+
# Writing back the raw, untrimmed text would create a row that
|
|
556
|
+
# this same diff would fail to match on the next run and would
|
|
557
|
+
# create again.
|
|
558
|
+
payload[key_field] = key
|
|
559
|
+
if data_field is None:
|
|
560
|
+
set_payload = dict(payload)
|
|
561
|
+
else:
|
|
562
|
+
set_payload = {data_field: dict(payload)}
|
|
563
|
+
set_payload.update(extra_set or {})
|
|
564
|
+
creates.append(
|
|
565
|
+
Change(
|
|
566
|
+
action="add",
|
|
567
|
+
key=key,
|
|
568
|
+
fields={c: (None, v, "row absent from target") for c, v in payload.items()},
|
|
569
|
+
set_payload=set_payload,
|
|
570
|
+
)
|
|
571
|
+
)
|
|
572
|
+
continue
|
|
573
|
+
|
|
574
|
+
if id(target) in matched_rows:
|
|
575
|
+
# The duplicate-source check above compares normalised source
|
|
576
|
+
# keys and is structurally unable to see this: two different
|
|
577
|
+
# strings can resolve to one row, because a name map indexes
|
|
578
|
+
# its row under every locale it holds. Two ops would be
|
|
579
|
+
# emitted for one row and the later would silently win, the
|
|
580
|
+
# same last-one-wins failure the source check exists to
|
|
581
|
+
# prevent, arriving by the one route that check cannot cover.
|
|
582
|
+
other = next(k for k in row_keys[id(target)] if k in matched and k != key)
|
|
583
|
+
raise DiffError(
|
|
584
|
+
f"refusing to diff: source keys {other!r} and {key!r} are different "
|
|
585
|
+
f"names for the SAME {kind} row in {env} ({target.get('id')}).\n"
|
|
586
|
+
"A name stored as an {en, ar} map indexes its row under both, so two "
|
|
587
|
+
"source rows can address one target row without ever colliding as "
|
|
588
|
+
"text. Applying both would let the later one silently win.\n"
|
|
589
|
+
"Nothing was compared."
|
|
590
|
+
)
|
|
591
|
+
matched.add(key)
|
|
592
|
+
matched_rows.add(id(target))
|
|
593
|
+
changed: dict[str, tuple[Any, Any, str]] = {}
|
|
594
|
+
shape_changes: list[str] = []
|
|
595
|
+
for column in columns:
|
|
596
|
+
if column not in row:
|
|
597
|
+
continue
|
|
598
|
+
target_value = _get(target, column, data_field)
|
|
599
|
+
source_value = row[column]
|
|
600
|
+
equal, why = values_equal(target_value, source_value)
|
|
601
|
+
if equal:
|
|
602
|
+
continue
|
|
603
|
+
if why.startswith("shape differs") and not allow_shape_change:
|
|
604
|
+
shape_changes.append(f"{column}: {why}")
|
|
605
|
+
continue
|
|
606
|
+
changed[column] = (target_value, source_value, why)
|
|
607
|
+
|
|
608
|
+
preserved = _preserved(target, columns, data_field)
|
|
609
|
+
if shape_changes:
|
|
610
|
+
blocked.append(
|
|
611
|
+
Change(action="update", key=key, fields=changed,
|
|
612
|
+
preserved=preserved, shape_changes=tuple(shape_changes))
|
|
613
|
+
)
|
|
614
|
+
elif changed:
|
|
615
|
+
if data_field is None:
|
|
616
|
+
set_payload = {c: v[1] for c, v in changed.items()}
|
|
617
|
+
expect_payload = {c: v[0] for c, v in changed.items()}
|
|
618
|
+
else:
|
|
619
|
+
# `$set` keys must be real columns, and a jsonb-backed kind
|
|
620
|
+
# has exactly one. So the write is the whole jsonb blob,
|
|
621
|
+
# merged over what the target already holds; a naive
|
|
622
|
+
# whole-column write would silently delete every field the
|
|
623
|
+
# source did not know about, which is the row rule applied
|
|
624
|
+
# one level down.
|
|
625
|
+
blob = target.get(data_field)
|
|
626
|
+
blob = dict(blob) if isinstance(blob, dict) else {}
|
|
627
|
+
merged = {**blob, **{c: v[1] for c, v in changed.items()}}
|
|
628
|
+
set_payload = {data_field: merged}
|
|
629
|
+
# The whole blob, so a concurrent change to any field in it
|
|
630
|
+
# invalidates this plan. That is the correct granularity:
|
|
631
|
+
# the write is the whole column, so the precondition must
|
|
632
|
+
# be too.
|
|
633
|
+
expect_payload = {data_field: target.get(data_field)}
|
|
634
|
+
updates.append(
|
|
635
|
+
Change(action="update", key=key, fields=changed, preserved=preserved,
|
|
636
|
+
set_payload=set_payload, expect_payload=expect_payload)
|
|
637
|
+
)
|
|
638
|
+
else:
|
|
639
|
+
unchanged.append(key)
|
|
640
|
+
|
|
641
|
+
# Every target row must land in exactly one bucket: a row that is read
|
|
642
|
+
# and then neither compared nor reported has gone missing silently, and
|
|
643
|
+
# the only visible symptom is a total that is slightly too small.
|
|
644
|
+
would_delete = sorted(
|
|
645
|
+
(row_keys[id(row)] or [str(row.get("id"))])[0]
|
|
646
|
+
for row in target_rows
|
|
647
|
+
if id(row) not in matched_rows
|
|
648
|
+
)
|
|
649
|
+
|
|
650
|
+
_reconcile(kind, len(target_rows), unchanged, updates, blocked, would_delete)
|
|
651
|
+
|
|
652
|
+
return DiffResult(
|
|
653
|
+
kind=kind, env=env, space=space, key_field=key_field, source=source,
|
|
654
|
+
creates=creates, updates=updates, unchanged=unchanged,
|
|
655
|
+
would_delete=would_delete, normalised=normalised, blocked_shape=blocked,
|
|
656
|
+
data_field=data_field,
|
|
657
|
+
target_count=len(target_rows),
|
|
658
|
+
ambiguous=sorted(
|
|
659
|
+
(key, [str(r.get("id")) for r in rows]) for key, rows in ambiguous.items()
|
|
660
|
+
),
|
|
661
|
+
)
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _preserved(target: dict, columns: list[str], data_field: str | None) -> tuple[str, ...]:
|
|
665
|
+
"""Target fields the source never mentioned.
|
|
666
|
+
|
|
667
|
+
They are carried, not dropped, the same rule as rows, and it
|
|
668
|
+
matters more for a jsonb column: ``$set`` keys must be real
|
|
669
|
+
columns, so an update to that kind writes the whole ``data`` blob,
|
|
670
|
+
and a naive write would silently delete every field the source did
|
|
671
|
+
not know about.
|
|
672
|
+
"""
|
|
673
|
+
if data_field is None:
|
|
674
|
+
return ()
|
|
675
|
+
blob = target.get(data_field)
|
|
676
|
+
if not isinstance(blob, dict):
|
|
677
|
+
return ()
|
|
678
|
+
return tuple(sorted(k for k in blob if k not in columns))
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def target_rows_for(
|
|
682
|
+
*, kind: str, target: Target, space: str, token: str | None,
|
|
683
|
+
entity_type: str | None, fetch=None,
|
|
684
|
+
) -> tuple[list[dict], str | None, dict]:
|
|
685
|
+
"""The target's rows, the jsonb column they live in, and any extra ``$set``.
|
|
686
|
+
|
|
687
|
+
A kind scoped by an entity type has no name column at all, so
|
|
688
|
+
``--entity-type`` is required for that kind: the diff must compare
|
|
689
|
+
within one type, and a create has to carry the FK.
|
|
690
|
+
"""
|
|
691
|
+
spec = resolve_mod.KINDS.get(kind)
|
|
692
|
+
if spec is None:
|
|
693
|
+
raise DiffError(
|
|
694
|
+
f"unknown kind {kind!r}. Valid kinds: {resolve_mod.kind_names()}."
|
|
695
|
+
)
|
|
696
|
+
if spec.no_name_reason:
|
|
697
|
+
raise DiffError(f"{kind} cannot be diffed by name: {spec.no_name_reason}")
|
|
698
|
+
|
|
699
|
+
rows = resolve_mod.list_rows(
|
|
700
|
+
kind=kind, target=target, space=space, token=token, fetch=fetch,
|
|
701
|
+
subject=f"{kind} in {target.name} cannot be diffed",
|
|
702
|
+
remedy=_PARTIAL_REMEDY,
|
|
703
|
+
paginate=True,
|
|
704
|
+
)
|
|
705
|
+
if spec.data_field is None:
|
|
706
|
+
if entity_type:
|
|
707
|
+
raise DiffError(
|
|
708
|
+
f"--entity-type is only meaningful for a jsonb-backed kind; {kind} "
|
|
709
|
+
"has real columns."
|
|
710
|
+
)
|
|
711
|
+
return rows, None, {}
|
|
712
|
+
|
|
713
|
+
if not entity_type:
|
|
714
|
+
raise DiffError(
|
|
715
|
+
f"--entity-type is required for {kind}.\n"
|
|
716
|
+
"A CustomEntity is scoped by entity_type_id and has no name column of "
|
|
717
|
+
"its own, so a diff without a type would compare Organisations against "
|
|
718
|
+
"Venues and propose creating every one of them.\n"
|
|
719
|
+
"Use `--entity-type \"Organization\"` (the EntityType's single_name)."
|
|
720
|
+
)
|
|
721
|
+
match = resolve_mod.resolve_one(
|
|
722
|
+
entity_type, kind="ENTITY_TYPE", target=target, space=space,
|
|
723
|
+
token=token, fetch=fetch,
|
|
724
|
+
)
|
|
725
|
+
scoped = [r for r in rows if str(r.get("entity_type_id")) == str(match.id)]
|
|
726
|
+
return scoped, spec.data_field, {"entity_type_id": match.id}
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
# ===========================================================================
|
|
730
|
+
# The CLI surface. Kept here so adding this command costs cli.py two lines.
|
|
731
|
+
# ===========================================================================
|
|
732
|
+
def add_parser(sub) -> None:
|
|
733
|
+
|
|
734
|
+
parser = sub.add_parser(
|
|
735
|
+
"diff",
|
|
736
|
+
help="compare a source of truth against an environment; emit fm update ops",
|
|
737
|
+
description=(
|
|
738
|
+
"`fm update` answers how to write. This answers what. It compares a "
|
|
739
|
+
"source of truth against a target environment and writes a payload "
|
|
740
|
+
"`fm update` consumes, so the two compose:\n"
|
|
741
|
+
" fm diff stakeholder --source rows.json --out ops.json\n"
|
|
742
|
+
" read it, then: fm update ... ops.json\n\n"
|
|
743
|
+
"Ops name rows by $name, never by id: ids differ between every "
|
|
744
|
+
"environment, and $name is re-resolved by `fm update` against the "
|
|
745
|
+
"target seconds before the write, with the resolved value copied into "
|
|
746
|
+
"$expect so the row itself refuses a mis-resolution.\n\n"
|
|
747
|
+
"THE EQUALITY RULES ARE PRINTED ON EVERY RUN, so that two readers of "
|
|
748
|
+
"the same report cannot silently disagree about what 'already equal' "
|
|
749
|
+
"means.\n\n"
|
|
750
|
+
f" match key: {KEY_RULE}\n"
|
|
751
|
+
f" values: {VALUE_RULE}\n\n"
|
|
752
|
+
"A row present in the target and absent from the source is NEVER a "
|
|
753
|
+
"delete. It is reported and nothing else. Sources: .json (array of row "
|
|
754
|
+
"objects, or {\"rows\": [...]}), .csv (header row), or --from-host to "
|
|
755
|
+
"use another environment as the source of truth."
|
|
756
|
+
),
|
|
757
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
758
|
+
)
|
|
759
|
+
parser.add_argument("--host", default=argparse.SUPPRESS, metavar="HOST",
|
|
760
|
+
help="deployment to compare AGAINST (default: formist)")
|
|
761
|
+
parser.add_argument("--space", help="space UUID (never a name)")
|
|
762
|
+
parser.add_argument(
|
|
763
|
+
"kind",
|
|
764
|
+
help=f"one of: {resolve_mod.kind_names()} (case-insensitive; - and _ alike)",
|
|
765
|
+
)
|
|
766
|
+
parser.add_argument("--entity-type", dest="entity_type",
|
|
767
|
+
help="EntityType single_name; required for the `custom` kind")
|
|
768
|
+
parser.add_argument("--source", help="path to a .json or .csv source of truth")
|
|
769
|
+
parser.add_argument("--from-host", dest="from_env", metavar="HOST",
|
|
770
|
+
help="use another deployment as the source of truth")
|
|
771
|
+
parser.add_argument("--key", default="name",
|
|
772
|
+
help="the column rows are matched by (default: name)")
|
|
773
|
+
parser.add_argument("--fields",
|
|
774
|
+
help="comma-separated columns to compare (default: every "
|
|
775
|
+
"column the source mentions)")
|
|
776
|
+
parser.add_argument("--allow-shape-change", dest="allow_shape", action="store_true",
|
|
777
|
+
help="emit ops that change a value's SHAPE (string vs map). "
|
|
778
|
+
"Off by default, since coalescing the two can hide a "
|
|
779
|
+
"real difference")
|
|
780
|
+
parser.add_argument("--out", help="write the ops payload here")
|
|
781
|
+
parser.add_argument("--report", help="write the full report (rules, deletions, "
|
|
782
|
+
"normalisations) here")
|
|
783
|
+
parser.add_argument("--json", action="store_true", help="print the report as JSON")
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
def cmd_diff(args) -> int:
|
|
787
|
+
import sys
|
|
788
|
+
|
|
789
|
+
# Imported inside the function because `cli` imports this module. Reusing
|
|
790
|
+
# its `_effective`/`_token_for` rather than copying these lines keeps one
|
|
791
|
+
# provenance banner in the tool.
|
|
792
|
+
from formist_cli.cli import _effective, _token_for
|
|
793
|
+
|
|
794
|
+
eff = _effective(args)
|
|
795
|
+
token = _token_for(eff)
|
|
796
|
+
kind = resolve_mod.require_kind(args.kind)
|
|
797
|
+
|
|
798
|
+
if bool(args.source) == bool(args.from_env):
|
|
799
|
+
raise DiffError(
|
|
800
|
+
"give exactly one of --source (a file) or --from-host (another "
|
|
801
|
+
"deployment).\n"
|
|
802
|
+
"Two sources of truth is not a source of truth."
|
|
803
|
+
)
|
|
804
|
+
|
|
805
|
+
# The local source is read first, before any network call: a mistyped
|
|
806
|
+
# --source is the most likely error in this command and the cheapest to
|
|
807
|
+
# detect, and fetching the target first would report it only after a
|
|
808
|
+
# round trip and, against prod, after an admin auth check. Cheap,
|
|
809
|
+
# local, certain checks go first.
|
|
810
|
+
source = load_source(args.source) if args.source else None
|
|
811
|
+
|
|
812
|
+
target_rows, data_field, extra_set = target_rows_for(
|
|
813
|
+
kind=kind, target=eff.target, space=eff.space, token=token,
|
|
814
|
+
entity_type=args.entity_type,
|
|
815
|
+
)
|
|
816
|
+
|
|
817
|
+
if source is None:
|
|
818
|
+
from formist_cli import prefs as prefs_mod
|
|
819
|
+
from formist_cli.targets import resolve as resolve_target
|
|
820
|
+
|
|
821
|
+
src_env = prefs_mod.canonical_env(args.from_env)
|
|
822
|
+
if src_env == eff.target.name:
|
|
823
|
+
raise DiffError(
|
|
824
|
+
f"--from-host {src_env!r} is the deployment being compared against. "
|
|
825
|
+
"A diff of an environment with itself is empty by construction."
|
|
826
|
+
)
|
|
827
|
+
src_target = resolve_target(src_env)
|
|
828
|
+
from formist_cli import tokens as tokens_mod
|
|
829
|
+
|
|
830
|
+
src_token, _ = tokens_mod.load_for(src_target.name)
|
|
831
|
+
print(f" source of truth: {src_target.name} ({src_target.api_base})",
|
|
832
|
+
file=sys.stderr)
|
|
833
|
+
source = source_from_env(
|
|
834
|
+
kind=kind, target=src_target, space=eff.space, token=src_token
|
|
835
|
+
)
|
|
836
|
+
|
|
837
|
+
result = build_diff(
|
|
838
|
+
kind=kind,
|
|
839
|
+
key_field=args.key,
|
|
840
|
+
fields=[f.strip() for f in args.fields.split(",")] if args.fields else None,
|
|
841
|
+
source=source,
|
|
842
|
+
target_rows=target_rows,
|
|
843
|
+
env=eff.target.name,
|
|
844
|
+
space=eff.space,
|
|
845
|
+
data_field=data_field,
|
|
846
|
+
allow_shape_change=args.allow_shape,
|
|
847
|
+
extra_set=extra_set,
|
|
848
|
+
)
|
|
849
|
+
|
|
850
|
+
report = result.report()
|
|
851
|
+
if args.json:
|
|
852
|
+
print(json.dumps(report, indent=2, ensure_ascii=False, default=str))
|
|
853
|
+
else:
|
|
854
|
+
_print_report(result, report)
|
|
855
|
+
|
|
856
|
+
if args.report:
|
|
857
|
+
with open(args.report, "w", encoding="utf-8") as handle:
|
|
858
|
+
json.dump(report, handle, indent=2, ensure_ascii=False, default=str)
|
|
859
|
+
print(f"\nreport -> {args.report}", file=sys.stderr)
|
|
860
|
+
|
|
861
|
+
ops = result.ops()
|
|
862
|
+
if args.out:
|
|
863
|
+
with open(args.out, "w", encoding="utf-8") as handle:
|
|
864
|
+
json.dump({"ops": ops}, handle, indent=2, ensure_ascii=False, default=str)
|
|
865
|
+
print(
|
|
866
|
+
f"ops -> {args.out} ({len(ops)} op(s))\n"
|
|
867
|
+
f"\nRead it, then:\n"
|
|
868
|
+
f" fm update --host {result.env} --space {result.space} {args.out}",
|
|
869
|
+
file=sys.stderr,
|
|
870
|
+
)
|
|
871
|
+
elif ops:
|
|
872
|
+
print(
|
|
873
|
+
f"\n{len(ops)} op(s) computed but not written. Pass --out FILE.",
|
|
874
|
+
file=sys.stderr,
|
|
875
|
+
)
|
|
876
|
+
return 0
|
|
877
|
+
|
|
878
|
+
|
|
879
|
+
def _print_report(result: DiffResult, report: dict) -> None:
|
|
880
|
+
counts = report["counts"]
|
|
881
|
+
print(f"{result.kind} in {result.env} / {result.space}")
|
|
882
|
+
print(f" source {result.source.origin} [{result.source.fmt}] "
|
|
883
|
+
f"sha256:{result.source.digest[:12]} {len(result.source.rows)} rows")
|
|
884
|
+
print(f" target {result.target_count} rows [reconciled: unchanged + update "
|
|
885
|
+
"+ blocked + would-delete]")
|
|
886
|
+
print()
|
|
887
|
+
print(f" match key {result.key_field}: {KEY_RULE}")
|
|
888
|
+
print(f" values {VALUE_RULE}")
|
|
889
|
+
print(f" writes {report['equality_rules']['written_column']}")
|
|
890
|
+
print()
|
|
891
|
+
print(f" create {counts['create']} update {counts['update']} "
|
|
892
|
+
f"unchanged {counts['unchanged']} "
|
|
893
|
+
f"blocked-on-shape {counts['blocked_on_shape']} "
|
|
894
|
+
f"would-delete {counts['would_delete']}")
|
|
895
|
+
|
|
896
|
+
if result.normalised:
|
|
897
|
+
print("\n keys that changed under normalisation "
|
|
898
|
+
"(an untrimmed name creates a phantom row):")
|
|
899
|
+
for raw, norm in result.normalised:
|
|
900
|
+
print(f" {raw!r} -> {norm!r}")
|
|
901
|
+
|
|
902
|
+
for change in result.creates:
|
|
903
|
+
print(f"\n + {change.key}")
|
|
904
|
+
for change in result.updates:
|
|
905
|
+
print(f"\n ~ {change.key}")
|
|
906
|
+
for column, (target_value, source_value, why) in change.fields.items():
|
|
907
|
+
print(f" {column}: {_show(target_value)} -> {_show(source_value)} [{why}]")
|
|
908
|
+
if change.preserved:
|
|
909
|
+
print(f" (preserved, not in source: {', '.join(change.preserved)})")
|
|
910
|
+
for change in result.blocked_shape:
|
|
911
|
+
print(f"\n ! {change.key} BLOCKED, shape change, no op emitted")
|
|
912
|
+
for reason in change.shape_changes:
|
|
913
|
+
print(f" {reason}")
|
|
914
|
+
print(" A bare string and an {en,...} map are different values; "
|
|
915
|
+
"coalescing them can hide a real difference.")
|
|
916
|
+
print(" Pass --allow-shape-change if that is genuinely what you mean.")
|
|
917
|
+
if result.ambiguous:
|
|
918
|
+
print(f"\n ambiguous in {result.env} ({len(result.ambiguous)}), NOT "
|
|
919
|
+
"referenced by the source, so nothing here was compared or written:")
|
|
920
|
+
for key, ids in result.ambiguous:
|
|
921
|
+
print(f" ? {key} -> {', '.join(ids)}")
|
|
922
|
+
print(" A name identifying two rows identifies neither. These are "
|
|
923
|
+
"pre-existing duplicates in the target, not something this diff "
|
|
924
|
+
"introduced, but a later source that does name one of them will be "
|
|
925
|
+
"refused outright.")
|
|
926
|
+
|
|
927
|
+
if result.would_delete:
|
|
928
|
+
print(f"\n would-delete ({len(result.would_delete)}), reported only, never "
|
|
929
|
+
"emitted as an op:")
|
|
930
|
+
for key in result.would_delete:
|
|
931
|
+
print(f" - {key}")
|
|
932
|
+
print(" A row missing from the source is not authority to delete it. The "
|
|
933
|
+
"source may simply be partial.")
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def _show(value: Any) -> str:
|
|
937
|
+
"""A value as JSON, never through str().
|
|
938
|
+
|
|
939
|
+
`str({"en": "FAO"})` is a dict repr: it matches nothing, and
|
|
940
|
+
printing one beside a real value invites comparing them by eye
|
|
941
|
+
and concluding they agree.
|
|
942
|
+
"""
|
|
943
|
+
return json.dumps(value, ensure_ascii=False, default=str)
|