formist-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
formist_cli/diff.py ADDED
@@ -0,0 +1,943 @@
1
+ """``fm diff``: what to write. ``fm update`` already answers how.
2
+
3
+ This compares a source of truth against a target environment and
4
+ emits a payload ``fm update`` consumes. The two compose: ``fm diff``,
5
+ read it, ``fm update``.
6
+
7
+ The output names rows by name, never by id. Ops carry ``$name``,
8
+ which ``fm update`` re-resolves against the target seconds before the
9
+ write and copies into ``$expect`` so the row itself refuses a
10
+ mis-resolution. Emitting ids here would move a transcribed-id failure
11
+ one file earlier and drop that copy-back. A target id never appears
12
+ in the output at all.
13
+
14
+ The comparison itself is where every defect in this kind of tool
15
+ tends to live, so its rules are deliberate and stated up front:
16
+
17
+ - Trailing and non-breaking whitespace in the match key. An untrimmed
18
+ name is a phantom new row, not a visible duplicate: it creates
19
+ silently. See :func:`normalise_key`.
20
+ - Curly vs straight quotes, en-dash vs hyphen, present in real names
21
+ on both sides.
22
+ - The same column holding two shapes: a value can be a bare string in
23
+ one environment and an ``{"en": ...}`` map in another, and the two
24
+ are not coalesced, since a shape change can hide a real difference.
25
+ Nothing is rendered through ``str()``: ``str({"en": "FAO"})`` is a
26
+ dict repr and matches nothing at all.
27
+ - Ids differ between environments. Matching is by name, and 0 or more
28
+ than 1 matches is a refusal rather than a choice.
29
+ - An id prefix is not a key: a short UUID prefix is not safe as a
30
+ primary key over a table where a prefix collision is possible.
31
+ - A row missing from the source is never authority to delete.
32
+ Additive and updating only; what would have been deleted is
33
+ reported.
34
+ - The equality rule used is printed, per field, on every run, so two
35
+ people reading the same report cannot silently disagree about what
36
+ "already equal" means.
37
+
38
+ Stdlib only: this tool must not add a runtime dependency, which is
39
+ also why xlsx is refused with a conversion hint rather than parsed by
40
+ hand.
41
+ """
42
+
43
+ from __future__ import annotations
44
+
45
+ import argparse
46
+ import csv
47
+ import hashlib
48
+ import json
49
+ import re
50
+ import unicodedata
51
+ from dataclasses import dataclass, field
52
+ from typing import Any
53
+
54
+ from formist_cli import resolve as resolve_mod
55
+ from formist_cli import textnorm
56
+ from formist_cli.targets import Target
57
+
58
+
59
+ class DiffError(Exception):
60
+ """A comparison that will not be made, or a payload that will not be written."""
61
+
62
+
63
+ #: The resolver's remedy for a truncated page is "narrow the search". A diff
64
+ #: has no such option: narrowing is what makes the unseen rows look absent,
65
+ #: and absent is what this tool reports as a would-delete. It is a whole
66
+ #: table or nothing, which is why these callers paginate, and why the only
67
+ #: honest remedy left once a paginated walk has still come up short is to
68
+ #: stop.
69
+ _PARTIAL_REMEDY = (
70
+ "A diff over a partial table is not a partial diff: every row the walk did not "
71
+ "reach would be reported as a would-delete, so there is no narrowing that helps "
72
+ "and nothing was compared.\n"
73
+ " This kind is larger than the walk's ceiling "
74
+ f"({resolve_mod.MAX_PAGES} x {resolve_mod.MAX_PAGE} rows), or the server stopped "
75
+ "returning rows before it reached the total it reported."
76
+ )
77
+
78
+
79
+ # ===========================================================================
80
+ # The two rules. They are different on purpose, and both are printed.
81
+ # ===========================================================================
82
+ # The three character tables (which characters are a space / a quote / a
83
+ # dash) live in textnorm.py, byte-identical, since `fm ai-benchmark` needs
84
+ # the same fact under a different rule. What stays here is the rule, and it
85
+ # stays case-sensitive. The tables are used through `textnorm.` below rather
86
+ # than re-bound to local names, so there is one definition.
87
+
88
+ KEY_RULE = (
89
+ "NFKC -> Unicode spaces (incl. U+00A0 and the zero-width U+200B / U+FEFF, which "
90
+ "no trim reaches) -> U+0020 -> collapse runs -> trim -> curly quotes -> straight, "
91
+ "en/em/non-breaking dashes -> '-'; CASE-SENSITIVE"
92
+ )
93
+ VALUE_RULE = (
94
+ "exact and SHAPE-SENSITIVE: a bare string never equals an {en,...} map, and no "
95
+ "value is rendered through str()"
96
+ )
97
+
98
+
99
+ def normalise_key(value: str) -> str:
100
+ """The match key. Lenient, because its job is to find the row.
101
+
102
+ The ``textnorm.SPACES`` pass runs before the trim, and that
103
+ ordering is the only part of this function that is load-bearing
104
+ rather than belt-and-braces: ``.strip()`` already removes U+00A0
105
+ (it is ``isspace()``), and NFKC maps it to U+0020 besides. What no
106
+ generic step reaches is U+200B and U+FEFF: zero-width,
107
+ ``isspace()``-false, unmatched by ``\\s``, unchanged by NFKC. They
108
+ are also invisible on screen, so the phantom row they create is
109
+ named identically to the row it duplicates.
110
+
111
+ Case is not folded: folding case is how a resolver can pick the
112
+ wrong row, and this must agree with the resolver that
113
+ ``fm update`` will use seconds later.
114
+ """
115
+ text = unicodedata.normalize("NFKC", value)
116
+ for space in textnorm.SPACES:
117
+ text = text.replace(space, " ")
118
+ for src, dst in {**textnorm.QUOTES, **textnorm.DASHES}.items():
119
+ text = text.replace(src, dst)
120
+ return re.sub(r"\s+", " ", text).strip()
121
+
122
+
123
+ def values_equal(left: Any, right: Any) -> tuple[bool, str]:
124
+ """Are two column values the same value? ``(equal, why)``.
125
+
126
+ Stricter than the key rule, and deliberately not the same
127
+ function. A key is matched so the row can be found; a value
128
+ decides whether to write. Shape is part of a value: ``"FAO"`` and
129
+ ``{"en": "FAO"}`` are different, and the endpoint's own
130
+ ``$expect`` check refuses to coalesce them.
131
+ """
132
+ if type(left) is not type(right):
133
+ if _shape_of(left) != _shape_of(right):
134
+ # NULL is absence, not a shape. Filling an empty column is the
135
+ # most ordinary update there is, and blocking it would block
136
+ # most of the real work. The reverse is NOT symmetric and stays
137
+ # blocked: a source null over a populated column is a
138
+ # field-level delete, and "a row missing from the source is
139
+ # never authority to delete" is this module's rule one level
140
+ # down. Clearing a value takes --allow-shape-change,
141
+ # deliberately.
142
+ if left is None:
143
+ return False, "target holds no value"
144
+ return False, f"shape differs: {_shape_of(left)} vs {_shape_of(right)}"
145
+ if isinstance(left, str) and isinstance(right, str):
146
+ if left == right:
147
+ return True, "identical"
148
+ if normalise_key(left) == normalise_key(right):
149
+ return False, "differs only by whitespace/quote/dash normalisation"
150
+ return False, "differs"
151
+ return (True, "identical") if left == right else (False, "differs")
152
+
153
+
154
+ def _shape_of(value: Any) -> str:
155
+ if value is None:
156
+ return "null"
157
+ if isinstance(value, str):
158
+ return "string"
159
+ if isinstance(value, dict):
160
+ return "map"
161
+ if isinstance(value, list):
162
+ return "list"
163
+ return type(value).__name__
164
+
165
+
166
+ def key_strings(value: Any) -> list[str]:
167
+ """Every normalised string a column value could be matched by.
168
+
169
+ Mirrors the resolver's own string extraction, so that a row this
170
+ module matches is the row ``fm update`` will resolve. Two
171
+ different notions of "the name of this row" in one pipeline is
172
+ how a diff proposes an op that then refuses.
173
+ """
174
+ if isinstance(value, str):
175
+ return [normalise_key(value)]
176
+ if isinstance(value, dict):
177
+ return [normalise_key(v) for v in value.values() if isinstance(v, str)]
178
+ return []
179
+
180
+
181
+ # ===========================================================================
182
+ # Sources of truth
183
+ # ===========================================================================
184
+ @dataclass(frozen=True)
185
+ class Source:
186
+ rows: list[dict]
187
+ origin: str
188
+ #: sha256 of the bytes compared, so a report can be tied to an input.
189
+ digest: str
190
+ fmt: str
191
+
192
+
193
+ def load_source(path: str) -> Source:
194
+ """A JSON array / ``{"rows": [...]}`` or a CSV with a header row.
195
+
196
+ xlsx is refused rather than parsed. This tool is stdlib-only by
197
+ contract, and a hand-rolled xlsx reader (shared strings, date
198
+ serials, inline vs shared cells) is precisely the class of thing
199
+ that reads almost right and is believed. The refusal names the
200
+ conversion instead.
201
+ """
202
+ lower = path.lower()
203
+ if lower.endswith((".xlsx", ".xls", ".xlsm")):
204
+ raise DiffError(
205
+ f"{path} is a spreadsheet, and fm refuses to parse one.\n"
206
+ "`fm` is stdlib-only so an xlsx reader here would be "
207
+ "hand-rolled: shared strings, date serials and inline-vs-shared cells "
208
+ "are where such a reader is subtly wrong, and a subtly wrong source of "
209
+ "truth is worse than none.\n"
210
+ "\n"
211
+ "Convert it first, then pass the result:\n"
212
+ " python -c \"import openpyxl,json,sys; "
213
+ "wb=openpyxl.load_workbook(sys.argv[1]); ws=wb.active; "
214
+ "rows=list(ws.values); hdr=[str(h).strip() for h in rows[0]]; "
215
+ "json.dump([dict(zip(hdr,r)) for r in rows[1:]], open(sys.argv[2],'w'), "
216
+ 'default=str)" book.xlsx rows.json'
217
+ )
218
+
219
+ try:
220
+ raw = open(path, "rb").read()
221
+ except FileNotFoundError:
222
+ raise DiffError(f"no such source file: {path}") from None
223
+ except OSError as exc:
224
+ raise DiffError(f"could not read {path}: {exc}") from None
225
+
226
+ digest = hashlib.sha256(raw).hexdigest()
227
+ text = raw.decode("utf-8-sig")
228
+
229
+ if lower.endswith(".csv"):
230
+ reader = csv.DictReader(text.splitlines())
231
+ rows = [dict(r) for r in reader]
232
+ fmt = "csv"
233
+ else:
234
+ try:
235
+ payload = json.loads(text)
236
+ except json.JSONDecodeError as exc:
237
+ raise DiffError(f"{path} is not valid JSON: {exc}") from None
238
+ if isinstance(payload, dict):
239
+ payload = payload.get("rows")
240
+ if not isinstance(payload, list):
241
+ raise DiffError(
242
+ f"{path} must hold a JSON array of row objects, or "
243
+ '{"rows": [...]}, got a ' + type(payload).__name__ + "."
244
+ )
245
+ rows = [r for r in payload if isinstance(r, dict)]
246
+ fmt = "json"
247
+
248
+ if not rows:
249
+ raise DiffError(f"{path} has no rows; nothing to compare.")
250
+ return Source(rows=rows, origin=path, digest=digest, fmt=fmt)
251
+
252
+
253
+ def source_from_env(
254
+ *, kind: str, target: Target, space: str, token: str | None, fetch=None
255
+ ) -> Source:
256
+ """Another environment as the source of truth.
257
+
258
+ The rows come back exactly as the server sent them, ids included,
259
+ and the ids are then never used for anything. They are carried so
260
+ the report can say which source row a change came from, in the
261
+ source environment's own vocabulary.
262
+ """
263
+ rows = resolve_mod.list_rows(
264
+ kind=kind, target=target, space=space, token=token, fetch=fetch,
265
+ subject=f"{kind} in {target.name} cannot be used as a source of truth",
266
+ remedy=_PARTIAL_REMEDY,
267
+ paginate=True,
268
+ )
269
+ blob = json.dumps(rows, sort_keys=True, ensure_ascii=False, default=str)
270
+ return Source(
271
+ rows=rows,
272
+ origin=f"{target.name}:{space}",
273
+ digest=hashlib.sha256(blob.encode()).hexdigest(),
274
+ fmt="env",
275
+ )
276
+
277
+
278
+ # ===========================================================================
279
+ # The comparison
280
+ # ===========================================================================
281
+ @dataclass(frozen=True)
282
+ class Change:
283
+ """One op, already reduced to the exact ``$set``/``$expect`` it will carry.
284
+
285
+ The payloads are computed here rather than in
286
+ :meth:`DiffResult.ops`, so exactly one place knows that a
287
+ jsonb-backed kind writes a whole column while a column-backed kind
288
+ writes fields. Two representations of the same op is how a diff
289
+ and the payload it emits come to disagree.
290
+ """
291
+
292
+ action: str # "add" | "update"
293
+ key: str
294
+ #: Column -> (target value, source value, why they differ). For display.
295
+ fields: dict[str, tuple[Any, Any, str]] = field(default_factory=dict)
296
+ #: Fields the target holds that the source did not mention. Carried forward,
297
+ #: never dropped, same rule as rows.
298
+ preserved: tuple[str, ...] = ()
299
+ shape_changes: tuple[str, ...] = ()
300
+ set_payload: dict = field(default_factory=dict)
301
+ expect_payload: dict = field(default_factory=dict)
302
+
303
+
304
+ @dataclass(frozen=True)
305
+ class DiffResult:
306
+ kind: str
307
+ env: str
308
+ space: str
309
+ key_field: str
310
+ source: Source
311
+ creates: list[Change]
312
+ updates: list[Change]
313
+ unchanged: list[str]
314
+ would_delete: list[str]
315
+ #: Source keys whose text changed under normalisation. A silent trim is
316
+ #: how a phantom row is born, so every one is named.
317
+ normalised: list[tuple[str, str]]
318
+ blocked_shape: list[Change]
319
+ data_field: str | None
320
+ #: How many target rows were actually read. Not derived from the other
321
+ #: buckets, so this count can catch a bucket that silently dropped a row
322
+ #: instead of agreeing with it by construction.
323
+ target_count: int = 0
324
+ #: Target names that identify more than one row, which the source does
325
+ #: not mention. Reported with ids: the no-ids rule governs
326
+ #: :meth:`ops`, the payload that gets written; a human report of rows
327
+ #: that need de-duplicating is actionable only by id, and nothing here
328
+ #: is ever fed back to a write.
329
+ ambiguous: list[tuple[str, list[str]]] = field(default_factory=list)
330
+
331
+ def ops(self) -> list[dict]:
332
+ """The payload ``fm update`` consumes.
333
+
334
+ Ops only: the endpoint forbids extra keys, so the deletion
335
+ list and the equality rules live in the ``--report`` sidecar
336
+ rather than in here.
337
+
338
+ ``$expect`` carries the values this diff read. That is
339
+ deliberate even though auto-filling a before-value from a
340
+ read milliseconds earlier in the same command would convert a
341
+ staleness check into one that cannot fail; here the read and
342
+ the write are two commands with a human review between them,
343
+ which is the gap ``$expect`` exists to cover: a plan is valid
344
+ only against the state it was generated from. The key field is
345
+ left out so that ``fm update``'s own copy-back can fill it
346
+ from its own resolution.
347
+ """
348
+ ops: list[dict] = []
349
+ for change in self.creates:
350
+ ops.append({"$action": "add", "$kind": self.kind, "$set": change.set_payload})
351
+ for change in self.updates:
352
+ op = {
353
+ "$action": "update",
354
+ "$kind": self.kind,
355
+ "$name": change.key,
356
+ "$set": change.set_payload,
357
+ }
358
+ if change.expect_payload:
359
+ op["$expect"] = change.expect_payload
360
+ ops.append(op)
361
+ return ops
362
+
363
+ def report(self) -> dict:
364
+ return {
365
+ "kind": self.kind,
366
+ "target": {"env": self.env, "space": self.space},
367
+ "source": {
368
+ "origin": self.source.origin,
369
+ "format": self.source.fmt,
370
+ "sha256": self.source.digest,
371
+ "rows": len(self.source.rows),
372
+ },
373
+ "equality_rules": {
374
+ "match_key": {"field": self.key_field, "rule": KEY_RULE},
375
+ "values": {"rule": VALUE_RULE},
376
+ "written_column": (
377
+ f"{self.data_field} (whole jsonb column; unmentioned fields "
378
+ "merged forward, never dropped)"
379
+ if self.data_field
380
+ else "per-column"
381
+ ),
382
+ },
383
+ "counts": {
384
+ "create": len(self.creates),
385
+ "update": len(self.updates),
386
+ "unchanged": len(self.unchanged),
387
+ "would_delete": len(self.would_delete),
388
+ "blocked_on_shape": len(self.blocked_shape),
389
+ "ambiguous_in_target": len(self.ambiguous),
390
+ "target_rows_read": self.target_count,
391
+ },
392
+ "normalised_keys": [
393
+ {"as_written": raw, "matched_as": norm} for raw, norm in self.normalised
394
+ ],
395
+ "creates": [c.key for c in self.creates],
396
+ "updates": [
397
+ {
398
+ "name": c.key,
399
+ "fields": {
400
+ col: {"target": tv, "source": sv, "why": why}
401
+ for col, (tv, sv, why) in c.fields.items()
402
+ },
403
+ "preserved": list(c.preserved),
404
+ }
405
+ for c in self.updates
406
+ ],
407
+ "blocked_on_shape": [
408
+ {"name": c.key, "reasons": list(c.shape_changes)}
409
+ for c in self.blocked_shape
410
+ ],
411
+ "ambiguous_in_target": [
412
+ {"name": key, "ids": ids} for key, ids in self.ambiguous
413
+ ],
414
+ "would_delete": self.would_delete,
415
+ }
416
+
417
+
418
+ def _get(row: dict, column: str, data_field: str | None) -> Any:
419
+ if data_field is None:
420
+ return row.get(column)
421
+ blob = row.get(data_field)
422
+ return blob.get(column) if isinstance(blob, dict) else None
423
+
424
+
425
+ def _reconcile(
426
+ kind: str, read: int, unchanged: list, updates: list, blocked: list,
427
+ would_delete: list,
428
+ ) -> None:
429
+ """Every target row must land in exactly one bucket, or nothing is emitted.
430
+
431
+ A function rather than inline, because inline it could not be
432
+ tested on its own: the guards above it mean nothing can currently
433
+ reach this in normal operation, so it is a backstop that needs to
434
+ be tested at its own level to prove it still fires if one of those
435
+ guards is ever loosened.
436
+ """
437
+ accounted = len(unchanged) + len(updates) + len(blocked) + len(would_delete)
438
+ if accounted != read:
439
+ raise DiffError(
440
+ f"internal: {read} {kind} rows were read but {accounted} were "
441
+ "accounted for (unchanged + update + blocked + would-delete).\n"
442
+ "A row that is read and then neither compared nor reported has gone "
443
+ "missing, and the only symptom would have been a total slightly too "
444
+ "small. Refusing to emit a payload computed from an incomplete pass."
445
+ )
446
+
447
+
448
+ def build_diff(
449
+ *,
450
+ kind: str,
451
+ key_field: str,
452
+ fields: list[str] | None,
453
+ source: Source,
454
+ target_rows: list[dict],
455
+ env: str,
456
+ space: str,
457
+ data_field: str | None,
458
+ allow_shape_change: bool = False,
459
+ extra_set: dict | None = None,
460
+ ) -> DiffResult:
461
+ """Compare, or refuse. Never guess which of two rows was meant."""
462
+ # A name stored as an {en, ar} map indexes the same row under two keys,
463
+ # which is the resolver's own "match across shapes" rule applied here
464
+ # too. But it means "which target rows were matched" is a question
465
+ # about rows, not about keys: matching the English key leaves the
466
+ # Arabic one unmatched, and counting keys would report a live row as a
467
+ # would-delete. Rows are tracked by identity.
468
+ index: dict[str, list[dict]] = {}
469
+ row_keys: dict[int, list[str]] = {}
470
+ for row in target_rows:
471
+ # dict.fromkeys, because a row whose {en, ar} halves hold the same
472
+ # text yields that key twice and would be indexed against itself,
473
+ # which would report it as ambiguous with its own id printed twice.
474
+ # A row is never ambiguous with itself; only distinct rows can be.
475
+ keys = list(dict.fromkeys(
476
+ k for k in key_strings(_get(row, key_field, data_field)) if k
477
+ ))
478
+ row_keys[id(row)] = keys
479
+ for key in keys:
480
+ index.setdefault(key, []).append(row)
481
+
482
+ ambiguous = {k: v for k, v in index.items() if len(v) > 1}
483
+
484
+ # The source is normalised and validated before anything is compared,
485
+ # both so a source defect is reported as a source defect, and because
486
+ # which ambiguities actually matter is a question about what the source
487
+ # references.
488
+ seen_source: dict[str, str] = {}
489
+ normalised: list[tuple[str, str]] = []
490
+ source_keys: list[str] = []
491
+ for position, row in enumerate(source.rows):
492
+ raw_key = row.get(key_field)
493
+ if not isinstance(raw_key, str) or not raw_key.strip():
494
+ raise DiffError(
495
+ f"source row {position} has no usable {key_field!r}: {raw_key!r}.\n"
496
+ "Every row must be identifiable by the match key; a row that is not "
497
+ "cannot be compared and must not be created blind."
498
+ )
499
+ key = normalise_key(raw_key)
500
+ if not key:
501
+ raise DiffError(f"source row {position}: {key_field!r} normalises to empty.")
502
+ if key != raw_key:
503
+ normalised.append((raw_key, key))
504
+ if key in seen_source:
505
+ raise DiffError(
506
+ f"refusing to diff: source rows {seen_source[key]} and {position} "
507
+ f"both normalise to {key!r}.\n"
508
+ "Two source rows for one target row is a source-of-truth defect, and "
509
+ "applying them in file order would silently let the last one win.\n"
510
+ f" as written: {raw_key!r}\n"
511
+ "Nothing was compared."
512
+ )
513
+ seen_source[key] = str(position)
514
+ source_keys.append(key)
515
+
516
+ # Only an ambiguity the source references can cause a wrong write, and
517
+ # that one is fatal. An ambiguity elsewhere in the target is a data
518
+ # defect this tool did not create and cannot resolve; refusing the
519
+ # whole run over it would make the command unusable against a real
520
+ # space that holds such pairs. They are reported, loudly, and their
521
+ # rows are never written to.
522
+ blocking = {k: v for k, v in ambiguous.items() if k in seen_source}
523
+ if blocking:
524
+ listing = "\n".join(
525
+ f" {key!r} -> " + ", ".join(str(r.get("id")) for r in rows)
526
+ for key, rows in sorted(blocking.items())
527
+ )
528
+ raise DiffError(
529
+ f"refusing to diff: {len(blocking)} name(s) in the SOURCE identify more "
530
+ f"than one {kind} row in {env}:\n{listing}\n"
531
+ "\n"
532
+ "A name that identifies two rows identifies neither, and picking one is "
533
+ "how a wrong row gets written. Nothing was compared.\n"
534
+ "De-duplicate the target, or address those rows by id in a payload "
535
+ "you write yourself."
536
+ )
537
+
538
+ matched_rows: set[int] = set()
539
+ creates: list[Change] = []
540
+ updates: list[Change] = []
541
+ blocked: list[Change] = []
542
+ unchanged: list[str] = []
543
+ matched: set[str] = set()
544
+
545
+ # strict=True: source_keys is built from source.rows one-for-one, and if
546
+ # that ever stops being true the pairs would silently shift and every
547
+ # row after the gap would be compared against the wrong key.
548
+ for row, key in zip(source.rows, source_keys, strict=True):
549
+ columns = fields or [c for c in row if c != key_field]
550
+ target = index.get(key, [None])[0]
551
+
552
+ if target is None:
553
+ payload = {c: row[c] for c in columns if c in row}
554
+ # The normalised key is what gets written, not the raw text.
555
+ # Writing back the raw, untrimmed text would create a row that
556
+ # this same diff would fail to match on the next run and would
557
+ # create again.
558
+ payload[key_field] = key
559
+ if data_field is None:
560
+ set_payload = dict(payload)
561
+ else:
562
+ set_payload = {data_field: dict(payload)}
563
+ set_payload.update(extra_set or {})
564
+ creates.append(
565
+ Change(
566
+ action="add",
567
+ key=key,
568
+ fields={c: (None, v, "row absent from target") for c, v in payload.items()},
569
+ set_payload=set_payload,
570
+ )
571
+ )
572
+ continue
573
+
574
+ if id(target) in matched_rows:
575
+ # The duplicate-source check above compares normalised source
576
+ # keys and is structurally unable to see this: two different
577
+ # strings can resolve to one row, because a name map indexes
578
+ # its row under every locale it holds. Two ops would be
579
+ # emitted for one row and the later would silently win, the
580
+ # same last-one-wins failure the source check exists to
581
+ # prevent, arriving by the one route that check cannot cover.
582
+ other = next(k for k in row_keys[id(target)] if k in matched and k != key)
583
+ raise DiffError(
584
+ f"refusing to diff: source keys {other!r} and {key!r} are different "
585
+ f"names for the SAME {kind} row in {env} ({target.get('id')}).\n"
586
+ "A name stored as an {en, ar} map indexes its row under both, so two "
587
+ "source rows can address one target row without ever colliding as "
588
+ "text. Applying both would let the later one silently win.\n"
589
+ "Nothing was compared."
590
+ )
591
+ matched.add(key)
592
+ matched_rows.add(id(target))
593
+ changed: dict[str, tuple[Any, Any, str]] = {}
594
+ shape_changes: list[str] = []
595
+ for column in columns:
596
+ if column not in row:
597
+ continue
598
+ target_value = _get(target, column, data_field)
599
+ source_value = row[column]
600
+ equal, why = values_equal(target_value, source_value)
601
+ if equal:
602
+ continue
603
+ if why.startswith("shape differs") and not allow_shape_change:
604
+ shape_changes.append(f"{column}: {why}")
605
+ continue
606
+ changed[column] = (target_value, source_value, why)
607
+
608
+ preserved = _preserved(target, columns, data_field)
609
+ if shape_changes:
610
+ blocked.append(
611
+ Change(action="update", key=key, fields=changed,
612
+ preserved=preserved, shape_changes=tuple(shape_changes))
613
+ )
614
+ elif changed:
615
+ if data_field is None:
616
+ set_payload = {c: v[1] for c, v in changed.items()}
617
+ expect_payload = {c: v[0] for c, v in changed.items()}
618
+ else:
619
+ # `$set` keys must be real columns, and a jsonb-backed kind
620
+ # has exactly one. So the write is the whole jsonb blob,
621
+ # merged over what the target already holds; a naive
622
+ # whole-column write would silently delete every field the
623
+ # source did not know about, which is the row rule applied
624
+ # one level down.
625
+ blob = target.get(data_field)
626
+ blob = dict(blob) if isinstance(blob, dict) else {}
627
+ merged = {**blob, **{c: v[1] for c, v in changed.items()}}
628
+ set_payload = {data_field: merged}
629
+ # The whole blob, so a concurrent change to any field in it
630
+ # invalidates this plan. That is the correct granularity:
631
+ # the write is the whole column, so the precondition must
632
+ # be too.
633
+ expect_payload = {data_field: target.get(data_field)}
634
+ updates.append(
635
+ Change(action="update", key=key, fields=changed, preserved=preserved,
636
+ set_payload=set_payload, expect_payload=expect_payload)
637
+ )
638
+ else:
639
+ unchanged.append(key)
640
+
641
+ # Every target row must land in exactly one bucket: a row that is read
642
+ # and then neither compared nor reported has gone missing silently, and
643
+ # the only visible symptom is a total that is slightly too small.
644
+ would_delete = sorted(
645
+ (row_keys[id(row)] or [str(row.get("id"))])[0]
646
+ for row in target_rows
647
+ if id(row) not in matched_rows
648
+ )
649
+
650
+ _reconcile(kind, len(target_rows), unchanged, updates, blocked, would_delete)
651
+
652
+ return DiffResult(
653
+ kind=kind, env=env, space=space, key_field=key_field, source=source,
654
+ creates=creates, updates=updates, unchanged=unchanged,
655
+ would_delete=would_delete, normalised=normalised, blocked_shape=blocked,
656
+ data_field=data_field,
657
+ target_count=len(target_rows),
658
+ ambiguous=sorted(
659
+ (key, [str(r.get("id")) for r in rows]) for key, rows in ambiguous.items()
660
+ ),
661
+ )
662
+
663
+
664
+ def _preserved(target: dict, columns: list[str], data_field: str | None) -> tuple[str, ...]:
665
+ """Target fields the source never mentioned.
666
+
667
+ They are carried, not dropped, the same rule as rows, and it
668
+ matters more for a jsonb column: ``$set`` keys must be real
669
+ columns, so an update to that kind writes the whole ``data`` blob,
670
+ and a naive write would silently delete every field the source did
671
+ not know about.
672
+ """
673
+ if data_field is None:
674
+ return ()
675
+ blob = target.get(data_field)
676
+ if not isinstance(blob, dict):
677
+ return ()
678
+ return tuple(sorted(k for k in blob if k not in columns))
679
+
680
+
681
+ def target_rows_for(
682
+ *, kind: str, target: Target, space: str, token: str | None,
683
+ entity_type: str | None, fetch=None,
684
+ ) -> tuple[list[dict], str | None, dict]:
685
+ """The target's rows, the jsonb column they live in, and any extra ``$set``.
686
+
687
+ A kind scoped by an entity type has no name column at all, so
688
+ ``--entity-type`` is required for that kind: the diff must compare
689
+ within one type, and a create has to carry the FK.
690
+ """
691
+ spec = resolve_mod.KINDS.get(kind)
692
+ if spec is None:
693
+ raise DiffError(
694
+ f"unknown kind {kind!r}. Valid kinds: {resolve_mod.kind_names()}."
695
+ )
696
+ if spec.no_name_reason:
697
+ raise DiffError(f"{kind} cannot be diffed by name: {spec.no_name_reason}")
698
+
699
+ rows = resolve_mod.list_rows(
700
+ kind=kind, target=target, space=space, token=token, fetch=fetch,
701
+ subject=f"{kind} in {target.name} cannot be diffed",
702
+ remedy=_PARTIAL_REMEDY,
703
+ paginate=True,
704
+ )
705
+ if spec.data_field is None:
706
+ if entity_type:
707
+ raise DiffError(
708
+ f"--entity-type is only meaningful for a jsonb-backed kind; {kind} "
709
+ "has real columns."
710
+ )
711
+ return rows, None, {}
712
+
713
+ if not entity_type:
714
+ raise DiffError(
715
+ f"--entity-type is required for {kind}.\n"
716
+ "A CustomEntity is scoped by entity_type_id and has no name column of "
717
+ "its own, so a diff without a type would compare Organisations against "
718
+ "Venues and propose creating every one of them.\n"
719
+ "Use `--entity-type \"Organization\"` (the EntityType's single_name)."
720
+ )
721
+ match = resolve_mod.resolve_one(
722
+ entity_type, kind="ENTITY_TYPE", target=target, space=space,
723
+ token=token, fetch=fetch,
724
+ )
725
+ scoped = [r for r in rows if str(r.get("entity_type_id")) == str(match.id)]
726
+ return scoped, spec.data_field, {"entity_type_id": match.id}
727
+
728
+
729
+ # ===========================================================================
730
+ # The CLI surface. Kept here so adding this command costs cli.py two lines.
731
+ # ===========================================================================
732
+ def add_parser(sub) -> None:
733
+
734
+ parser = sub.add_parser(
735
+ "diff",
736
+ help="compare a source of truth against an environment; emit fm update ops",
737
+ description=(
738
+ "`fm update` answers how to write. This answers what. It compares a "
739
+ "source of truth against a target environment and writes a payload "
740
+ "`fm update` consumes, so the two compose:\n"
741
+ " fm diff stakeholder --source rows.json --out ops.json\n"
742
+ " read it, then: fm update ... ops.json\n\n"
743
+ "Ops name rows by $name, never by id: ids differ between every "
744
+ "environment, and $name is re-resolved by `fm update` against the "
745
+ "target seconds before the write, with the resolved value copied into "
746
+ "$expect so the row itself refuses a mis-resolution.\n\n"
747
+ "THE EQUALITY RULES ARE PRINTED ON EVERY RUN, so that two readers of "
748
+ "the same report cannot silently disagree about what 'already equal' "
749
+ "means.\n\n"
750
+ f" match key: {KEY_RULE}\n"
751
+ f" values: {VALUE_RULE}\n\n"
752
+ "A row present in the target and absent from the source is NEVER a "
753
+ "delete. It is reported and nothing else. Sources: .json (array of row "
754
+ "objects, or {\"rows\": [...]}), .csv (header row), or --from-host to "
755
+ "use another environment as the source of truth."
756
+ ),
757
+ formatter_class=argparse.RawDescriptionHelpFormatter,
758
+ )
759
+ parser.add_argument("--host", default=argparse.SUPPRESS, metavar="HOST",
760
+ help="deployment to compare AGAINST (default: formist)")
761
+ parser.add_argument("--space", help="space UUID (never a name)")
762
+ parser.add_argument(
763
+ "kind",
764
+ help=f"one of: {resolve_mod.kind_names()} (case-insensitive; - and _ alike)",
765
+ )
766
+ parser.add_argument("--entity-type", dest="entity_type",
767
+ help="EntityType single_name; required for the `custom` kind")
768
+ parser.add_argument("--source", help="path to a .json or .csv source of truth")
769
+ parser.add_argument("--from-host", dest="from_env", metavar="HOST",
770
+ help="use another deployment as the source of truth")
771
+ parser.add_argument("--key", default="name",
772
+ help="the column rows are matched by (default: name)")
773
+ parser.add_argument("--fields",
774
+ help="comma-separated columns to compare (default: every "
775
+ "column the source mentions)")
776
+ parser.add_argument("--allow-shape-change", dest="allow_shape", action="store_true",
777
+ help="emit ops that change a value's SHAPE (string vs map). "
778
+ "Off by default, since coalescing the two can hide a "
779
+ "real difference")
780
+ parser.add_argument("--out", help="write the ops payload here")
781
+ parser.add_argument("--report", help="write the full report (rules, deletions, "
782
+ "normalisations) here")
783
+ parser.add_argument("--json", action="store_true", help="print the report as JSON")
784
+
785
+
786
+ def cmd_diff(args) -> int:
787
+ import sys
788
+
789
+ # Imported inside the function because `cli` imports this module. Reusing
790
+ # its `_effective`/`_token_for` rather than copying these lines keeps one
791
+ # provenance banner in the tool.
792
+ from formist_cli.cli import _effective, _token_for
793
+
794
+ eff = _effective(args)
795
+ token = _token_for(eff)
796
+ kind = resolve_mod.require_kind(args.kind)
797
+
798
+ if bool(args.source) == bool(args.from_env):
799
+ raise DiffError(
800
+ "give exactly one of --source (a file) or --from-host (another "
801
+ "deployment).\n"
802
+ "Two sources of truth is not a source of truth."
803
+ )
804
+
805
+ # The local source is read first, before any network call: a mistyped
806
+ # --source is the most likely error in this command and the cheapest to
807
+ # detect, and fetching the target first would report it only after a
808
+ # round trip and, against prod, after an admin auth check. Cheap,
809
+ # local, certain checks go first.
810
+ source = load_source(args.source) if args.source else None
811
+
812
+ target_rows, data_field, extra_set = target_rows_for(
813
+ kind=kind, target=eff.target, space=eff.space, token=token,
814
+ entity_type=args.entity_type,
815
+ )
816
+
817
+ if source is None:
818
+ from formist_cli import prefs as prefs_mod
819
+ from formist_cli.targets import resolve as resolve_target
820
+
821
+ src_env = prefs_mod.canonical_env(args.from_env)
822
+ if src_env == eff.target.name:
823
+ raise DiffError(
824
+ f"--from-host {src_env!r} is the deployment being compared against. "
825
+ "A diff of an environment with itself is empty by construction."
826
+ )
827
+ src_target = resolve_target(src_env)
828
+ from formist_cli import tokens as tokens_mod
829
+
830
+ src_token, _ = tokens_mod.load_for(src_target.name)
831
+ print(f" source of truth: {src_target.name} ({src_target.api_base})",
832
+ file=sys.stderr)
833
+ source = source_from_env(
834
+ kind=kind, target=src_target, space=eff.space, token=src_token
835
+ )
836
+
837
+ result = build_diff(
838
+ kind=kind,
839
+ key_field=args.key,
840
+ fields=[f.strip() for f in args.fields.split(",")] if args.fields else None,
841
+ source=source,
842
+ target_rows=target_rows,
843
+ env=eff.target.name,
844
+ space=eff.space,
845
+ data_field=data_field,
846
+ allow_shape_change=args.allow_shape,
847
+ extra_set=extra_set,
848
+ )
849
+
850
+ report = result.report()
851
+ if args.json:
852
+ print(json.dumps(report, indent=2, ensure_ascii=False, default=str))
853
+ else:
854
+ _print_report(result, report)
855
+
856
+ if args.report:
857
+ with open(args.report, "w", encoding="utf-8") as handle:
858
+ json.dump(report, handle, indent=2, ensure_ascii=False, default=str)
859
+ print(f"\nreport -> {args.report}", file=sys.stderr)
860
+
861
+ ops = result.ops()
862
+ if args.out:
863
+ with open(args.out, "w", encoding="utf-8") as handle:
864
+ json.dump({"ops": ops}, handle, indent=2, ensure_ascii=False, default=str)
865
+ print(
866
+ f"ops -> {args.out} ({len(ops)} op(s))\n"
867
+ f"\nRead it, then:\n"
868
+ f" fm update --host {result.env} --space {result.space} {args.out}",
869
+ file=sys.stderr,
870
+ )
871
+ elif ops:
872
+ print(
873
+ f"\n{len(ops)} op(s) computed but not written. Pass --out FILE.",
874
+ file=sys.stderr,
875
+ )
876
+ return 0
877
+
878
+
879
+ def _print_report(result: DiffResult, report: dict) -> None:
880
+ counts = report["counts"]
881
+ print(f"{result.kind} in {result.env} / {result.space}")
882
+ print(f" source {result.source.origin} [{result.source.fmt}] "
883
+ f"sha256:{result.source.digest[:12]} {len(result.source.rows)} rows")
884
+ print(f" target {result.target_count} rows [reconciled: unchanged + update "
885
+ "+ blocked + would-delete]")
886
+ print()
887
+ print(f" match key {result.key_field}: {KEY_RULE}")
888
+ print(f" values {VALUE_RULE}")
889
+ print(f" writes {report['equality_rules']['written_column']}")
890
+ print()
891
+ print(f" create {counts['create']} update {counts['update']} "
892
+ f"unchanged {counts['unchanged']} "
893
+ f"blocked-on-shape {counts['blocked_on_shape']} "
894
+ f"would-delete {counts['would_delete']}")
895
+
896
+ if result.normalised:
897
+ print("\n keys that changed under normalisation "
898
+ "(an untrimmed name creates a phantom row):")
899
+ for raw, norm in result.normalised:
900
+ print(f" {raw!r} -> {norm!r}")
901
+
902
+ for change in result.creates:
903
+ print(f"\n + {change.key}")
904
+ for change in result.updates:
905
+ print(f"\n ~ {change.key}")
906
+ for column, (target_value, source_value, why) in change.fields.items():
907
+ print(f" {column}: {_show(target_value)} -> {_show(source_value)} [{why}]")
908
+ if change.preserved:
909
+ print(f" (preserved, not in source: {', '.join(change.preserved)})")
910
+ for change in result.blocked_shape:
911
+ print(f"\n ! {change.key} BLOCKED, shape change, no op emitted")
912
+ for reason in change.shape_changes:
913
+ print(f" {reason}")
914
+ print(" A bare string and an {en,...} map are different values; "
915
+ "coalescing them can hide a real difference.")
916
+ print(" Pass --allow-shape-change if that is genuinely what you mean.")
917
+ if result.ambiguous:
918
+ print(f"\n ambiguous in {result.env} ({len(result.ambiguous)}), NOT "
919
+ "referenced by the source, so nothing here was compared or written:")
920
+ for key, ids in result.ambiguous:
921
+ print(f" ? {key} -> {', '.join(ids)}")
922
+ print(" A name identifying two rows identifies neither. These are "
923
+ "pre-existing duplicates in the target, not something this diff "
924
+ "introduced, but a later source that does name one of them will be "
925
+ "refused outright.")
926
+
927
+ if result.would_delete:
928
+ print(f"\n would-delete ({len(result.would_delete)}), reported only, never "
929
+ "emitted as an op:")
930
+ for key in result.would_delete:
931
+ print(f" - {key}")
932
+ print(" A row missing from the source is not authority to delete it. The "
933
+ "source may simply be partial.")
934
+
935
+
936
+ def _show(value: Any) -> str:
937
+ """A value as JSON, never through str().
938
+
939
+ `str({"en": "FAO"})` is a dict repr: it matches nothing, and
940
+ printing one beside a real value invites comparing them by eye
941
+ and concluding they agree.
942
+ """
943
+ return json.dumps(value, ensure_ascii=False, default=str)