optivibe-reference 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. optivibe_reference/__init__.py +14 -0
  2. optivibe_reference/_envelope.py +19 -0
  3. optivibe_reference/_paths.py +66 -0
  4. optivibe_reference/_probe_common.py +142 -0
  5. optivibe_reference/build_data.py +271 -0
  6. optivibe_reference/build_steps.py +182 -0
  7. optivibe_reference/catalog_build.py +1138 -0
  8. optivibe_reference/data/operand_semantics.json +2192 -0
  9. optivibe_reference/data/operand_synonyms.json +1687 -0
  10. optivibe_reference/data/provenance_grants.txt +5 -0
  11. optivibe_reference/data/tolerance_operand_synonyms.json +307 -0
  12. optivibe_reference/errors.py +57 -0
  13. optivibe_reference/glass_build.py +433 -0
  14. optivibe_reference/glass_dispersion.py +207 -0
  15. optivibe_reference/glass_math.py +383 -0
  16. optivibe_reference/glass_parse.py +168 -0
  17. optivibe_reference/manual_build.py +672 -0
  18. optivibe_reference/manual_enrich.py +490 -0
  19. optivibe_reference/probe_operands.py +226 -0
  20. optivibe_reference/probe_tolerances.py +371 -0
  21. optivibe_reference/server.py +547 -0
  22. optivibe_reference/tolerance_build.py +554 -0
  23. optivibe_reference/tools/__init__.py +5 -0
  24. optivibe_reference/tools/_glass_common.py +137 -0
  25. optivibe_reference/tools/_glass_kind.py +133 -0
  26. optivibe_reference/tools/_query_params.py +57 -0
  27. optivibe_reference/tools/_rag_match.py +92 -0
  28. optivibe_reference/tools/find_glass_pair.py +574 -0
  29. optivibe_reference/tools/find_glasses.py +339 -0
  30. optivibe_reference/tools/lookup_glass.py +262 -0
  31. optivibe_reference/tools/lookup_operand.py +463 -0
  32. optivibe_reference/tools/search_reference.py +321 -0
  33. optivibe_reference-0.2.0.dist-info/METADATA +44 -0
  34. optivibe_reference-0.2.0.dist-info/RECORD +37 -0
  35. optivibe_reference-0.2.0.dist-info/WHEEL +5 -0
  36. optivibe_reference-0.2.0.dist-info/entry_points.txt +2 -0
  37. optivibe_reference-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1138 @@
1
+ """catalog_build.py — build the MeritOperandType operand catalog (JSON + DB).
2
+
3
+ The catalog has THREE artifacts and this module owns the pipeline between them:
4
+
5
+ 1. ``build_catalog_json`` — turn the two LIVE PROBE captures
6
+ (``operand_inventory_438.json`` = the 438-code primary-key set, and
7
+ ``probe_operands_capture.json`` = the per-operand cell-layout battery) into a
8
+ normalized catalog dict (one row per code, schema §2).
9
+ 2. ``write_catalog_json`` — persist that dict as a committed, normalized-LF
10
+ (``newline="\n"``), ``indent=2`` JSON under ``data/operand_catalog.json``.
11
+ Committed as TEXT so the forbidden-token scanner sees it (the binary ``.db``
12
+ is non-reproducible and is NEVER committed — §2 / §7).
13
+ 3. ``build_db`` / ``open_catalog`` — build the SQLite keyed ``operand`` table +
14
+ external-content FTS5 ``operand_fts`` from the committed JSON, at TEST/RUNTIME.
15
+
16
+ Provenance discipline (§4): Step-1 live probe RESOLVED NEGATIVE on engine labels
17
+ (``op.TypeName`` / ``op.RowTypeName`` return only the CODE, no prose), so
18
+ ``description_source='live_label'`` is NEVER available for a meaning. All 438 rows
19
+ start with ``description=None`` / ``description_source='authored'`` /
20
+ ``description_pending=1`` pending the manual/authored ingest. The 4 hand-authored
21
+ probe-literal strings (DIST/REAY/REAX/EFFL) must NEVER be tagged ``live_label``.
22
+
23
+ The .NET enum INTEGER value is version-unstable and is NEVER persisted — only the
24
+ code NAME string keys (§2).
25
+ """
26
+ import json
27
+ import os
28
+ import re
29
+ import sqlite3
30
+ import sys
31
+
32
+ from . import _paths, manual_build
33
+ from .manual_build import normalize as _normalize_text
34
+
35
+
36
+ class BuildContractError(AssertionError):
37
+ """A production BUILD-PATH contract gate failed (F5 — survives ``python -O``).
38
+
39
+ Subclasses ``AssertionError`` deliberately: the ``assert`` STATEMENT is stripped
40
+ under ``python -O`` / ``PYTHONOPTIMIZE`` (re-opening every §3-matrix / provenance /
41
+ coverage fail-loud hole at once — external F5), but a ``raise`` statement is
42
+ NEVER stripped, so a build-path contract violation still HARD-ERRORS under ``-O``.
43
+ Subclassing ``AssertionError`` keeps the many existing ``pytest.raises(AssertionError)``
44
+ contract pins green while giving the gate an explicit, named build-error class.
45
+ A pure internal-consistency invariant that can NEVER fire on user/committed input
46
+ (e.g. the field-disjointness assert over module constants) stays a plain ``assert``.
47
+ """
48
+
49
+
50
+ def _require(cond, msg):
51
+ """Raise ``BuildContractError(msg)`` unless ``cond`` — the F5 assert→raise helper.
52
+
53
+ Use for every BUILD-PATH contract gate over user/committed data so the check
54
+ survives ``python -O``; keep a genuinely-internal (can't-fire-on-input) sanity
55
+ check as a plain ``assert``.
56
+ """
57
+ if not cond:
58
+ raise BuildContractError(msg)
59
+
60
+
61
+ # The provenance source tags allowed on a catalog row (§2). ``live_label`` is in
62
+ # the SET but Step-1 proved it is never *available* for a meaning — a guard
63
+ # asserts no row actually carries it. ``manual_verbatim`` is the honest
64
+ # tag for the gitignored, locally-built verbatim descriptions merged from the raw
65
+ # manual extract; a build assert forbids it in ANY COMMITTED JSON (R6/R7).
66
+ ALLOWED_DESCRIPTION_SOURCES = ("live_label", "chm", "authored", "manual_verbatim")
67
+ ALLOWED_UNITS_SOURCES = ("chm", "authored") # NEVER 'live_label' (§2: no live unit attr)
68
+
69
+ # The operand-semantics §3/§4 closed vocabularies. Imported by the §9 tests so
70
+ # the enum sets have a single source of truth. ``None`` is permitted alongside
71
+ # every set (honest-unknown) but is NOT a member token.
72
+ ALLOWED_UNITS = (
73
+ "lens_units",
74
+ "inverse_lens_units",
75
+ "waves",
76
+ "degrees",
77
+ "radians",
78
+ "percent",
79
+ "dimensionless",
80
+ ) # §3 closed units vocab (7 tokens + null)
81
+ ALLOWED_SIGN_CONVENTIONS = (
82
+ "boundary_ge",
83
+ "boundary_le",
84
+ "equality",
85
+ "minimize",
86
+ "maximize",
87
+ "measurement",
88
+ ) # §4 closed sign enum (6 values + null)
89
+ ALLOWED_SIGN_SOURCES = ("suffix", "family", "manual") # §4 source set (+ null); NEVER live_label
90
+ # A directional sign (one of these) REQUIRES a non-null sign_convention_source
91
+ # (§7 guard 2). ``measurement`` / ``null`` may be sourceless.
92
+ _DIRECTIONAL_SIGNS = ("boundary_ge", "boundary_le", "equality", "minimize", "maximize")
93
+
94
+ # §4 source⇔sign COMPATIBILITY table: each provenance source may only carry the
95
+ # signs the rule that source represents can produce. ``suffix`` is the mechanical
96
+ # 2-letter rule (GT/LT/VA). ``family`` is an MN*/MX* + oracle keyword (ge/le only).
97
+ # ``manual`` is the cited minimize/maximize overlay. ``measurement`` and a null
98
+ # sign are sourceless (source MUST be None — enforced separately). A
99
+ # ``{boundary_ge, manual}`` or ``{measurement, suffix}`` row is a mistagged
100
+ # direction and MUST be rejected (defense in depth — the catalog build is the
101
+ # authoritative gate; the generator self-validates the same table pre-write).
102
+ SIGN_SOURCE_COMPATIBILITY = {
103
+ "suffix": frozenset({"boundary_ge", "boundary_le", "equality"}),
104
+ "family": frozenset({"boundary_ge", "boundary_le"}),
105
+ "manual": frozenset({"minimize", "maximize"}),
106
+ }
107
+
108
+ # Suffix-collision carve-out (semantics §4). These codes END in
109
+ # GT/LT but are NOT boundary operands (LOGT=log transform, FCGT=tangential field
110
+ # curvature, NSLT=non-sequential LightningTrace). This frozenset is the SINGLE
111
+ # canonical definition — ``scripts/build_operand_semantics.py`` imports it (no
112
+ # duplicate literal, no drift hazard). The §7.4 contradiction guard SKIPS these so
113
+ # LOGT's incidental log-domain "less than ... zero" wording cannot false-positive
114
+ # the assert. The guard still BITES a genuine ``*GT`` boundary whose oracle says
115
+ # "less than the target" — only these 3 are excused.
116
+ SUFFIX_COLLISIONS = frozenset({"LOGT", "FCGT", "NSLT"})
117
+ # Back-compat alias for existing internal references / tests.
118
+ _SUFFIX_COLLISIONS = SUFFIX_COLLISIONS
119
+
120
+ # The 4 in-repo "descriptions" are HAND-AUTHORED probe-script literals, NOT
121
+ # engine-harvested (§4). No row may tag these ``live_label``. Kept here as
122
+ # the build-guard reference set; for this increment all descriptions are null so
123
+ # none is tagged anything but ``authored``.
124
+ PROBE_LITERAL_CODES = ("DIST", "REAY", "REAX", "EFFL")
125
+
126
+ # The current catalog schema version (§2: additive growth). Bumped 1->2 by the
127
+ # operand-semantics cycle (adds units/units_source + sign_convention/sign_convention_source).
128
+ # Bumped 2->3 by the tolerance cycle (adds the 3 tolerance-safety columns
129
+ # category/precondition_class/run_verdict — NULL for every merit row, real values
130
+ # only on the separate tolerance_operand catalog built by tolerance_build).
131
+ SCHEMA_VERSION = 3
132
+
133
+ # Cols 2..9 are the operand-specific PARAM cells; cols 1 ("Type") and 10..13
134
+ # (Target/Weight/Value/% Contrib, fixed across all operands) are NOT params.
135
+ _PARAM_COL_MIN = 2
136
+ _PARAM_COL_MAX = 9
137
+
138
+ # Path anchors (package-relative, NOT cwd-relative — agent cwd resets §5).
139
+ _HERE = os.path.dirname(os.path.abspath(__file__))
140
+ DATA_DIR = os.path.join(_HERE, "data")
141
+ CATALOG_JSON_PATH = os.path.join(DATA_DIR, "operand_catalog.json")
142
+ # Committed SYNONYMS + units source of truth (§1): one key per enriched
143
+ # merit code (the keyset IS the expected-enriched set — M1). Carries
144
+ # ``synonyms``/``units``/``units_source`` ONLY (description/citation/source are
145
+ # verbatim-local + gitignored). ``schema_version`` MUST be 1 (asserted).
146
+ SYNONYMS_JSON_PATH = os.path.join(DATA_DIR, "operand_synonyms.json")
147
+ # The GITIGNORED, user-built VERBATIM raw operand extract (the manual oracle),
148
+ # emitted by ``scripts/build_manual_corpus.py --emit-raw``. Reshaped into the
149
+ # ``description``/``citation_handle``/``description_source='manual_verbatim'``
150
+ # merge fields at build time. ABSENT on a fresh install -> synonyms-only build.
151
+ RAW_DESCRIPTIONS_PATH = os.path.join(
152
+ _paths.captures_dir(), "operand_raw_descriptions.json",
153
+ )
154
+ # Committed, regenerable deterministic sign-convention artifact (semantics §1) emitted
155
+ # by ``scripts/build_operand_semantics.py``. Merged into the catalog at build time.
156
+ SEMANTICS_JSON_PATH = os.path.join(DATA_DIR, "operand_semantics.json")
157
+
158
+ # The synonyms-envelope schema version (§1). An unknown version is a HARD
159
+ # ERROR (loud) — the build refuses to merge a synonyms file it cannot understand.
160
+ SYNONYMS_SCHEMA_VERSION = 1
161
+
162
+ # The verbatim-derived fields that NEVER appear in a committed synonyms file (they
163
+ # are gitignored verbatim-local, §1). A committed-JSON assert enforces this.
164
+ _SHIP_FORBIDDEN_FIELDS = ("description", "citation_handle", "description_source")
165
+
166
+ # Field ownership for the disjoint by-field merge (§2.2): the synonyms file
167
+ # owns synonyms/units (+ tolerance safety fields); the local descriptions own the
168
+ # verbatim fields. Asserted disjoint so a conflicting ``units`` collision is
169
+ # structurally impossible.
170
+ _SYNONYM_FIELD_NAMES = frozenset(
171
+ {"synonyms", "units", "units_source",
172
+ "category", "precondition_class", "run_verdict"}
173
+ )
174
+ _LOCAL_DESC_FIELD_NAMES = frozenset(_SHIP_FORBIDDEN_FIELDS)
175
+
176
+
177
+ def _load_json(path):
178
+ with open(path, "r", encoding="utf-8") as fh:
179
+ return json.load(fh)
180
+
181
+
182
+ def _synonyms_to_text(synonyms):
183
+ """Coerce a row's ``synonyms`` field to a clean space-joined FTS string.
184
+
185
+ The committed ``operand_synonyms.json`` stores synonyms as a single
186
+ space-joined STRING (``load_synonyms_rows`` requires a ``str``), but a catalog
187
+ JSON handed to ``build_db`` may still carry a list/tuple (e.g.
188
+ ``["anti reflection", "AR coating"]``). A list rendered through ``str``/format
189
+ would emit a Python repr (``['anti reflection', 'AR coating']``) and pollute
190
+ the FTS index with brackets, quotes, and commas. This normalizes:
191
+
192
+ - ``None`` -> ``""`` (no trailing junk);
193
+ - ``str`` -> the SAME string byte-for-byte (current behavior is preserved);
194
+ - ``list``/``tuple`` -> its items space-joined, each coerced to ``str``.
195
+ """
196
+ if synonyms is None:
197
+ return ""
198
+ if isinstance(synonyms, str):
199
+ return synonyms
200
+ if isinstance(synonyms, (list, tuple)):
201
+ return " ".join(str(item) for item in synonyms)
202
+ return str(synonyms)
203
+
204
+
205
+ def _oracle_boundary_dir(raw_text):
206
+ """Classify the BOUNDARY direction documented in oracle prose, or None.
207
+
208
+ Returns ``'ge'`` if the prose states a "greater than the target/specified"
209
+ boundary, ``'le'`` for "less than the target/specified", else ``None``.
210
+
211
+ Deliberately matches only the BOUNDARY-TARGET phrasing real boundary operands
212
+ use (constrain a value vs the target/specified value), NOT an incidental
213
+ "less than" (e.g. "...if the value is less than or equal to zero..."), so a
214
+ non-boundary operand whose mechanical 2-letter suffix happens to be GT/LT is
215
+ not misread as a boundary contradiction (§7.4 firewall against false
216
+ positives). The oracle text is run-together for some entries, so the
217
+ whitespace-stripped form is also probed.
218
+ """
219
+ text = re.sub(r"\s+", " ", (raw_text or "")).lower()
220
+ squished = text.replace(" ", "")
221
+ greater = ("greater than the target" in text or "greater than the spe" in text
222
+ or "greaterthanthetarget" in squished or "greaterthanthespe" in squished)
223
+ less = ("less than the target" in text or "less than the spe" in text
224
+ or "lessthanthetarget" in squished or "lessthanthespe" in squished)
225
+ if greater and not less:
226
+ return "ge"
227
+ if less and not greater:
228
+ return "le"
229
+ return None
230
+
231
+
232
+ def _param_cells_for(operand_entry):
233
+ """Extract the cols 2..9 non-blank-header param cells from a probe entry.
234
+
235
+ Returns a list of ``{"col", "header", "data_type"}`` dicts (in column order)
236
+ for cells whose ``Header`` is non-blank. A blank header (``" "`` or ``""``)
237
+ means that column is an unused trailing param slot for this operand and is
238
+ dropped. Cols 1 and 10..13 are structural and never included.
239
+ """
240
+ cells = operand_entry.get("cells", {})
241
+ out = []
242
+ for col in range(_PARAM_COL_MIN, _PARAM_COL_MAX + 1):
243
+ cell = cells.get(str(col))
244
+ # A GetCellAt error surfaces as a string, not a dict — skip it defensively.
245
+ if not isinstance(cell, dict):
246
+ continue
247
+ header = cell.get("Header", "")
248
+ if header is None or header.strip() == "":
249
+ continue
250
+ out.append(
251
+ {
252
+ "col": cell.get("Col", col),
253
+ "header": header.strip(),
254
+ "data_type": cell.get("DataType"),
255
+ }
256
+ )
257
+ return out
258
+
259
+
260
+ # The 3 tolerance-safety fields every committed TOLERANCE synonyms row must carry
261
+ # (a hardening finding). Merit rows never carry them.
262
+ _TOLERANCE_SAFETY_FIELDS = ("category", "precondition_class", "run_verdict")
263
+
264
+ # Per-schema committed-synonyms field ALLOWLISTS (F3 — the provenance boundary is
265
+ # schema-CLOSED, not a 3-field denylist that fails open on an unknown field). A
266
+ # committed MERIT synonyms row may carry ONLY these fields; a TOLERANCE row adds the
267
+ # 3 safety fields. ANY field outside the allowlist — an oracle ``raw_text`` carrying
268
+ # verbatim prose, or the verbatim ``description``/``citation_handle``/
269
+ # ``description_source`` fields — is a HARD ERROR (unknown/verbatim content can never
270
+ # ride a committed artifact, R6/R7). This SUBSUMES the old ``_SHIP_FORBIDDEN_FIELDS``
271
+ # denylist (those 3 are simply not in the allowlist).
272
+ _SYNONYMS_ALLOWED_FIELDS = frozenset({"synonyms", "units", "units_source"})
273
+ _TOLERANCE_SYNONYMS_ALLOWED_FIELDS = _SYNONYMS_ALLOWED_FIELDS | frozenset(
274
+ _TOLERANCE_SAFETY_FIELDS
275
+ )
276
+
277
+ # The committed ``operand_semantics.json`` row ALLOWLIST (schema-CLOSED, the same
278
+ # discipline as the synonyms allowlists above) — the ONE definition;
279
+ # ``scripts/project_release_semantics.py`` imports it rather than carrying a second
280
+ # literal. The merge CONSUMES ``sign_convention`` + ``sign_convention_source`` (both
281
+ # REQUIRED on every row, explicit ``null`` allowed). ``citation_handle`` is OPTIONAL
282
+ # and allowed-but-ignored by the build (a page POINTER; the release projection nulls
283
+ # it). Any other key is a HARD ERROR. Enforced by ``_check_semantics_rows``.
284
+ _ALLOWED_SEMANTICS_FIELDS = frozenset(
285
+ {"sign_convention", "sign_convention_source", "citation_handle"}
286
+ )
287
+ _REQUIRED_SEMANTICS_FIELDS = frozenset({"sign_convention", "sign_convention_source"})
288
+
289
+
290
+ def _check_semantics_rows(semantics):
291
+ """Validate each ``operand_semantics.json`` row's SHAPE (schema-closed allowlist).
292
+
293
+ Every row is a ``dict``; it carries no key outside ``_ALLOWED_SEMANTICS_FIELDS``;
294
+ and it carries BOTH consumed keys (``_REQUIRED_SEMANTICS_FIELDS``, explicit
295
+ ``null`` allowed) — a ``{}`` row would otherwise merge as ``None``/``None`` while
296
+ ``semantics_state`` reads ``"present"``. Each failure is a ``BuildContractError``
297
+ naming the operand and the key(s).
298
+ """
299
+ _require(isinstance(semantics, dict), (
300
+ f"operand_semantics.json is not a {{code: row}} object "
301
+ f"({type(semantics).__name__})"
302
+ ))
303
+ for code, row in semantics.items():
304
+ _require(isinstance(row, dict), (
305
+ f"{code}: operand_semantics.json row is not a dict "
306
+ f"({type(row).__name__})"
307
+ ))
308
+ extra = set(row) - _ALLOWED_SEMANTICS_FIELDS
309
+ _require(not extra, (
310
+ f"{code}: operand_semantics.json row carries field(s) {sorted(extra)} "
311
+ f"outside the allowlist {sorted(_ALLOWED_SEMANTICS_FIELDS)} "
312
+ "(unknown content must never ride the semantics merge)"
313
+ ))
314
+ missing = _REQUIRED_SEMANTICS_FIELDS - set(row)
315
+ _require(not missing, (
316
+ f"{code}: operand_semantics.json row is missing required field(s) "
317
+ f"{sorted(missing)} (explicit null is allowed; an absent key is not)"
318
+ ))
319
+
320
+
321
+ def load_synonyms_rows(synonyms_path, *, tolerance=False):
322
+ """Load + validate a committed synonyms file; return its ``rows`` dict.
323
+
324
+ Asserts ``schema_version == 1`` (an unknown version is a HARD ERROR — the build
325
+ refuses to merge a file it cannot understand, §1) and that no committed
326
+ synonyms row carries a ship-forbidden verbatim field (R6/R7 committed-JSON
327
+ assert).
328
+
329
+ Per-row CONTENT validation (a hardening finding — a malformed synonym
330
+ HARD-ERRORS at build instead of silently shipping a lost synonym):
331
+
332
+ - each ``row`` is a ``dict``;
333
+ - ``"synonyms"`` is PRESENT and a ``str`` (``""`` is permitted — the KEY is the
334
+ coverage signal — but a ``null``/missing/non-``str`` synonyms is a HARD ERROR,
335
+ never silently "present");
336
+ - the ``units`` ⇔ ``units_source`` PAIRING invariant (both present or both
337
+ absent — a half-specified units, either an unsourced unit OR an orphaned
338
+ source tag, is a HARD ERROR);
339
+ - when ``tolerance`` is True, every row carries all 3 tolerance-safety fields
340
+ (``category`` / ``precondition_class`` / ``run_verdict``).
341
+
342
+ Returns ``{code: {synonyms, units?, units_source?, <tol safety?>}}``.
343
+ """
344
+ doc = _load_json(synonyms_path)
345
+ version = doc.get("schema_version")
346
+ _require(version == SYNONYMS_SCHEMA_VERSION, (
347
+ f"{synonyms_path}: unknown synonyms schema_version {version!r} "
348
+ f"(expected {SYNONYMS_SCHEMA_VERSION})"
349
+ ))
350
+ rows = doc.get("rows", {})
351
+ # F1: an EMPTY keyset on a PROVIDED committed synonyms file is a broken checkout
352
+ # (the keyset IS the expected-enriched set; zero rows would silently publish a
353
+ # synonym-less catalog at exit 0). A HARD ERROR — the shared locus for both the
354
+ # merit and the tolerance build (both call this loader with a real path).
355
+ _require(isinstance(rows, dict) and len(rows) > 0, (
356
+ f"{synonyms_path}: committed synonyms file has an EMPTY 'rows' keyset "
357
+ "(a zero-row committed synonyms file is a broken checkout — the keyset is "
358
+ "the expected-enriched set)"
359
+ ))
360
+ allowed = (
361
+ _TOLERANCE_SYNONYMS_ALLOWED_FIELDS if tolerance else _SYNONYMS_ALLOWED_FIELDS
362
+ )
363
+ for code, row in rows.items():
364
+ _require(isinstance(row, dict), (
365
+ f"{code}: committed synonyms row is not a dict ({type(row).__name__})"
366
+ ))
367
+ # F3: per-schema ALLOWLIST — the provenance boundary is schema-CLOSED. ANY
368
+ # field outside the allowlist (an oracle ``raw_text`` / a verbatim field /
369
+ # any unknown key) is a HARD ERROR, not silently carried.
370
+ extra = set(row) - allowed
371
+ _require(not extra, (
372
+ f"{code}: committed synonyms row carries field(s) {sorted(extra)} "
373
+ f"outside the allowlist {sorted(allowed)} (verbatim/unknown content "
374
+ "must never ride a committed artifact)"
375
+ ))
376
+ # synonyms KEY present + a str (empty permitted; null/non-str is a HARD ERROR).
377
+ _require("synonyms" in row, (
378
+ f"{code}: committed synonyms row is MISSING the 'synonyms' key "
379
+ f"(a lost synonym must not ship silently)"
380
+ ))
381
+ _require(isinstance(row["synonyms"], str), (
382
+ f"{code}: 'synonyms' must be a str (empty permitted), got "
383
+ f"{type(row['synonyms']).__name__} {row['synonyms']!r}"
384
+ ))
385
+ # units ⇔ units_source pairing — VALUE-aware (A3): both PRESENT with non-null
386
+ # values OR both ABSENT. A half-specified units (an orphaned source tag OR a
387
+ # sourceless unit) AND a present-but-null units/units_source both HARD-ERROR.
388
+ has_units = "units" in row
389
+ has_units_source = "units_source" in row
390
+ _require(has_units == has_units_source, (
391
+ f"{code}: units/units_source must be paired — got "
392
+ f"units={'present' if has_units else 'absent'}, "
393
+ f"units_source={'present' if has_units_source else 'absent'}"
394
+ ))
395
+ if has_units:
396
+ _require(
397
+ row["units"] is not None and row["units_source"] is not None, (
398
+ f"{code}: units/units_source are present but a value is null "
399
+ f"(units={row['units']!r}, units_source={row['units_source']!r}) "
400
+ "— a null-valued units pair is a half-specified unit"
401
+ )
402
+ )
403
+ if tolerance:
404
+ for field in _TOLERANCE_SAFETY_FIELDS:
405
+ _require(field in row, (
406
+ f"{code}: committed TOLERANCE synonyms row is MISSING the "
407
+ f"required safety field {field!r}"
408
+ ))
409
+ return rows
410
+
411
+
412
+ def reshape_raw_descriptions(raw):
413
+ """Reshape the gitignored raw manual extract into the merge-field shape.
414
+
415
+ ``raw`` = ``{code: {code, page, raw_text}}`` (the ``--emit-raw`` oracle) ->
416
+ ``{code: {description, citation_handle, description_source}}`` where
417
+ ``description`` is the ``normalize``-cleaned verbatim ``raw_text`` (R1: collapse
418
+ the FACT-4 glyph-spacing whitespace before it reaches the FTS body),
419
+ ``citation_handle='manual:p{page+1}'`` (the 1-based physical page via
420
+ ``manual_build.citation_page``; ``page`` stays 0-based),
421
+ ``description_source='manual_verbatim'``.
422
+
423
+ A row with an empty ``raw_text`` OR a missing ``page`` is SKIPPED (a description
424
+ can never land without a page cite — the provenance invariant).
425
+ """
426
+ out = {}
427
+ for code, entry in raw.items():
428
+ if isinstance(entry, dict):
429
+ raw_text = entry.get("raw_text")
430
+ page = entry.get("page")
431
+ else: # tolerant of a bare-string oracle value
432
+ raw_text, page = entry, None
433
+ if not raw_text or page is None:
434
+ continue
435
+ out[code] = {
436
+ "description": _normalize_text(raw_text),
437
+ "citation_handle": "manual:p{}".format(manual_build.citation_page(page)),
438
+ "description_source": "manual_verbatim",
439
+ }
440
+ return out
441
+
442
+
443
+ def merge_enrichment(synonyms, local_descriptions, expected_codes):
444
+ """Merge synonyms + (optional) verbatim-local descriptions into per-code rows.
445
+
446
+ Field-disjoint by ownership (§2.2): ``synonyms``/``units``/... come from
447
+ ``synonyms``; ``description``/``citation_handle``/``description_source`` from
448
+ ``local_descriptions``. ``expected_codes`` = the synonyms keyset (M1: the keyset
449
+ IS the expected-enriched set).
450
+
451
+ Returns ``(per_code_enrichment, report)``. Never emits fewer descriptions than
452
+ expected without a loud report:
453
+
454
+ - ``local_descriptions is None`` -> state ``synonyms_only`` (fresh install; NORMAL)
455
+ - present but missing some expected -> state ``partial`` (a REAL drop; LOUD)
456
+ - present, covers every expected -> state ``enriched``
457
+
458
+ ``report = {descriptions_state, enriched:int, expected:int, missing:sorted[:20]}``.
459
+ A code is "covered" iff ``local_descriptions`` supplies a non-null description.
460
+ """
461
+ per_code = {}
462
+ covered = set()
463
+ for code in expected_codes:
464
+ # Copy the synonyms-owned fields (synonyms + units + tolerance safety).
465
+ base = dict(synonyms.get(code, {}))
466
+ if local_descriptions is not None:
467
+ desc = local_descriptions.get(code)
468
+ if desc is not None and desc.get("description"):
469
+ base["description"] = desc["description"]
470
+ base["citation_handle"] = desc["citation_handle"]
471
+ base["description_source"] = desc.get(
472
+ "description_source", "manual_verbatim"
473
+ )
474
+ covered.add(code)
475
+ per_code[code] = base
476
+
477
+ expected = len(expected_codes)
478
+ if local_descriptions is None:
479
+ state = "synonyms_only"
480
+ missing = []
481
+ else:
482
+ missing = sorted(set(expected_codes) - covered)
483
+ state = "enriched" if not missing else "partial"
484
+
485
+ report = {
486
+ "descriptions_state": state,
487
+ "enriched": len(covered),
488
+ "expected": expected,
489
+ "missing": missing[:20],
490
+ }
491
+ return per_code, report
492
+
493
+
494
+ def build_catalog_json(
495
+ inventory_path,
496
+ probe_capture_path,
497
+ *,
498
+ synonyms_path=None,
499
+ local_descriptions_path=None,
500
+ semantics_path=None,
501
+ ):
502
+ """Build the normalized catalog dict via the field-disjoint two-source merge.
503
+
504
+ One row per code (schema §2). The by-field ownership (§2.2):
505
+ ``synonyms``/``units``/``units_source`` <- the committed synonyms file ONLY;
506
+ ``description``/``citation_handle``/``description_source`` <- the gitignored
507
+ verbatim-local raw extract ONLY; signs <- ``operand_semantics.json`` only. A
508
+ synonyms-only build (``local_descriptions_path=None`` — the fresh-install
509
+ state) carries synonyms yet every row stays ``description=None`` /
510
+ ``description_pending=1`` (honest, NOT an error). ``build_report`` is stamped
511
+ into the top-level so the merge state is readable downstream.
512
+
513
+ ``cell_layout`` carries the cols 2..9 param-cell list ONLY for probe-covered
514
+ operands; un-covered operands get ``None`` (NEVER an inferred layout — §3).
515
+
516
+ Asserts the code set is unique AND its size equals the inventory file's
517
+ ``total_members`` (NO hardcoded 438 — §3: assert against the live count) AND
518
+ the field-disjoint merge is structurally conflict-free.
519
+ """
520
+ inventory = _load_json(inventory_path)
521
+ probe = _load_json(probe_capture_path)
522
+
523
+ synonyms = load_synonyms_rows(synonyms_path) if synonyms_path else {}
524
+ # The verbatim-local descriptions (gitignored). ``None`` (path absent or not
525
+ # supplied) => synonyms-only build; present => reshape the raw extract.
526
+ if local_descriptions_path:
527
+ local_descriptions = reshape_raw_descriptions(
528
+ _load_json(local_descriptions_path)
529
+ )
530
+ else:
531
+ local_descriptions = None
532
+ # Semantics §1: the deterministic sign-convention artifact (suffix/family/overlay),
533
+ # keyed by code. Absent PATH -> empty merge (every row's sign stays None). F1: the
534
+ # gate is on whether the FILE was PROVIDED (``semantics_path is not None``), NEVER
535
+ # on truthiness — an empty/short PARSED semantics from a provided path is a broken
536
+ # projection, caught by the coverage check below, not silently skipped.
537
+ semantics_provided = semantics_path is not None
538
+ semantics = _load_json(semantics_path) if semantics_provided else {}
539
+
540
+ # Structural disjointness (§2.2): the two enrichment sources own DISJOINT
541
+ # fields, so a conflicting ``units`` collision is impossible. Asserted.
542
+ assert not (_LOCAL_DESC_FIELD_NAMES & _SYNONYM_FIELD_NAMES) - {"code"}, (
543
+ "local-description and synonym field ownership overlap"
544
+ )
545
+
546
+ members = inventory["members"]
547
+ codes = [m["code"] for m in members]
548
+ total_members = inventory["total_members"]
549
+ osv = inventory.get("optic_studio_version")
550
+
551
+ # Python-layer dedupe + completeness guard (necessary, not sufficient; the SQL
552
+ # PRIMARY KEY is the storage-layer enforcement). Assert against the file's
553
+ # total_members, NEVER a literal. F5: an explicit raise (survives ``-O``).
554
+ _require(len(codes) == len(set(codes)) == total_members, (
555
+ f"code-set integrity: {len(codes)} codes, {len(set(codes))} unique, "
556
+ f"{total_members} declared total_members"
557
+ ))
558
+
559
+ code_set = set(codes)
560
+
561
+ # Semantics §7 guard 5 (PRE-MERGE, non-vacuous): every semantics key is a known
562
+ # inventory code (no stale/renamed key survives). AND (§2.2 / F1) when a
563
+ # semantics FILE was PROVIDED it must COVER the full inventory code set — a short
564
+ # OR EMPTY (``{}``) parsed semantics is a broken projection (HARD ERROR, closes the
565
+ # stale-oracle silent sign-drop). Gated on ``semantics_provided`` (is-not-None),
566
+ # NEVER truthiness — an empty ``{}`` from a provided path now HARD-ERRORS via the
567
+ # ``missing_sem`` check rather than silently writing 438 null signs at exit 0.
568
+ #
569
+ # SEMANTICS ROW CONTRACT — the semantics file's own ROWS are visible only here (the
570
+ # ``assert_*_invariants`` guards run over the BUILT catalog), so the row-shape
571
+ # allowlist is enforced here, BEFORE guard 5: ``_check_semantics_rows``. The merge
572
+ # below CONSUMES exactly ``sign_convention`` + ``sign_convention_source`` (both
573
+ # REQUIRED per row, explicit ``null`` allowed). ``citation_handle`` is
574
+ # allowed-and-IGNORED here: it is a page POINTER (the catalog's handle comes from
575
+ # the local descriptions only) and the release projection nulls it — which is why
576
+ # this allowlist differs from ``operand_synonyms.json``'s (a synonyms row
577
+ # never carries it). The allowlist is schema-CLOSED: an unknown field fails closed.
578
+ if semantics_provided:
579
+ _check_semantics_rows(semantics)
580
+ stale_keys = set(semantics) - code_set
581
+ _require(not stale_keys, (
582
+ f"operand_semantics.json carries {len(stale_keys)} key(s) not in the "
583
+ f"inventory code set: {sorted(stale_keys)[:5]}"
584
+ ))
585
+ missing_sem = code_set - set(semantics)
586
+ _require(not missing_sem, (
587
+ f"operand_semantics.json is SHORT — missing {len(missing_sem)} "
588
+ f"inventory code(s): {sorted(missing_sem)[:5]} (broken projection)"
589
+ ))
590
+
591
+ expected_codes = set(synonyms)
592
+ per_code, report = merge_enrichment(synonyms, local_descriptions, expected_codes)
593
+
594
+ # a hardening finding: stamp a machine-readable semantics-presence
595
+ # signal into build_report so a downstream caller that writes THIS returned dict
596
+ # directly (a fixture / library primitive) has a signal that the sign columns are
597
+ # null. ``build_catalog_json`` is a library primitive; ``main()`` is the SOLE
598
+ # publish gate that raises on synonyms/semantics-absent — no production path
599
+ # writes this primitive's return value directly.
600
+ report["semantics_state"] = "present" if semantics_provided else "absent"
601
+
602
+ cell_layout_section = probe.get("cell_layout", {})
603
+ covered = set(cell_layout_section.get("covered", []))
604
+ operands_probe = cell_layout_section.get("operands", {})
605
+
606
+ rows = []
607
+ for code in codes:
608
+ if code in covered:
609
+ entry = operands_probe.get(code, {})
610
+ cell_layout = _param_cells_for(entry)
611
+ row_type_name = entry.get("row_type_name")
612
+ else:
613
+ cell_layout = None
614
+ row_type_name = None
615
+
616
+ enrichment = per_code.get(code)
617
+ if enrichment is not None:
618
+ # Field-disjoint merge (§2.2). ``synonyms``/``units`` always ride (the
619
+ # code is in the synonyms keyset); ``description``/``citation`` ride ONLY
620
+ # when the verbatim-local extract covered the code.
621
+ synonyms_val = enrichment.get("synonyms")
622
+ units = enrichment.get("units")
623
+ units_source = enrichment.get("units_source")
624
+ description = enrichment.get("description")
625
+ citation_handle = enrichment.get("citation_handle")
626
+ if description is not None:
627
+ description_source = enrichment.get(
628
+ "description_source", "manual_verbatim"
629
+ )
630
+ description_pending = 0
631
+ else:
632
+ # Synonyms-only row: synonyms present, description pending.
633
+ description_source = "authored"
634
+ description_pending = 1
635
+ else:
636
+ synonyms_val = None
637
+ units = None
638
+ units_source = None
639
+ description = None
640
+ citation_handle = None
641
+ description_source = "authored"
642
+ description_pending = 1
643
+
644
+ # Semantics §4: deterministic sign-convention merge from operand_semantics.json
645
+ # (suffix/family/overlay). Absent code -> None (honest unknown).
646
+ sem = semantics.get(code, {})
647
+ sign_convention = sem.get("sign_convention")
648
+ sign_convention_source = sem.get("sign_convention_source")
649
+
650
+ rows.append(
651
+ {
652
+ "code": code,
653
+ "description": description,
654
+ "description_source": description_source,
655
+ "description_pending": description_pending,
656
+ "citation_handle": citation_handle,
657
+ "units": units,
658
+ "units_source": units_source,
659
+ "sign_convention": sign_convention,
660
+ "sign_convention_source": sign_convention_source,
661
+ "synonyms": synonyms_val,
662
+ "cell_layout": cell_layout,
663
+ "row_type_name": row_type_name,
664
+ "optic_studio_version": osv,
665
+ "schema_version": SCHEMA_VERSION,
666
+ # Tolerance cycle (schema 2->3): the 3 tolerance-safety columns. NULL for
667
+ # every merit row (real values only on the separate tolerance_operand
668
+ # catalog). Carried explicitly so the committed JSON matches its own
669
+ # schema_version=3 row shape and a schema-3-aware re-dump is byte-stable.
670
+ "category": None,
671
+ "precondition_class": None,
672
+ "run_verdict": None,
673
+ }
674
+ )
675
+
676
+ return {
677
+ "schema_version": SCHEMA_VERSION,
678
+ "optic_studio_version": osv,
679
+ "total_members": total_members,
680
+ "build_report": report,
681
+ "rows": rows,
682
+ }
683
+
684
+
685
+ def assert_provenance_invariants(catalog):
686
+ """Assert the §4 provenance invariants over a catalog dict.
687
+
688
+ - every row's ``description_source`` is set AND in the allowed set;
689
+ - NO row is tagged ``description_source='live_label'`` (Step 1 proved no
690
+ engine label exists — it can never be load-bearing for a meaning);
691
+ - a row whose ``description_source != 'live_label'`` and that HAS a non-null
692
+ description must carry a non-empty ``citation_handle`` (for this increment
693
+ descriptions are null so this is vacuously satisfied — but the invariant is
694
+ ASSERTED, not assumed);
695
+ - ``units_source`` is never ``live_label``.
696
+
697
+ Raises ``AssertionError`` on any violation. Used by the build and the tests.
698
+ """
699
+ for row in catalog["rows"]:
700
+ code = row["code"]
701
+ src = row["description_source"]
702
+ _require(src in ALLOWED_DESCRIPTION_SOURCES, f"{code}: bad description_source {src!r}")
703
+ _require(src != "live_label", f"{code}: no row may be tagged live_label (Step 1)")
704
+ if src != "live_label" and row["description"] is not None:
705
+ _require(row["citation_handle"], (
706
+ f"{code}: non-live description requires a citation_handle"
707
+ ))
708
+ usrc = row["units_source"]
709
+ _require(usrc in (None,) + ALLOWED_UNITS_SOURCES, f"{code}: bad units_source {usrc!r}")
710
+ _require(usrc != "live_label", f"{code}: units_source may never be live_label")
711
+ # Semantics §7: sign_convention_source ∈ {None, suffix, family, manual},
712
+ # NEVER live_label (no live sign attribute exists — probe resolve-negative).
713
+ ssrc = row.get("sign_convention_source")
714
+ _require(ssrc in (None,) + ALLOWED_SIGN_SOURCES, (
715
+ f"{code}: bad sign_convention_source {ssrc!r}"
716
+ ))
717
+ _require(ssrc != "live_label", f"{code}: sign_convention_source may never be live_label")
718
+
719
+
720
+ def assert_semantics_invariants(catalog, oracle_path=None):
721
+ """Assert the semantics §7 build-time guards over a catalog dict.
722
+
723
+ Guards (fail the BUILD, not just CI, so the committed catalog can never
724
+ regenerate malformed):
725
+
726
+ 1. **Enum closure** — every non-null ``units`` ∈ §3 vocab; every non-null
727
+ ``sign_convention`` ∈ §4 enum; every ``sign_convention_source`` ∈ §4 source
728
+ set; none ``live_label``.
729
+ 2. **Sourceless-direction forbidden** — a directional sign
730
+ (``ge``/``le``/``equality``/``minimize``/``maximize``) ⇒
731
+ ``sign_convention_source`` non-null. ``measurement``/``null`` may be sourceless.
732
+ 2b. **Source⇔sign compatibility** — each source carries only the signs its
733
+ rule produces (suffix→{ge,le,equality}; family→{ge,le};
734
+ manual→{minimize,maximize}); a measurement/null sign must be sourceless.
735
+ Rejects a mistagged direction (``{boundary_ge, manual}``).
736
+ 3. **Orphan units** — ``units != null`` ⇒ ``units_source ∈ {chm, authored}``
737
+ (an authored unit can never be sourceless).
738
+ 4. **Suffix↔oracle no-contradiction** — no ``*GT`` whose oracle prose says
739
+ "less than" (and not "greater than"); symmetric for ``*LT``. Catches a
740
+ mis-captured oracle. SKIPPED gracefully when the oracle is absent (mirrors
741
+ the verbatim-overlap test's optional-oracle idiom).
742
+ 5. **Semantics-key membership** — every code carrying a non-null
743
+ ``sign_convention`` OR ``sign_convention_source`` is a known catalog code
744
+ (no stale entry survives after a rename). The catalog is the code set; this
745
+ holds by construction here, but the assert pins it.
746
+
747
+ Raises ``AssertionError`` on any violation.
748
+ """
749
+ catalog_codes = {row["code"] for row in catalog["rows"]}
750
+
751
+ # Guard 4 oracle (optional): load only if a path is supplied and the file
752
+ # exists, like the verbatim test. Absent -> guard 4 is skipped, not failed.
753
+ oracle = None
754
+ if oracle_path and os.path.isfile(oracle_path):
755
+ oracle = _load_json(oracle_path)
756
+
757
+ for row in catalog["rows"]:
758
+ code = row["code"]
759
+ # --- Guard 1: enum closure -------------------------------------------
760
+ units = row.get("units")
761
+ _require(units is None or units in ALLOWED_UNITS, (
762
+ f"{code}: units {units!r} not in the §3 closed vocab"
763
+ ))
764
+ sign = row.get("sign_convention")
765
+ _require(sign is None or sign in ALLOWED_SIGN_CONVENTIONS, (
766
+ f"{code}: sign_convention {sign!r} not in the §4 enum"
767
+ ))
768
+ ssrc = row.get("sign_convention_source")
769
+ _require(ssrc in (None,) + ALLOWED_SIGN_SOURCES, (
770
+ f"{code}: sign_convention_source {ssrc!r} not in the §4 source set"
771
+ ))
772
+ _require(ssrc != "live_label", f"{code}: sign_convention_source live_label forbidden")
773
+
774
+ # --- Guard 2: sourceless direction forbidden -------------------------
775
+ if sign in _DIRECTIONAL_SIGNS:
776
+ _require(ssrc is not None, (
777
+ f"{code}: directional sign {sign!r} requires a non-null "
778
+ f"sign_convention_source"
779
+ ))
780
+
781
+ # --- Guard 2b: source⇔sign COMPATIBILITY table -----------------------
782
+ # A non-null source may only carry the signs its rule can produce
783
+ # (suffix→{ge,le,equality}; family→{ge,le}; manual→{minimize,maximize}).
784
+ # A {boundary_ge, manual} or {measurement, suffix} row is a mistagged
785
+ # direction → raise. Conversely ``measurement``/null sign MUST be
786
+ # sourceless (no provenance for "no inherent direction").
787
+ if ssrc is not None:
788
+ allowed = SIGN_SOURCE_COMPATIBILITY[ssrc]
789
+ _require(sign in allowed, (
790
+ f"{code}: sign {sign!r} incompatible with source {ssrc!r} "
791
+ f"(allowed: {sorted(allowed)})"
792
+ ))
793
+ else:
794
+ _require(sign in (None, "measurement"), (
795
+ f"{code}: sign {sign!r} requires a non-null "
796
+ f"sign_convention_source (only measurement/null may be sourceless)"
797
+ ))
798
+
799
+ # --- Guard 3: orphan units -------------------------------------------
800
+ if units is not None:
801
+ _require(row.get("units_source") in ALLOWED_UNITS_SOURCES, (
802
+ f"{code}: units {units!r} present but units_source "
803
+ f"{row.get('units_source')!r} not in {ALLOWED_UNITS_SOURCES}"
804
+ ))
805
+
806
+ # --- Guard 4: suffix↔oracle no-contradiction (oracle-gated) ----------
807
+ # §7.4 catches a MIS-CAPTURED oracle: a *GT whose prose documents the
808
+ # OPPOSITE boundary direction. We test the BOUNDARY-TARGET phrasing
809
+ # ("less than the target/specified...") rather than any incidental
810
+ # "less than", so a non-boundary operand whose suffix is mechanically
811
+ # GT/LT (e.g. LOGT = log base 10, prose "...less than or equal to zero")
812
+ # is NOT a false positive: incidental "less than" is not a boundary claim.
813
+ # Real boundary operands phrase the direction against the target/spec, so
814
+ # a genuine mis-capture (a *GT documented as "less than the target") still
815
+ # trips this. ``_oracle_boundary_dir`` returns 'ge'/'le'/None.
816
+ if oracle is not None and code not in _SUFFIX_COLLISIONS:
817
+ entry = oracle.get(code)
818
+ if entry is not None:
819
+ direction = _oracle_boundary_dir(entry.get("raw_text"))
820
+ if code[-2:] == "GT":
821
+ _require(direction != "le", (
822
+ f"{code}: *GT but oracle documents a 'less than the "
823
+ f"target' boundary (mis-captured oracle)"
824
+ ))
825
+ if code[-2:] == "LT":
826
+ _require(direction != "ge", (
827
+ f"{code}: *LT but oracle documents a 'greater than the "
828
+ f"target' boundary (mis-captured oracle)"
829
+ ))
830
+
831
+ # --- Guard 5: semantics-key membership -------------------------------
832
+ if sign is not None or ssrc is not None:
833
+ _require(code in catalog_codes, (
834
+ f"{code}: semantics for a non-catalog code (stale entry)"
835
+ ))
836
+
837
+
838
+ def write_catalog_json(out_path, catalog):
839
+ """Write ``catalog`` as normalized-LF, indent=2 JSON (committed TEXT).
840
+
841
+ ``newline="\n"`` forces LF on every platform so the committed artifact is
842
+ byte-stable and the reproducibility hash is over THIS file (§2). Creates the
843
+ parent dir if absent.
844
+
845
+ ATOMIC (a hardening finding): the JSON is written to a same-dir
846
+ ``<out_path>.tmp`` then ``os.replace``d onto the final path (atomic on Windows +
847
+ POSIX), so a concurrent reader (e.g. an xdist worker opening the catalog) never
848
+ sees a half-written file — it reads either the OLD complete file or the NEW
849
+ complete file, never a partial one. A failed write leaves no ``.tmp`` behind on
850
+ the success path; on an exception mid-write the ``.tmp`` is cleaned up.
851
+ """
852
+ text = json.dumps(catalog, indent=2, ensure_ascii=True)
853
+ _atomic_write_text(out_path, text)
854
+
855
+
856
+ def _atomic_write_text(out_path, text):
857
+ """Write ``text`` + a trailing newline to ``out_path`` ATOMICALLY (D5 / F8).
858
+
859
+ The SHARED atomic-write primitive: the text is written to a same-dir
860
+ ``<out_path>.tmp`` then ``os.replace``d onto the final path (atomic on Windows +
861
+ POSIX). A concurrent reader never sees a half-written file, and a failure DURING
862
+ the write (mid-``fh.write``) leaves the ORIGINAL ``out_path`` byte-intact (the
863
+ ``.tmp`` is discarded) — the guarantee a direct ``open(out_path, "w")`` write
864
+ cannot make. Reused by ``write_catalog_json`` AND the release-semantics projection
865
+ writer (``project_release_semantics.main``) so both share ONE proven atomic path.
866
+ Creates the parent dir if absent; normalized-LF (``newline="\n"``).
867
+ """
868
+ os.makedirs(os.path.dirname(out_path), exist_ok=True)
869
+ tmp_path = out_path + ".tmp"
870
+ try:
871
+ with open(tmp_path, "w", encoding="utf-8", newline="\n") as fh:
872
+ fh.write(text)
873
+ fh.write("\n")
874
+ os.replace(tmp_path, out_path) # atomic same-dir rename
875
+ except Exception:
876
+ # Never leave a stray partial ``.tmp`` on a failed write.
877
+ try:
878
+ os.remove(tmp_path)
879
+ except OSError:
880
+ pass
881
+ raise
882
+
883
+
884
+ _CREATE_OPERAND = """
885
+ CREATE TABLE operand (
886
+ code TEXT PRIMARY KEY,
887
+ description TEXT,
888
+ description_source TEXT NOT NULL,
889
+ description_pending INTEGER,
890
+ citation_handle TEXT,
891
+ units TEXT,
892
+ units_source TEXT,
893
+ sign_convention TEXT,
894
+ sign_convention_source TEXT,
895
+ cell_layout TEXT,
896
+ row_type_name TEXT,
897
+ optic_studio_version TEXT,
898
+ schema_version INTEGER,
899
+ category TEXT,
900
+ precondition_class TEXT,
901
+ run_verdict TEXT,
902
+ body TEXT
903
+ )
904
+ """
905
+
906
+ # external-content FTS5: the content table (``operand``) must expose EVERY FTS
907
+ # column by name (``code``, ``body``) or a query/rebuild raises
908
+ # ``no such column: T.body``. ``body`` is therefore a real (derived) storage
909
+ # column on ``operand`` — it is NOT a catalog-row schema field (§2 rows are
910
+ # unchanged); it is the searchable text FTS5 indexes. ``body`` is rebuilt from
911
+ # the content table via the FTS5 ``'rebuild'`` command after the inserts.
912
+ _CREATE_FTS = """
913
+ CREATE VIRTUAL TABLE operand_fts
914
+ USING fts5(code, body, content='operand', content_rowid='rowid')
915
+ """
916
+
917
+
918
+ def _validate_catalog_shape(catalog, catalog_json_path):
919
+ """Validate a loaded catalog's SHAPE before ``build_db`` touches it (F2).
920
+
921
+ Raises ``ValueError`` (a member of the server degrade set) on a present-but-
922
+ WRONG-SHAPE or ZERO-ROW catalog so ``_safe_open_catalog`` degrades it to
923
+ ``operand_catalog_unavailable`` instead of letting ``build_db`` crash the
924
+ dispatcher's ``__init__`` with a raw ``KeyError``/``TypeError``/``AttributeError``
925
+ (finding 10 was BYTE-shaped only; a valid-JSON wrong-SHAPE — ``{}`` / ``[]`` /
926
+ ``{"schema_version":3}`` / ``{"rows":"str"}`` — slipped past it). A ZERO-ROW
927
+ (``{"rows":[]}``) catalog is a corruption too (existence ≠ readiness): a zero-row
928
+ connection would answer ``operand_unknown``, masking an unbuilt catalog — so it
929
+ degrades to unavailable, NOT a silently-empty catalog.
930
+ """
931
+ if not isinstance(catalog, dict):
932
+ raise ValueError(
933
+ f"{catalog_json_path}: catalog root is not a JSON object "
934
+ f"({type(catalog).__name__}) — degrading"
935
+ )
936
+ rows = catalog.get("rows")
937
+ if not isinstance(rows, list):
938
+ raise ValueError(
939
+ f"{catalog_json_path}: catalog 'rows' is not a list "
940
+ f"({type(rows).__name__}) — degrading"
941
+ )
942
+ if len(rows) == 0:
943
+ raise ValueError(
944
+ f"{catalog_json_path}: catalog has ZERO rows (existence != readiness) "
945
+ "— degrading to operand_catalog_unavailable"
946
+ )
947
+
948
+
949
+ def build_db(catalog_json_path, conn):
950
+ """Build the keyed ``operand`` table + external-content FTS5 from the JSON.
951
+
952
+ Creates the schema (§2), inserts every row (the SQL ``code TEXT PRIMARY KEY``
953
+ raises ``IntegrityError`` on a duplicate — the storage-layer dedupe), then
954
+ rebuilds ``operand_fts`` from the content table. The FTS ``body`` is
955
+ ``code + ' ' + (synonyms or '')`` — the DESCRIPTION is DELIBERATELY EXCLUDED
956
+ from the searchable body (the verbatim-local build amendment-1 A1). This makes the enriched and
957
+ the fresh-install (synonyms-only) states share ONE ranking corpus
958
+ (``code + synonyms``) by construction — a MATCH ranks identically whether or not
959
+ the user built the verbatim oracle (the state-parity win), and a whitespace-free
960
+ glyph-run in a verbatim description can never reach the tokenized index. The
961
+ ``description`` STAYS stored in the ``operand.description`` column and is returned
962
+ verbatim by ``lookup_operand`` (its DISPLAY value is untouched — only its ranking
963
+ contribution is removed).
964
+
965
+ ``cell_layout`` is stored as a JSON string (or NULL). Returns ``conn``.
966
+ """
967
+ catalog = _load_json(catalog_json_path)
968
+ # F2: validate SHAPE + non-zero rows BEFORE touching the DB, so a present-but-
969
+ # wrong-shape / zero-row catalog raises a degrade-classified ValueError (caught by
970
+ # _safe_open_catalog) instead of a raw KeyError/TypeError crashing __init__.
971
+ _validate_catalog_shape(catalog, catalog_json_path)
972
+ conn.execute(_CREATE_OPERAND)
973
+ conn.execute(_CREATE_FTS)
974
+
975
+ for row in catalog["rows"]:
976
+ cell_layout = row.get("cell_layout")
977
+ cell_layout_json = (
978
+ json.dumps(cell_layout) if cell_layout is not None else None
979
+ )
980
+ description = row.get("description")
981
+ # Coerce synonyms to a clean string BEFORE the body format so a
982
+ # list/tuple-valued entry is space-joined (not rendered as a Python repr
983
+ # that would pollute the FTS index with brackets/quotes/commas). A string
984
+ # synonyms passes through byte-identical to the prior `synonyms or ""`.
985
+ synonyms_text = _synonyms_to_text(row.get("synonyms"))
986
+ # the verbatim-local build amendment-1 A1: the searchable FTS body is CODE + SYNONYMS ONLY.
987
+ # The description is DELIBERATELY NOT in the body (it stays in the stored
988
+ # `description` column below + is returned by lookup_operand). Excluding it
989
+ # makes the enriched and synonyms-only builds share ONE ranking corpus, so
990
+ # the ranking is identical regardless of whether the verbatim oracle was
991
+ # built, and a glyph-run description artifact never becomes a search token.
992
+ body = "{} {}".format(row["code"], synonyms_text).strip()
993
+ conn.execute(
994
+ "INSERT INTO operand (code, description, description_source, "
995
+ "description_pending, citation_handle, units, units_source, "
996
+ "sign_convention, sign_convention_source, cell_layout, row_type_name, "
997
+ "optic_studio_version, schema_version, category, precondition_class, "
998
+ "run_verdict, body) "
999
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1000
+ (
1001
+ row["code"],
1002
+ description,
1003
+ row["description_source"],
1004
+ row.get("description_pending"),
1005
+ row.get("citation_handle"),
1006
+ row.get("units"),
1007
+ row.get("units_source"),
1008
+ row.get("sign_convention"),
1009
+ row.get("sign_convention_source"),
1010
+ cell_layout_json,
1011
+ row.get("row_type_name"),
1012
+ row.get("optic_studio_version"),
1013
+ row.get("schema_version"),
1014
+ # Tolerance cycle: the 3 tolerance-safety columns. NULL for every merit row
1015
+ # (operand_synonyms.json / operand_semantics.json never supply
1016
+ # them) — real values only on the separate tolerance_operand catalog.
1017
+ row.get("category"),
1018
+ row.get("precondition_class"),
1019
+ row.get("run_verdict"),
1020
+ body,
1021
+ ),
1022
+ )
1023
+ # Rebuild the external-content FTS index from the content table (`operand`):
1024
+ # FTS5 reads `code` + `body` per rowid. This is the locked "rebuild
1025
+ # operand_fts from content" step — one rebuild, not per-row mirroring.
1026
+ conn.execute("INSERT INTO operand_fts(operand_fts) VALUES('rebuild')")
1027
+ conn.commit()
1028
+ return conn
1029
+
1030
+
1031
+ def open_catalog(db_path_or_memory=":memory:", catalog_json_path=None):
1032
+ """Open a catalog connection, building the schema from the committed JSON.
1033
+
1034
+ The DB is ALWAYS built from the committed normalized-LF JSON (the binary
1035
+ ``.db`` is never committed — §2). The JSON path resolves via a
1036
+ package-relative anchor (NOT cwd-relative — the agent cwd resets between
1037
+ calls, §5). ``db_path_or_memory`` defaults to an in-memory DB.
1038
+
1039
+ Returns a ``sqlite3.Connection``. The caller owns closing it.
1040
+ """
1041
+ if catalog_json_path is None:
1042
+ catalog_json_path = CATALOG_JSON_PATH
1043
+ # check_same_thread=False: the catalog connection is READ-ONLY at runtime and
1044
+ # may be used from an async MCP worker thread (a different thread than the one
1045
+ # that opened it). sqlite3 otherwise raises ProgrammingError on cross-thread
1046
+ # use. Safe here because the runtime path issues no writes (H-2).
1047
+ conn = sqlite3.connect(db_path_or_memory, check_same_thread=False)
1048
+ build_db(catalog_json_path, conn)
1049
+ return conn
1050
+
1051
+
1052
+ def main():
1053
+ """Build the (gitignored, user-built) catalog JSON — fail-loud (§3).
1054
+
1055
+ Returns an exit code: 0 for ``enriched``/``synonyms_only`` (the ONE exit-0
1056
+ degraded state, LOUDLY marked), nonzero for ``partial`` (a REAL drop). RAISES
1057
+ on synonyms-absent (committed -> broken checkout) or semantics-absent (committed
1058
+ -> broken/projection error) — see the §3 matrix.
1059
+ """
1060
+ captures = _paths.captures_dir()
1061
+ inventory_path = os.path.join(captures, "operand_inventory_438.json")
1062
+ probe_path = os.path.join(captures, "probe_operands_capture.json")
1063
+
1064
+ # S committed -> a missing synonyms file is a BROKEN checkout: HARD ERROR.
1065
+ if not os.path.isfile(SYNONYMS_JSON_PATH):
1066
+ raise FileNotFoundError(
1067
+ f"operand_synonyms.json is committed but ABSENT ({SYNONYMS_JSON_PATH}) "
1068
+ "— broken checkout"
1069
+ )
1070
+ # M committed -> a missing semantics file is a BROKEN checkout / projection
1071
+ # error: HARD ERROR.
1072
+ if not os.path.isfile(SEMANTICS_JSON_PATH):
1073
+ raise FileNotFoundError(
1074
+ f"operand_semantics.json is committed but ABSENT ({SEMANTICS_JSON_PATH}) "
1075
+ "— broken checkout / projection error"
1076
+ )
1077
+
1078
+ # The gitignored verbatim-local descriptions. Forced OFF by
1079
+ # OPTIVIBE_NO_LOCAL_DESCRIPTIONS=1 (the fresh-install reproduction, §2.8/§6);
1080
+ # otherwise merged when the raw oracle is present, synonyms-only when absent.
1081
+ if os.environ.get("OPTIVIBE_NO_LOCAL_DESCRIPTIONS"):
1082
+ local_descriptions_path = None
1083
+ elif os.path.isfile(RAW_DESCRIPTIONS_PATH):
1084
+ local_descriptions_path = RAW_DESCRIPTIONS_PATH
1085
+ else:
1086
+ local_descriptions_path = None
1087
+
1088
+ catalog = build_catalog_json(
1089
+ inventory_path,
1090
+ probe_path,
1091
+ synonyms_path=SYNONYMS_JSON_PATH,
1092
+ local_descriptions_path=local_descriptions_path,
1093
+ semantics_path=SEMANTICS_JSON_PATH,
1094
+ )
1095
+ # §7 guards fail the BUILD so the catalog can never regenerate malformed. The
1096
+ # semantics guard accepts the oracle path for guard 4 and skips gracefully if
1097
+ # the oracle (gitignored) is absent.
1098
+ assert_provenance_invariants(catalog)
1099
+ assert_semantics_invariants(catalog, oracle_path=RAW_DESCRIPTIONS_PATH)
1100
+
1101
+ report = catalog["build_report"]
1102
+ state = report["descriptions_state"]
1103
+
1104
+ # PARTIAL is a REAL drop: REFUSE-and-report (do NOT write a lossy catalog);
1105
+ # name the missing codes; exit nonzero.
1106
+ if state == "partial":
1107
+ print(
1108
+ "WARN: operand catalog build is PARTIAL — {}/{} descriptions merged, "
1109
+ "missing {}: {}".format(
1110
+ report["enriched"], report["expected"],
1111
+ report["expected"] - report["enriched"], report["missing"],
1112
+ ),
1113
+ file=sys.stderr,
1114
+ )
1115
+ print(
1116
+ "REFUSED: not writing a lossy catalog; run "
1117
+ "scripts/build_manual_corpus.py --emit-raw (see PROVENANCE.md)",
1118
+ file=sys.stderr,
1119
+ )
1120
+ return 2
1121
+
1122
+ write_catalog_json(CATALOG_JSON_PATH, catalog)
1123
+ print("wrote {} rows -> {}".format(len(catalog["rows"]), CATALOG_JSON_PATH))
1124
+ print("build_report: {}".format(json.dumps(report)))
1125
+
1126
+ # SYNONYMS_ONLY is the ONE exit-0 degraded state — LOUDLY marked.
1127
+ if state == "synonyms_only":
1128
+ print(
1129
+ "WARN: operand catalog built SYNONYMS-ONLY "
1130
+ "(descriptions_state=synonyms_only) — every description_pending=1; run "
1131
+ "scripts/build_manual_corpus.py --emit-raw to enrich (see PROVENANCE.md)",
1132
+ file=sys.stderr,
1133
+ )
1134
+ return 0
1135
+
1136
+
1137
+ if __name__ == "__main__":
1138
+ raise SystemExit(main())