tablassert 19.5.0__tar.gz → 19.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {tablassert-19.5.0 → tablassert-19.6.0}/PKG-INFO +49 -16
  2. {tablassert-19.5.0 → tablassert-19.6.0}/README.md +47 -14
  3. {tablassert-19.5.0 → tablassert-19.6.0}/pyproject.toml +2 -2
  4. {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/fullmap.rs +5 -5
  5. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/biolink.py +209 -72
  6. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/cli.py +65 -20
  7. {tablassert-19.5.0 → tablassert-19.6.0}/LICENSE +0 -0
  8. {tablassert-19.5.0 → tablassert-19.6.0}/rust/Cargo.lock +0 -0
  9. {tablassert-19.5.0 → tablassert-19.6.0}/rust/Cargo.toml +0 -0
  10. {tablassert-19.5.0 → tablassert-19.6.0}/rust/examples/count_tables.rs +0 -0
  11. {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/json.rs +0 -0
  12. {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/lib.rs +0 -0
  13. {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/ndjson.rs +0 -0
  14. {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/nlp.rs +0 -0
  15. {tablassert-19.5.0 → tablassert-19.6.0}/rust/src/uuid.rs +0 -0
  16. {tablassert-19.5.0 → tablassert-19.6.0}/rust/tests/build_golden.rs +0 -0
  17. {tablassert-19.5.0 → tablassert-19.6.0}/rust/tests/common/mod.rs +0 -0
  18. {tablassert-19.5.0 → tablassert-19.6.0}/rust/tests/extract_prebuilt.rs +0 -0
  19. {tablassert-19.5.0 → tablassert-19.6.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
  20. {tablassert-19.5.0 → tablassert-19.6.0}/rust/tests/nlp_golden.rs +0 -0
  21. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/__init__.py +0 -0
  22. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/_lazy.py +0 -0
  23. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/agent.py +0 -0
  24. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/cli_entry.py +0 -0
  25. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/coerce.py +0 -0
  26. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
  27. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/distill.py +0 -0
  28. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/distill_reward.py +0 -0
  29. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/enums.py +0 -0
  30. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/errors.py +0 -0
  31. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/extras.py +0 -0
  32. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/fullmap.py +0 -0
  33. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/graph_target.py +0 -0
  34. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/ingests.py +0 -0
  35. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/lib.py +0 -0
  36. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/log.py +0 -0
  37. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/models.py +0 -0
  38. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/net.py +0 -0
  39. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/nlp.py +0 -0
  40. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/progress.py +0 -0
  41. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/qc.py +0 -0
  42. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/rig.py +0 -0
  43. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/rs.pyi +0 -0
  44. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/runcache.py +0 -0
  45. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/study.py +0 -0
  46. {tablassert-19.5.0 → tablassert-19.6.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 19.5.0
3
+ Version: 19.6.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -22,7 +22,7 @@ Classifier: Framework :: Pydantic :: 2
22
22
  Classifier: Operating System :: POSIX :: Linux
23
23
  Classifier: Operating System :: MacOS :: MacOS X
24
24
  Classifier: Environment :: Console
25
- Requires-Dist: biolink-model>=4.4.4
25
+ Requires-Dist: biolink-model>=4.4.5
26
26
  Requires-Dist: polars>=2.0.0rc2
27
27
  Requires-Dist: pyarrow>=21.0.0
28
28
  Requires-Dist: rapidfuzz>=3.14.3
@@ -77,6 +77,36 @@ KGX-compliant nodes and edges.
77
77
  **[Full Documentation](https://skyeav.github.io/Tablassert/)**: installation guides, tutorial,
78
78
  configuration reference, and API docs.
79
79
 
80
+ ## Statement of need
81
+
82
+ Biomedical knowledge lives in spreadsheets: association tables, assay results, curated gene-disease
83
+ lists. Getting those rows into an NCATS Translator knowledge graph means writing an ingest that maps
84
+ columns to Biolink statements, resolves free text ("TP53", "lung cancer") to standard CURIEs, and
85
+ records provenance, terms of use, and statistical annotations. Today that ingest is Python code
86
+ maintained per source, and entity resolution usually means calling a hosted name-resolution service
87
+ at build time.
88
+
89
+ Tablassert replaces the per-source code with a declarative YAML mapping plus an embedded, offline
90
+ entity-resolution database built from RENCI BABEL exports, so a build is reproducible, auditable,
91
+ and network-free. It is aimed at Translator ingest authors and knowledge-graph data engineers who
92
+ hold tabular biomedical sources, and at bioinformatics groups that want a validated KGX output
93
+ without writing a transform pipeline. The configuration model and the Biolink/KGX contracts it
94
+ emits follow current [`NCATSTranslator/translator-ingests`](https://github.com/NCATSTranslator/translator-ingests)
95
+ practice, and each build can emit the Resource Ingest Guide (RIG) metadata those submissions require.
96
+
97
+ ## Getting help
98
+
99
+ - **Questions and bug reports:** open an issue at
100
+ [github.com/SkyeAv/Tablassert/issues](https://github.com/SkyeAv/Tablassert/issues) (use the
101
+ `question` label for usage questions). Include the command you ran, its output, and your
102
+ `tablassert --version`.
103
+ - **Contributing code or docs:** see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, the local quality
104
+ gates, and pull-request expectations. By participating in this project you agree to follow the
105
+ [Code of Conduct](CODE_OF_CONDUCT.md).
106
+ - **Reference:** the [CLI Reference](https://skyeav.github.io/Tablassert/cli/) and
107
+ [configuration guides](https://skyeav.github.io/Tablassert/configuration/graph/) document every
108
+ flag and field.
109
+
80
110
  ## Quick Start
81
111
 
82
112
  ```bash
@@ -105,14 +135,17 @@ template:
105
135
  ```
106
136
 
107
137
  Wrap it in a graph config (`graph.yaml`) pointing at your fullmap entity-resolution database
108
- and carrying the required `rig:` metadata for the generated Resource Ingest Guide:
138
+ and carrying the required `rig:` metadata for the generated Resource Ingest Guide. Build or
139
+ download that database once first (`tablassert build-fullmap`, a multi-GB download; see the
140
+ [Fullmap guide](https://skyeav.github.io/Tablassert/fullmap/)), which makes `./fullmap` below the
141
+ right path:
109
142
 
110
143
  ```yaml
111
144
  name: MY_KG
112
145
  version: 1.0.0
113
146
  tables:
114
147
  - ./table.yaml
115
- fullmap: /path/to/fullmap
148
+ fullmap: ./fullmap
116
149
  rig:
117
150
  source_info:
118
151
  infores_id: infores:my-kg
@@ -159,7 +192,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
159
192
  low-confidence mappings
160
193
  - **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
161
194
  and predicates
162
- - **Autonomous agent**: `tablassert agent` derives, builds, and refines configs for whole papers
195
+ - **Autonomous agent (experimental)**: `tablassert agent` derives, builds, and refines configs for
196
+ whole papers
163
197
  - **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
164
198
  development environment
165
199
 
@@ -169,8 +203,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
169
203
  pip install tablassert
170
204
  ```
171
205
 
172
- Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API;
173
- install `[cli]` to use the `tablassert` command and optional extras for additional runtime and pipeline capabilities:
206
+ Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API; `[cli]`
207
+ provides the `tablassert` command. Other extras are opt-in:
174
208
 
175
209
  | Extra | Adds | Install |
176
210
  | ----- | ---- | ------- |
@@ -178,17 +212,16 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
178
212
  | `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
179
213
  | `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
180
214
  | `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
181
- | `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
182
- | `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
183
- | `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
215
+ | `agent` | autonomous agent (smolagents, litellm, article/table context) -- experimental, API may change | `pip install "tablassert[agent]"` |
216
+ | `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) -- experimental, API may change | `pip install "tablassert[optimize]"` |
217
+ | `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) -- experimental, API may change | `pip install "tablassert[distill]"` |
184
218
  | `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
185
219
 
186
- The `tablassert` command requires `[cli]`; without it, the console launcher reports the exact install command. QC is opt-in at build time (`build-kg --qc`). Reaching a feature whose extra is not installed never
187
- produces a bare `ModuleNotFoundError`: the failure names the missing package and the exact install
188
- command, and for `build-kg --qc` and `tablassert agent` it arrives before the run starts rather than
189
- partway through. Logging is the exception: without the `log` extra Tablassert produces no logs
190
- instead of failing. See the
191
- [Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix and the
220
+ Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
221
+ names the missing package and the install command that fixes it. QC is opt-in at build time
222
+ (`build-kg --qc`). See the
223
+ [Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix,
224
+ per-command preflight behavior, and the
192
225
  [CLI Reference](https://skyeav.github.io/Tablassert/cli/) for every flag.
193
226
 
194
227
  ## Entity Resolution API
@@ -16,6 +16,36 @@ KGX-compliant nodes and edges.
16
16
  **[Full Documentation](https://skyeav.github.io/Tablassert/)**: installation guides, tutorial,
17
17
  configuration reference, and API docs.
18
18
 
19
+ ## Statement of need
20
+
21
+ Biomedical knowledge lives in spreadsheets: association tables, assay results, curated gene-disease
22
+ lists. Getting those rows into an NCATS Translator knowledge graph means writing an ingest that maps
23
+ columns to Biolink statements, resolves free text ("TP53", "lung cancer") to standard CURIEs, and
24
+ records provenance, terms of use, and statistical annotations. Today that ingest is Python code
25
+ maintained per source, and entity resolution usually means calling a hosted name-resolution service
26
+ at build time.
27
+
28
+ Tablassert replaces the per-source code with a declarative YAML mapping plus an embedded, offline
29
+ entity-resolution database built from RENCI BABEL exports, so a build is reproducible, auditable,
30
+ and network-free. It is aimed at Translator ingest authors and knowledge-graph data engineers who
31
+ hold tabular biomedical sources, and at bioinformatics groups that want a validated KGX output
32
+ without writing a transform pipeline. The configuration model and the Biolink/KGX contracts it
33
+ emits follow current [`NCATSTranslator/translator-ingests`](https://github.com/NCATSTranslator/translator-ingests)
34
+ practice, and each build can emit the Resource Ingest Guide (RIG) metadata those submissions require.
35
+
36
+ ## Getting help
37
+
38
+ - **Questions and bug reports:** open an issue at
39
+ [github.com/SkyeAv/Tablassert/issues](https://github.com/SkyeAv/Tablassert/issues) (use the
40
+ `question` label for usage questions). Include the command you ran, its output, and your
41
+ `tablassert --version`.
42
+ - **Contributing code or docs:** see [CONTRIBUTING.md](CONTRIBUTING.md) for setup, the local quality
43
+ gates, and pull-request expectations. By participating in this project you agree to follow the
44
+ [Code of Conduct](CODE_OF_CONDUCT.md).
45
+ - **Reference:** the [CLI Reference](https://skyeav.github.io/Tablassert/cli/) and
46
+ [configuration guides](https://skyeav.github.io/Tablassert/configuration/graph/) document every
47
+ flag and field.
48
+
19
49
  ## Quick Start
20
50
 
21
51
  ```bash
@@ -44,14 +74,17 @@ template:
44
74
  ```
45
75
 
46
76
  Wrap it in a graph config (`graph.yaml`) pointing at your fullmap entity-resolution database
47
- and carrying the required `rig:` metadata for the generated Resource Ingest Guide:
77
+ and carrying the required `rig:` metadata for the generated Resource Ingest Guide. Build or
78
+ download that database once first (`tablassert build-fullmap`, a multi-GB download; see the
79
+ [Fullmap guide](https://skyeav.github.io/Tablassert/fullmap/)), which makes `./fullmap` below the
80
+ right path:
48
81
 
49
82
  ```yaml
50
83
  name: MY_KG
51
84
  version: 1.0.0
52
85
  tables:
53
86
  - ./table.yaml
54
- fullmap: /path/to/fullmap
87
+ fullmap: ./fullmap
55
88
  rig:
56
89
  source_info:
57
90
  infores_id: infores:my-kg
@@ -98,7 +131,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
98
131
  low-confidence mappings
99
132
  - **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
100
133
  and predicates
101
- - **Autonomous agent**: `tablassert agent` derives, builds, and refines configs for whole papers
134
+ - **Autonomous agent (experimental)**: `tablassert agent` derives, builds, and refines configs for
135
+ whole papers
102
136
  - **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
103
137
  development environment
104
138
 
@@ -108,8 +142,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
108
142
  pip install tablassert
109
143
  ```
110
144
 
111
- Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API;
112
- install `[cli]` to use the `tablassert` command and optional extras for additional runtime and pipeline capabilities:
145
+ Or with uv: `uv tool install "tablassert[cli]"`. The base install provides the Python API; `[cli]`
146
+ provides the `tablassert` command. Other extras are opt-in:
113
147
 
114
148
  | Extra | Adds | Install |
115
149
  | ----- | ---- | ------- |
@@ -117,17 +151,16 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
117
151
  | `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
118
152
  | `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
119
153
  | `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
120
- | `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
121
- | `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
122
- | `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
154
+ | `agent` | autonomous agent (smolagents, litellm, article/table context) -- experimental, API may change | `pip install "tablassert[agent]"` |
155
+ | `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) -- experimental, API may change | `pip install "tablassert[optimize]"` |
156
+ | `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) -- experimental, API may change | `pip install "tablassert[distill]"` |
123
157
  | `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
124
158
 
125
- The `tablassert` command requires `[cli]`; without it, the console launcher reports the exact install command. QC is opt-in at build time (`build-kg --qc`). Reaching a feature whose extra is not installed never
126
- produces a bare `ModuleNotFoundError`: the failure names the missing package and the exact install
127
- command, and for `build-kg --qc` and `tablassert agent` it arrives before the run starts rather than
128
- partway through. Logging is the exception: without the `log` extra Tablassert produces no logs
129
- instead of failing. See the
130
- [Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix and the
159
+ Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
160
+ names the missing package and the install command that fixes it. QC is opt-in at build time
161
+ (`build-kg --qc`). See the
162
+ [Installation guide](https://skyeav.github.io/Tablassert/installation/) for the full matrix,
163
+ per-command preflight behavior, and the
131
164
  [CLI Reference](https://skyeav.github.io/Tablassert/cli/) for every flag.
132
165
 
133
166
  ## Entity Resolution API
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "19.5.0"
3
+ version = "19.6.0"
4
4
  description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -47,7 +47,7 @@ classifiers = [
47
47
  ]
48
48
  requires-python = ">=3.11"
49
49
  dependencies = [
50
- "biolink-model>=4.4.4",
50
+ "biolink-model>=4.4.5",
51
51
  "polars>=2.0.0rc2",
52
52
  "pyarrow>=21.0.0",
53
53
  "rapidfuzz>=3.14.3",
@@ -2950,11 +2950,11 @@ fn extract_validate_rename(
2950
2950
  archive.display()
2951
2951
  ))
2952
2952
  };
2953
- let database = open_read_only(&primary).map_err(&validation_ctx)?;
2954
- validate_schema(&database).map_err(&validation_ctx)?;
2955
- let build_id = read_build_id(&database).map_err(&validation_ctx)?;
2956
- let shard_count = shard_count_of(&database).map_err(&validation_ctx)?;
2957
- let recorded_allowlist = read_taxon_allowlist_identity(&database).map_err(&validation_ctx)?;
2953
+ let database = open_read_only(&primary).map_err(validation_ctx)?;
2954
+ validate_schema(&database).map_err(validation_ctx)?;
2955
+ let build_id = read_build_id(&database).map_err(validation_ctx)?;
2956
+ let shard_count = shard_count_of(&database).map_err(validation_ctx)?;
2957
+ let recorded_allowlist = read_taxon_allowlist_identity(&database).map_err(validation_ctx)?;
2958
2958
  drop(database);
2959
2959
 
2960
2960
  // A published archive built WITHOUT the required filter is not the database
@@ -46,6 +46,7 @@ from __future__ import annotations
46
46
 
47
47
  import inspect
48
48
  import json
49
+ import os
49
50
  import re
50
51
  from collections import Counter
51
52
  from enum import Enum
@@ -86,6 +87,7 @@ __all__ = [
86
87
  "legal_predicates",
87
88
  "node_class",
88
89
  "numeric_slot_kind",
90
+ "prune_kgx_edges",
89
91
  "resolve_association_class",
90
92
  "resolve_node_category",
91
93
  "resolve_node_class",
@@ -591,28 +593,8 @@ release from making ``species_context_qualifier`` silently emittable again.
591
593
 
592
594
 
593
595
  CLASS_FIELD_OVERRIDES: dict[str, frozenset[str]] = {
594
- "EntityToDiseaseAssociation": frozenset(
595
- {
596
- "anatomical_context_qualifier",
597
- "disease_context_qualifier",
598
- "frequency_qualifier",
599
- "population_context_qualifier",
600
- "regulatory_approvals",
601
- "sex_qualifier",
602
- "temporal_context_qualifier",
603
- }
604
- ),
605
- "EntityToPhenotypicFeatureAssociation": frozenset(
606
- {
607
- "anatomical_context_qualifier",
608
- "disease_context_qualifier",
609
- "frequency_qualifier",
610
- "population_context_qualifier",
611
- "regulatory_approvals",
612
- "sex_qualifier",
613
- "temporal_context_qualifier",
614
- }
615
- ),
596
+ "EntityToDiseaseAssociation": frozenset({"population_context_qualifier", "sex_qualifier", "temporal_context_qualifier"}),
597
+ "EntityToPhenotypicFeatureAssociation": frozenset({"population_context_qualifier", "temporal_context_qualifier"}),
616
598
  }
617
599
  """Per-class grants of edge fields the resolved association class does not declare.
618
600
 
@@ -620,22 +602,21 @@ Keys are bare association class names (``association_class(cat).__name__``), val
620
602
  the slots ``lib.prune_to_class`` keeps on rows resolved to that class even though the
621
603
  installed model attaches them elsewhere.
622
604
 
623
- The motivating case is a DAKP contraindication edge: ``regulatory_approvals`` is a
624
- canonical slot not yet attached by the installed model, while
625
- ``disease_context_qualifier`` is declared only on the
626
- ``ChemicalEntityToDiseaseOrPhenotypicFeatureAssociation`` lineage -- so one edge can
627
- natively carry both only through these explicit class-scoped grants. The same gap
628
- covers DAKP's sparse qualifier stack (``anatomical_context_qualifier``,
629
- ``sex_qualifier``, ``population_context_qualifier``, ``frequency_qualifier``,
630
- ``temporal_context_qualifier``): each is a satisfiable Biolink qualifier slot that no
631
- pinned class declares, so without a grant ``prune_to_class`` would null it off the
632
- edge. Deliberately excluded: ``species_context_qualifier`` (in
633
- ``DISABLED_EDGE_FIELDS`` -- never emittable), ``temporal_interval_qualifier`` and
634
- ``severity_qualifier`` (in ``UNSATISFIABLE_EDGE_FIELDS`` -- attached to no class, so
635
- a grant could never validate). Tablassert
636
- deliberately emits the granted fields on the pinned classes ahead of the pinned model
605
+ The motivating case is a DAKP contraindication edge carrying the sparse qualifier
606
+ stack ahead of the pinned model: ``population_context_qualifier`` and
607
+ ``temporal_context_qualifier`` are satisfiable qualifier slots the pinned classes do
608
+ not declare, and ``sex_qualifier`` is declared only on the phenotypic-feature side of
609
+ the family -- so without a grant ``prune_to_class`` would null them off the edge.
610
+ Upstream Biolink has absorbed most of the original stack (the biolink-model 4.4.5
611
+ mixin consolidation attached ``disease_context_qualifier``,
612
+ ``anatomical_context_qualifier``, and ``frequency_qualifier`` to the whole
613
+ disease/phenotype family, renamed ``FDA_regulatory_approvals`` to the canonical
614
+ ``regulatory_approvals`` on the pinned classes, and added ``sex_qualifier`` to
615
+ ``EntityToPhenotypicFeatureAssociation``); the tripwire removed those grants the day
616
+ the release landed, and the remaining ones stay deliberately ahead of the model
637
617
  (pending an upstream Biolink widening). ``_validation_record`` strips granted fields
638
- before record validation so the deliberate gap is not reported as ``extra_forbidden``.
618
+ before record validation so the deliberate gap is not reported as
619
+ ``extra_forbidden``.
639
620
 
640
621
  A tripwire test asserts every granted field is still absent from its class: the moment
641
622
  a biolink-model release attaches the slot, the suite fails and the stale grant is
@@ -965,45 +946,201 @@ def validate_kgx(nodes_path: Path, edges_path: Path, limit: int = 20) -> dict[st
965
946
  """
966
947
  report: dict[str, Any] = {"biolink_version": BIOLINK_VERSION, "ok": True, "ok_excluding_pending": True}
967
948
  for label, path, edge in (("nodes", nodes_path, False), ("edges", edges_path, True)):
968
- total: int = 0
969
- valid: int = 0
970
- valid_excluding_pending: int = 0
971
- problems: Counter[str] = Counter()
972
- examples: list[dict[str, Any]] = []
973
- # A missing path must never read as a clean bill of health: counting zero records
974
- # out of zero would otherwise exit 0 on a typo'd filename and hide a broken build.
975
- missing: bool = not path.is_file()
976
- if not missing:
977
- with path.open(encoding="utf-8") as handle:
978
- for line in handle:
949
+ section: dict[str, Any] = _validate_file(path, edge=edge, limit=limit)
950
+ report[label] = section
951
+ if section["missing"] or section["total"] != section["valid"]:
952
+ report["ok"] = False
953
+ if section["missing"] or section["total"] != section["valid_excluding_pending"]:
954
+ report["ok_excluding_pending"] = False
955
+ return report
956
+
957
+
958
+ def _validate_file(path: Path, *, edge: bool, limit: int) -> dict[str, Any]:
959
+ """Validate one KGX NDJSON file record by record (the per-file half of ``validate_kgx``).
960
+
961
+ Args:
962
+ path: Path to the ``.ndjson`` file; a missing path reports ``missing`` instead of
963
+ counting zero records out of zero as a pass.
964
+ edge: ``True`` to dispatch on the association family, ``False`` for nodes.
965
+ limit: Maximum number of example failures to retain.
966
+
967
+ Returns:
968
+ The same per-file section ``validate_kgx`` nests under ``"nodes"`` / ``"edges"``.
969
+ """
970
+ total: int = 0
971
+ valid: int = 0
972
+ valid_excluding_pending: int = 0
973
+ problems: Counter[str] = Counter()
974
+ examples: list[dict[str, Any]] = []
975
+ # A missing path must never read as a clean bill of health: counting zero records
976
+ # out of zero would otherwise exit 0 on a typo'd filename and hide a broken build.
977
+ missing: bool = not path.is_file()
978
+ if not missing:
979
+ with path.open(encoding="utf-8") as handle:
980
+ for line in handle:
981
+ if not line.strip():
982
+ continue
983
+ total += 1
984
+ record: dict[str, Any] = json.loads(line)
985
+ errors: list[str] = validate_record(record, edge=edge)
986
+ if not errors:
987
+ valid += 1
988
+ valid_excluding_pending += 1
989
+ continue
990
+ if _record_has_only_pending_problems(record, errors):
991
+ valid_excluding_pending += 1
992
+ problems.update(errors)
993
+ if len(examples) < limit:
994
+ examples.append({"id": record.get("id"), "errors": errors})
995
+ return {
996
+ "total": total,
997
+ "valid": valid,
998
+ "valid_excluding_pending": valid_excluding_pending,
999
+ "failures": total - valid,
1000
+ "missing": missing,
1001
+ "problems": dict(problems.most_common()),
1002
+ "examples": examples,
1003
+ }
1004
+
1005
+
1006
+ def prune_kgx_edges(edges_path: Path, limit: int = 20) -> dict[str, Any]:
1007
+ """Drop every edge record that is not strictly KGX/Biolink-compliant, in place.
1008
+
1009
+ A record survives only when it validates against the Biolink Pydantic class named
1010
+ by its own ``category`` with zero errors -- the same strict bar a plain
1011
+ ``validate_kgx`` run uses for its ``ok`` verdict, so a prune can never disagree
1012
+ with what validation reports:
1013
+
1014
+ * a strictly valid record is kept;
1015
+ * a record whose every failure is a known, intentional model gap (the KGX
1016
+ denormalized carryovers in :data:`KNOWN_PENDING_EDGE_FIELDS`, or a
1017
+ :data:`PREDICATE_OVERRIDES` predicate the pinned classes reject) is dropped --
1018
+ these carryovers are most of a real graph, so the caller asked for a
1019
+ strictly-compliant graph over maximal data retention when invoking this;
1020
+ * every other failing record is a real defect and is dropped;
1021
+ * a line that is not valid JSON at all is dropped and counted separately in
1022
+ ``malformed``.
1023
+
1024
+ Nodes are never touched: dropping one would strand references, and cascading that
1025
+ into the edges is a modelling decision, not a validation repair.
1026
+
1027
+ The rewrite is ATOMIC and lossless for kept records: kept lines are written verbatim
1028
+ (byte-identical, order preserved, blank lines passed through) into a sibling
1029
+ ``.{name}.tmp`` and swapped in with :func:`os.replace` only once every line was
1030
+ written, so a crash never leaves a torn edges file. When nothing is prunable the
1031
+ destination is not rewritten at all (``rewritten`` is ``False`` and the bytes are
1032
+ untouched), and a missing file is reported (``missing``) without ever being created
1033
+ -- a typo'd path must never read as a prune.
1034
+
1035
+ Args:
1036
+ edges_path: Path to ``<name>_<version>.edges.ndjson``.
1037
+ limit: Maximum number of example failures to retain per report section.
1038
+
1039
+ Returns:
1040
+ Mapping with ``before`` (the same per-file section ``validate_kgx`` builds over
1041
+ the original file), ``after`` (the same shape over the kept records, i.e. the
1042
+ file's post-prune state, equal to what ``validate_kgx`` reports on the rewritten
1043
+ file), ``dropped`` (the ``pruned`` records' problem histogram and examples --
1044
+ the records that were removed), ``pruned`` / ``kept`` / ``malformed`` /
1045
+ ``rewritten`` counts, and ``ok`` / ``ok_excluding_pending`` computed over the
1046
+ post-prune state. For a readable file ``ok`` is therefore always true -- every
1047
+ kept record is strictly valid -- and ``ok_excluding_pending`` agrees, because
1048
+ nothing pending survives the prune.
1049
+
1050
+ Raises:
1051
+ OSError: the destination's directory is unwritable; the error propagates and the
1052
+ original file keeps its previous content.
1053
+ """
1054
+ total: int = 0
1055
+ valid: int = 0
1056
+ pending: int = 0 # pending-only failures: counted for the BEFORE report, never kept
1057
+ pruned: int = 0
1058
+ malformed: int = 0
1059
+ all_problems: Counter[str] = Counter()
1060
+ all_examples: list[dict[str, Any]] = []
1061
+ pruned_problems: Counter[str] = Counter()
1062
+ pruned_examples: list[dict[str, Any]] = []
1063
+ # A missing path must never read as a clean bill of health: reporting zero records
1064
+ # out of zero would otherwise exit 0 on a typo'd filename and hide a broken build.
1065
+ missing: bool = not edges_path.is_file()
1066
+ if not missing:
1067
+ tmp: Path = edges_path.with_name(f".{edges_path.name}.tmp")
1068
+ try:
1069
+ # newline="" on both ends keeps every kept line byte-identical: the default
1070
+ # universal-newlines mode would silently rewrite "\r\n" as "\n".
1071
+ with edges_path.open(encoding="utf-8", newline="") as src, tmp.open("w", encoding="utf-8", newline="") as dst:
1072
+ for line in src:
979
1073
  if not line.strip():
1074
+ dst.write(line) # blank lines are not records; pass through verbatim
980
1075
  continue
981
1076
  total += 1
982
- record: dict[str, Any] = json.loads(line)
983
- errors: list[str] = validate_record(record, edge=edge)
984
- if not errors:
985
- valid += 1
986
- valid_excluding_pending += 1
1077
+ try:
1078
+ record: dict[str, Any] = json.loads(line)
1079
+ except json.JSONDecodeError:
1080
+ malformed += 1
1081
+ pruned += 1
1082
+ pruned_problems["record: json_parse_error"] += 1
1083
+ if len(pruned_examples) < limit:
1084
+ pruned_examples.append({"id": None, "errors": ["record: json_parse_error"]})
987
1085
  continue
988
- if _record_has_only_pending_problems(record, errors):
989
- valid_excluding_pending += 1
990
- problems.update(errors)
991
- if len(examples) < limit:
992
- examples.append({"id": record.get("id"), "errors": errors})
993
- report[label] = {
994
- "total": total,
995
- "valid": valid,
996
- "valid_excluding_pending": valid_excluding_pending,
997
- "failures": total - valid,
998
- "missing": missing,
999
- "problems": dict(problems.most_common()),
1000
- "examples": examples,
1001
- }
1002
- if missing or total != valid:
1003
- report["ok"] = False
1004
- if missing or total != valid_excluding_pending:
1005
- report["ok_excluding_pending"] = False
1006
- return report
1086
+ errors: list[str] = validate_record(record, edge=True)
1087
+ all_problems.update(errors)
1088
+ if len(all_examples) < limit:
1089
+ all_examples.append({"id": record.get("id"), "errors": errors})
1090
+ if errors:
1091
+ # The before-section keeps validate_kgx's pending score so the
1092
+ # pre-report matches a plain validation run; the STRICT bar still
1093
+ # drops the record -- real defects AND deliberate pending gaps.
1094
+ if _record_has_only_pending_problems(record, errors):
1095
+ pending += 1
1096
+ pruned += 1
1097
+ pruned_problems.update(errors)
1098
+ if len(pruned_examples) < limit:
1099
+ pruned_examples.append({"id": record.get("id"), "errors": errors})
1100
+ continue
1101
+ valid += 1
1102
+ dst.write(line)
1103
+ # ATOMIC: the swap happens only after every kept line was written, so a crash
1104
+ # mid-write leaves the original file untouched and the temp never read back.
1105
+ rewritten: bool = pruned > 0
1106
+ if rewritten:
1107
+ os.replace(tmp, edges_path)
1108
+ finally:
1109
+ tmp.unlink(missing_ok=True) # no-op after a successful replace; cleanup after a raise
1110
+ else:
1111
+ rewritten = False
1112
+ kept: int = total - pruned
1113
+ before: dict[str, Any] = {
1114
+ "total": total,
1115
+ "valid": valid,
1116
+ "valid_excluding_pending": valid + pending,
1117
+ "failures": total - valid,
1118
+ "missing": missing,
1119
+ "problems": dict(all_problems.most_common()),
1120
+ "examples": all_examples,
1121
+ }
1122
+ dropped: dict[str, Any] = {"total": pruned, "problems": dict(pruned_problems.most_common()), "examples": pruned_examples}
1123
+ after: dict[str, Any] = {
1124
+ "total": kept,
1125
+ "valid": valid,
1126
+ "valid_excluding_pending": valid, # nothing pending survives, so the scores agree
1127
+ "failures": kept - valid,
1128
+ "missing": missing,
1129
+ "problems": {},
1130
+ "examples": [],
1131
+ }
1132
+ return {
1133
+ "biolink_version": BIOLINK_VERSION,
1134
+ "before": before,
1135
+ "after": after,
1136
+ "dropped": dropped,
1137
+ "pruned": pruned,
1138
+ "kept": kept,
1139
+ "malformed": malformed,
1140
+ "rewritten": rewritten,
1141
+ "ok": not missing and kept == valid,
1142
+ "ok_excluding_pending": not missing and kept == valid,
1143
+ }
1007
1144
 
1008
1145
 
1009
1146
  # Node slots Tablassert can always populate from a fullmap hit. A class requiring
@@ -955,11 +955,29 @@ def validate(
955
955
  run(3, validate_pipeline, configuration_file)
956
956
 
957
957
 
958
+ def _print_kgx_section(label: str, section: dict[str, Any], path: Path) -> None:
959
+ """Print one ``validate-kgx`` per-file section (shared by the plain and ``--prune`` paths)."""
960
+ if section["missing"]:
961
+ # Never let a typo'd path read as a pass: 0/0 valid would otherwise exit 0.
962
+ print(f"{label}: file not found ({path})", file=sys.stderr)
963
+ logger.info(f"validate-kgx {label}: file not found")
964
+ return
965
+ pending: int = section["valid_excluding_pending"] - section["valid"]
966
+ suffix: str = f"; {pending} pending biolink-model support" if pending else ""
967
+ print(f"{label}: {section['valid']}/{section['total']} valid ({section['failures']} failures{suffix})", file=sys.stderr)
968
+ for problem, count in section["problems"].items():
969
+ print(f" {count:>9} {problem}", file=sys.stderr)
970
+ for example in section["examples"][:3]:
971
+ print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
972
+ logger.info(f"validate-kgx {label}: {section['valid']}/{section['total']} valid")
973
+
974
+
958
975
  @APP.command(name="validate-kgx")
959
976
  def validate_kgx_command(
960
977
  nodes: Annotated[Path, cyclopts.Parameter(name=["--nodes", "-n"])],
961
978
  edges: Annotated[Path, cyclopts.Parameter(name=["--edges", "-e"])],
962
979
  limit: Annotated[int, cyclopts.Parameter(name=["--limit"])] = 20,
980
+ prune: Annotated[bool, cyclopts.Parameter(name=["--prune", "-p"])] = False,
963
981
  ) -> None:
964
982
  """Validate built KGX NDJSON against the Biolink Model.
965
983
 
@@ -968,33 +986,60 @@ def validate_kgx_command(
968
986
  and reports failures grouped by field and error type. Exits non-zero when any
969
987
  record fails, so a build can be gated in CI.
970
988
 
989
+ ``--prune`` rewrites the EDGES file in place, keeping only edges that validate
990
+ strictly. Real defects (non-coercible values, extras the model will never declare,
991
+ rejected predicates), the deliberate pending carryovers (``synonym``/``xref``/
992
+ ``relation``/``provided_by``, ...), and lines that are not JSON at all are all
993
+ removed. On a real graph the carryovers are most of the edges, so ``--prune``
994
+ trades them for a strictly-compliant graph. The rewrite is atomic and kept lines
995
+ stay byte-identical; a clean file is not rewritten at all, and a missing edges file
996
+ is reported without ever being pruned. Nodes are validated and reported, never
997
+ pruned. The exit code stays strict over the post-prune state: after a prune, every
998
+ kept edge is strictly valid, so the run is compliant exactly when the nodes are.
999
+
971
1000
  Args:
972
1001
  nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
973
1002
  edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
974
1003
  limit: Maximum example failures retained per file for the ``e.g.`` report lines;
975
1004
  every record is still validated regardless of this cap. A missing file is
976
1005
  reported and fails the run rather than reading as a pass.
1006
+ prune: Rewrite the edges file in place, removing every edge record that is not
1007
+ strictly KGX/Biolink-compliant, including the deliberate pending carryovers.
1008
+ The nodes file is never modified; a clean edges file is left untouched.
977
1009
  """
978
- from tablassert.biolink import validate_kgx
979
-
980
- report: dict[str, Any] = validate_kgx(nodes, edges, limit=limit)
981
- print(f"biolink-model {report['biolink_version']}", file=sys.stderr)
982
- for label in ("nodes", "edges"):
983
- section: dict[str, Any] = report[label]
984
- if section["missing"]:
985
- # Never let a typo'd path read as a pass: 0/0 valid would otherwise exit 0.
986
- print(f"{label}: file not found ({nodes if label == 'nodes' else edges})", file=sys.stderr)
987
- logger.info(f"validate-kgx {label}: file not found")
988
- continue
989
- pending: int = section["valid_excluding_pending"] - section["valid"]
990
- suffix: str = f"; {pending} pending biolink-model support" if pending else ""
991
- print(f"{label}: {section['valid']}/{section['total']} valid ({section['failures']} failures{suffix})", file=sys.stderr)
992
- for problem, count in section["problems"].items():
993
- print(f" {count:>9} {problem}", file=sys.stderr)
994
- for example in section["examples"][:3]:
995
- print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
996
- logger.info(f"validate-kgx {label}: {section['valid']}/{section['total']} valid")
997
- if not report["ok"]:
1010
+ from tablassert.biolink import _validate_file, prune_kgx_edges, validate_kgx
1011
+
1012
+ if not prune:
1013
+ report: dict[str, Any] = validate_kgx(nodes, edges, limit=limit)
1014
+ print(f"biolink-model {report['biolink_version']}", file=sys.stderr)
1015
+ _print_kgx_section("nodes", report["nodes"], nodes)
1016
+ _print_kgx_section("edges", report["edges"], edges)
1017
+ if not report["ok"]:
1018
+ print("KGX output is not Biolink-compliant.", file=sys.stderr)
1019
+ raise SystemExit(1)
1020
+ print("KGX output is Biolink-compliant.", file=sys.stderr)
1021
+ return
1022
+
1023
+ # The prune path validates nodes-only itself and drives prune_kgx_edges' own single
1024
+ # pass for both edges sections, so the edges file is never read and written twice.
1025
+ nodes_section: dict[str, Any] = _validate_file(nodes, edge=False, limit=limit)
1026
+ pruned_report: dict[str, Any] = prune_kgx_edges(edges, limit=limit)
1027
+ print(f"biolink-model {pruned_report['biolink_version']}", file=sys.stderr)
1028
+ _print_kgx_section("nodes", nodes_section, nodes)
1029
+ _print_kgx_section("edges", pruned_report["before"], edges)
1030
+ if pruned_report["before"]["missing"]:
1031
+ print("KGX output is not Biolink-compliant.", file=sys.stderr)
1032
+ raise SystemExit(1)
1033
+ malformed_note: str = f" ({pruned_report['malformed']} malformed lines)" if pruned_report["malformed"] else ""
1034
+ print(f"edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} non-compliant records{malformed_note}", file=sys.stderr)
1035
+ logger.info(f"validate-kgx edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} (rewritten={pruned_report['rewritten']})")
1036
+ for problem, count in pruned_report["dropped"]["problems"].items():
1037
+ print(f" {count:>9} {problem}", file=sys.stderr)
1038
+ for example in pruned_report["dropped"]["examples"][:3]:
1039
+ print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
1040
+ _print_kgx_section("edges", pruned_report["after"], edges)
1041
+ nodes_ok: bool = not nodes_section["missing"] and nodes_section["total"] == nodes_section["valid"]
1042
+ if not (nodes_ok and pruned_report["ok"]):
998
1043
  print("KGX output is not Biolink-compliant.", file=sys.stderr)
999
1044
  raise SystemExit(1)
1000
1045
  print("KGX output is Biolink-compliant.", file=sys.stderr)
File without changes
File without changes
File without changes
File without changes
File without changes