tablassert 19.2.0__tar.gz → 19.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {tablassert-19.2.0 → tablassert-19.4.0}/PKG-INFO +14 -16
  2. {tablassert-19.2.0 → tablassert-19.4.0}/README.md +9 -12
  3. {tablassert-19.2.0 → tablassert-19.4.0}/pyproject.toml +5 -4
  4. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/cli.py +81 -12
  5. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/errors.py +1 -0
  6. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/fullmap.py +4 -1
  7. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/lib.py +81 -6
  8. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/runcache.py +2 -0
  9. {tablassert-19.2.0 → tablassert-19.4.0}/LICENSE +0 -0
  10. {tablassert-19.2.0 → tablassert-19.4.0}/rust/Cargo.lock +0 -0
  11. {tablassert-19.2.0 → tablassert-19.4.0}/rust/Cargo.toml +0 -0
  12. {tablassert-19.2.0 → tablassert-19.4.0}/rust/examples/count_tables.rs +0 -0
  13. {tablassert-19.2.0 → tablassert-19.4.0}/rust/src/fullmap.rs +0 -0
  14. {tablassert-19.2.0 → tablassert-19.4.0}/rust/src/json.rs +0 -0
  15. {tablassert-19.2.0 → tablassert-19.4.0}/rust/src/lib.rs +0 -0
  16. {tablassert-19.2.0 → tablassert-19.4.0}/rust/src/ndjson.rs +0 -0
  17. {tablassert-19.2.0 → tablassert-19.4.0}/rust/src/nlp.rs +0 -0
  18. {tablassert-19.2.0 → tablassert-19.4.0}/rust/src/uuid.rs +0 -0
  19. {tablassert-19.2.0 → tablassert-19.4.0}/rust/tests/build_golden.rs +0 -0
  20. {tablassert-19.2.0 → tablassert-19.4.0}/rust/tests/common/mod.rs +0 -0
  21. {tablassert-19.2.0 → tablassert-19.4.0}/rust/tests/extract_prebuilt.rs +0 -0
  22. {tablassert-19.2.0 → tablassert-19.4.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
  23. {tablassert-19.2.0 → tablassert-19.4.0}/rust/tests/nlp_golden.rs +0 -0
  24. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/__init__.py +0 -0
  25. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/_lazy.py +0 -0
  26. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/agent.py +0 -0
  27. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/biolink.py +0 -0
  28. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/cli_entry.py +0 -0
  29. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/coerce.py +0 -0
  30. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
  31. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/distill.py +0 -0
  32. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/distill_reward.py +0 -0
  33. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/enums.py +0 -0
  34. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/extras.py +0 -0
  35. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/graph_target.py +0 -0
  36. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/ingests.py +0 -0
  37. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/log.py +0 -0
  38. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/models.py +0 -0
  39. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/net.py +0 -0
  40. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/nlp.py +0 -0
  41. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/progress.py +0 -0
  42. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/qc.py +0 -0
  43. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/rig.py +0 -0
  44. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/rs.pyi +0 -0
  45. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/study.py +0 -0
  46. {tablassert-19.2.0 → tablassert-19.4.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 19.2.0
3
+ Version: 19.4.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -23,11 +23,12 @@ Classifier: Operating System :: POSIX :: Linux
23
23
  Classifier: Operating System :: MacOS :: MacOS X
24
24
  Classifier: Environment :: Console
25
25
  Requires-Dist: biolink-model>=4.4.4
26
- Requires-Dist: polars>=1.39.0
26
+ Requires-Dist: polars>=2.0.0rc2
27
+ Requires-Dist: pyarrow>=21.0.0
27
28
  Requires-Dist: rapidfuzz>=3.14.3
28
29
  Requires-Dist: pydantic>=2.12.5
29
30
  Requires-Dist: pyyaml>=6.0.3
30
- Requires-Dist: fastexcel>=0.20.2
31
+ Requires-Dist: fastexcel>=0.21.0
31
32
  Requires-Dist: smolagents>=1.26.0 ; extra == 'agent'
32
33
  Requires-Dist: litellm>=1.93.0 ; extra == 'agent'
33
34
  Requires-Dist: aria2==0.0.1b0 ; extra == 'aria2'
@@ -38,7 +39,7 @@ Requires-Dist: loguru>=0.7.3 ; extra == 'log'
38
39
  Requires-Dist: dspy>=3.2.1 ; extra == 'optimize'
39
40
  Requires-Dist: scikit-learn>=1.8.0 ; extra == 'qc'
40
41
  Requires-Dist: sentence-transformers>=5.3.0 ; extra == 'qc'
41
- Requires-Dist: polars[rtcompat]>=1.40.1 ; extra == 'rt'
42
+ Requires-Dist: polars[rtcompat]>=2.0.0rc2 ; extra == 'rt'
42
43
  Provides-Extra: agent
43
44
  Provides-Extra: aria2
44
45
  Provides-Extra: cli
@@ -60,13 +61,10 @@ Project-URL: Source, https://github.com/SkyeAv/Tablassert
60
61
 
61
62
  # Tablassert
62
63
 
63
- [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
64
- [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
65
64
  [![PyPI version](https://badge.fury.io/py/tablassert.svg)](https://badge.fury.io/py/tablassert)
66
- [![Downloads](https://static.pepy.tech/badge/tablassert/month)](https://pepy.tech/project/tablassert)
67
65
  [![CI](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml/badge.svg)](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
68
66
  [![Docs](https://img.shields.io/github/deployments/SkyeAv/Tablassert/github-pages?label=docs)](https://skyeav.github.io/Tablassert/)
69
- [![GitHub stars](https://img.shields.io/github/stars/SkyeAv/Tablassert.svg)](https://github.com/SkyeAv/Tablassert/stargazers)
67
+ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
70
68
 
71
69
  > Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON,
72
70
  > declaratively, with entity resolution built in and optional quality control.
@@ -82,7 +80,7 @@ configuration reference, and API docs.
82
80
  ## Quick Start
83
81
 
84
82
  ```bash
85
- pip install tablassert
83
+ pip install "tablassert[cli]"
86
84
  ```
87
85
 
88
86
  Given a CSV of gene-disease associations with p-values and sample sizes, declare the mapping in a
@@ -147,7 +145,7 @@ Output is one JSON object per line: nodes with Biolink categories, edges with an
147
145
  ```
148
146
 
149
147
  ```json
150
- {"subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}}}
148
+ {"id":"2cfea591-0f8f-33af-a7df-03da531d3359","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
151
149
  ```
152
150
 
153
151
  See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
@@ -157,7 +155,7 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
157
155
  - **Declarative YAML configuration**: define data transformations without writing code
158
156
  - **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
159
157
  CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
160
- - **Optional quality control**: a four-stage audit (exact → fuzzy → abbreviation → SapBERT embeddings) flags
158
+ - **Optional quality control**: a four-stage audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) flags
161
159
  low-confidence mappings
162
160
  - **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
163
161
  and predicates
@@ -179,7 +177,7 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
179
177
  | `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
180
178
  | `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
181
179
  | `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
182
- | `qc` | four-stage QC audit (exact → fuzzy → abbreviation → SapBERT embeddings) | `pip install "tablassert[qc]"` |
180
+ | `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
183
181
  | `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
184
182
  | `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
185
183
  | `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
@@ -216,11 +214,11 @@ no LazyFrame setup or NLP preprocessing required. See the
216
214
 
217
215
  - **[Installation](https://skyeav.github.io/Tablassert/installation/)**: install methods, extras, and development setup
218
216
  - **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
219
- - **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
220
217
  - **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
218
+ - **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
221
219
  - **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
222
- - **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
223
220
  - **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
221
+ - **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
224
222
  - **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
225
223
  - **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
226
224
  - **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
@@ -241,8 +239,8 @@ request guidelines.
241
239
  If you use Tablassert, please cite it as described in [CITATION.cff](CITATION.cff). The approach is
242
240
  described in:
243
241
 
244
- > Skye Lane Goetz, Alex K. Glen, and Gwênlyn Glusman. “MicrobiomeKG: bridging microbiome research
245
- > and host health through knowledge graphs.” *Frontiers in Systems Biology* 5 (2025).
242
+ > Skye Lane Goetz, Amy K. Glen, and Gwênlyn Glusman. "MicrobiomeKG: bridging microbiome research
243
+ > and host health through knowledge graphs." *Frontiers in Systems Biology* 5 (2025).
246
244
  > [doi:10.3389/fsysb.2025.1544432](https://doi.org/10.3389/fsysb.2025.1544432)
247
245
 
248
246
  ## License
@@ -1,12 +1,9 @@
1
1
  # Tablassert
2
2
 
3
- [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
4
- [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
5
3
  [![PyPI version](https://badge.fury.io/py/tablassert.svg)](https://badge.fury.io/py/tablassert)
6
- [![Downloads](https://static.pepy.tech/badge/tablassert/month)](https://pepy.tech/project/tablassert)
7
4
  [![CI](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml/badge.svg)](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
8
5
  [![Docs](https://img.shields.io/github/deployments/SkyeAv/Tablassert/github-pages?label=docs)](https://skyeav.github.io/Tablassert/)
9
- [![GitHub stars](https://img.shields.io/github/stars/SkyeAv/Tablassert.svg)](https://github.com/SkyeAv/Tablassert/stargazers)
6
+ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
10
7
 
11
8
  > Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON,
12
9
  > declaratively, with entity resolution built in and optional quality control.
@@ -22,7 +19,7 @@ configuration reference, and API docs.
22
19
  ## Quick Start
23
20
 
24
21
  ```bash
25
- pip install tablassert
22
+ pip install "tablassert[cli]"
26
23
  ```
27
24
 
28
25
  Given a CSV of gene-disease associations with p-values and sample sizes, declare the mapping in a
@@ -87,7 +84,7 @@ Output is one JSON object per line: nodes with Biolink categories, edges with an
87
84
  ```
88
85
 
89
86
  ```json
90
- {"subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}}}
87
+ {"id":"2cfea591-0f8f-33af-a7df-03da531d3359","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.csv","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
91
88
  ```
92
89
 
93
90
  See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
@@ -97,7 +94,7 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
97
94
  - **Declarative YAML configuration**: define data transformations without writing code
98
95
  - **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
99
96
  CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
100
- - **Optional quality control**: a four-stage audit (exact → fuzzy → abbreviation → SapBERT embeddings) flags
97
+ - **Optional quality control**: a four-stage audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) flags
101
98
  low-confidence mappings
102
99
  - **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
103
100
  and predicates
@@ -119,7 +116,7 @@ install `[cli]` to use the `tablassert` command and optional extras for addition
119
116
  | `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
120
117
  | `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
121
118
  | `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
122
- | `qc` | four-stage QC audit (exact → fuzzy → abbreviation → SapBERT embeddings) | `pip install "tablassert[qc]"` |
119
+ | `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
123
120
  | `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
124
121
  | `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
125
122
  | `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
@@ -156,11 +153,11 @@ no LazyFrame setup or NLP preprocessing required. See the
156
153
 
157
154
  - **[Installation](https://skyeav.github.io/Tablassert/installation/)**: install methods, extras, and development setup
158
155
  - **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
159
- - **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
160
156
  - **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
157
+ - **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
161
158
  - **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
162
- - **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
163
159
  - **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
160
+ - **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
164
161
  - **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
165
162
  - **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
166
163
  - **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
@@ -181,8 +178,8 @@ request guidelines.
181
178
  If you use Tablassert, please cite it as described in [CITATION.cff](CITATION.cff). The approach is
182
179
  described in:
183
180
 
184
- > Skye Lane Goetz, Alex K. Glen, and Gwênlyn Glusman. “MicrobiomeKG: bridging microbiome research
185
- > and host health through knowledge graphs.” *Frontiers in Systems Biology* 5 (2025).
181
+ > Skye Lane Goetz, Amy K. Glen, and Gwênlyn Glusman. "MicrobiomeKG: bridging microbiome research
182
+ > and host health through knowledge graphs." *Frontiers in Systems Biology* 5 (2025).
186
183
  > [doi:10.3389/fsysb.2025.1544432](https://doi.org/10.3389/fsysb.2025.1544432)
187
184
 
188
185
  ## License
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "19.2.0"
3
+ version = "19.4.0"
4
4
  description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -48,11 +48,12 @@ classifiers = [
48
48
  requires-python = ">=3.11"
49
49
  dependencies = [
50
50
  "biolink-model>=4.4.4",
51
- "polars>=1.39.0",
51
+ "polars>=2.0.0rc2",
52
+ "pyarrow>=21.0.0",
52
53
  "rapidfuzz>=3.14.3",
53
54
  "pydantic>=2.12.5",
54
55
  "pyyaml>=6.0.3",
55
- "fastexcel>=0.20.2",
56
+ "fastexcel>=0.21.0",
56
57
  ]
57
58
 
58
59
  [project.urls]
@@ -81,7 +82,7 @@ cli = [
81
82
  "rich>=13.0.0",
82
83
  ]
83
84
  rt = [
84
- "polars[rtcompat]>=1.40.1",
85
+ "polars[rtcompat]>=2.0.0rc2",
85
86
  ]
86
87
  # PyPI distribution "aria2" exposes import module "aria2c" with a bundled aria2c binary.
87
88
  aria2 = [
@@ -912,6 +912,20 @@ def build_kg(
912
912
  demoted edges from unpinned sections (a row escaped its pin and shipped without the
913
913
  class-specific slots ``prune_to_class`` nulled into ``has_supporting_studies``) --
914
914
  and fails the build (non-zero exit) when any assertion is violated.
915
+
916
+ Args:
917
+ graph_configuration_file: Path to the Graph YAML that orchestrates one or more table
918
+ configs into a single knowledge-graph build (accepted positionally or via
919
+ ``--configuration-file``/``-f``).
920
+ release: Emit release-mode artifacts: a slim, significant-only graph that drops
921
+ ``not_significant`` edges before resolution (cached under its own ``.release``
922
+ store, so it never reuses a default build).
923
+ qc: Run quality-control audits and the final study stage over the emitted NDJSON;
924
+ requires the ``[qc]`` extra and exits non-zero when any assertion is violated.
925
+ log: Enable verbose per-section logging (requires the ``[log]`` extra; without it
926
+ Tablassert produces no logs).
927
+ head: Fast output-shape preview: a random sample of up to five rows per section,
928
+ cached under its own ``.head`` store, so it never reuses or replaces a full build.
915
929
  """
916
930
  if qc:
917
931
  extras.require("qc", required_by="--qc")
@@ -928,6 +942,12 @@ def validate(
928
942
  ``--schema graph`` validates the Graph model AND every referenced table; ``--schema
929
943
  table`` validates section syntax only. The schema is selected explicitly rather than
930
944
  sniffed from the YAML, so a config is always checked against the schema you expected.
945
+
946
+ Args:
947
+ configuration_file: Path to the YAML configuration file to validate (accepted
948
+ positionally or via ``--configuration-file``/``-f``).
949
+ schema: Which schema to validate against: ``graph`` checks the Graph model and every
950
+ referenced table; ``table`` checks section syntax of a single table YAML only.
931
951
  """
932
952
  if schema == "graph":
933
953
  run(2, validate_graph_pipeline, configuration_file)
@@ -947,6 +967,13 @@ def validate_kgx_command(
947
967
  ``category`` -- the same classes ``NCATSTranslator/translator-ingests`` builds --
948
968
  and reports failures grouped by field and error type. Exits non-zero when any
949
969
  record fails, so a build can be gated in CI.
970
+
971
+ Args:
972
+ nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
973
+ edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
974
+ limit: Maximum example failures retained per file for the ``e.g.`` report lines;
975
+ every record is still validated regardless of this cap. A missing file is
976
+ reported and fails the run rather than reading as a pass.
950
977
  """
951
978
  from tablassert.biolink import validate_kgx
952
979
 
@@ -975,14 +1002,16 @@ def validate_kgx_command(
975
1002
 
976
1003
  @APP.command(name="quick-map")
977
1004
  def quick_map_command(
978
- terms: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False)],
1005
+ # negative="" suppresses cyclopts' auto-generated --empty-terms help row: the reset
1006
+ # negation is unused noise that rendered as a phantom flag on the help page.
1007
+ terms: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False, negative="")],
979
1008
  *,
980
1009
  fullmap: Annotated[Path, cyclopts.Parameter(name=["--fullmap", "-f"])],
981
1010
  taxon: Annotated[int | None, cyclopts.Parameter(name=["--taxon", "-t"])] = 9606,
982
- prioritize: Annotated[list[str] | None, cyclopts.Parameter(name=["--prioritize", "-p"])] = None,
983
- avoid: Annotated[list[str] | None, cyclopts.Parameter(name=["--avoid", "-a"])] = None,
984
- exclude_prefixes: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-prefixes", "-ep"])] = None,
985
- exclude_regex: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-regex", "-er"])] = None,
1011
+ prioritize: Annotated[list[str] | None, cyclopts.Parameter(name=["--prioritize", "-p"], negative="")] = None,
1012
+ avoid: Annotated[list[str] | None, cyclopts.Parameter(name=["--avoid", "-a"], negative="")] = None,
1013
+ exclude_prefixes: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-prefixes", "-ep"], negative="")] = None,
1014
+ exclude_regex: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-regex", "-er"], negative="")] = None,
986
1015
  ) -> None:
987
1016
  """Show what fullmap entity resolution does with one or more terms, as a build would see it.
988
1017
 
@@ -1096,7 +1125,9 @@ def quick_map_command(
1096
1125
 
1097
1126
  @APP.command(name="agent")
1098
1127
  def agent(
1099
- pmc_ids: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False)],
1128
+ # negative="" suppresses cyclopts' auto-generated --empty-pmc-ids help row, which
1129
+ # rendered as a phantom REQUIRED flag on the help page.
1130
+ pmc_ids: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False, negative="")],
1100
1131
  *,
1101
1132
  graph_configuration_file: Annotated[Path, cyclopts.Parameter(name=["--configuration-file", "-f"])],
1102
1133
  model_id: Annotated[str | None, cyclopts.Parameter(name=["--model-id", "-m"])] = None,
@@ -1112,7 +1143,7 @@ def agent(
1112
1143
  judge_model: Annotated[str | None, cyclopts.Parameter(name=["--judge-model"])] = None,
1113
1144
  judge_threshold: Annotated[float | None, cyclopts.Parameter(name=["--judge-threshold"])] = None,
1114
1145
  biolink_threshold: Annotated[float, cyclopts.Parameter(name=["--biolink-threshold"])] = 0.0,
1115
- local: Annotated[list[str] | None, cyclopts.Parameter(name=["--local", "-l"])] = None,
1146
+ local: Annotated[list[str] | None, cyclopts.Parameter(name=["--local", "-l"], negative="")] = None,
1116
1147
  optimize: Annotated[bool, cyclopts.Parameter(name=["--optimize", "-o"], negative="")] = False,
1117
1148
  distill: Annotated[bool, cyclopts.Parameter(name=["--distill", "-d", "-dt"], negative="")] = False,
1118
1149
  instructions_file: Annotated[Path | None, cyclopts.Parameter(name=["--instructions-file"])] = None,
@@ -1375,7 +1406,9 @@ def distill_weigh(
1375
1406
  *,
1376
1407
  distill_dir: Annotated[Path, cyclopts.Parameter(name=["--distill-dir", "-dd"])],
1377
1408
  out: Annotated[Path, cyclopts.Parameter(name=["--out", "-o"])],
1378
- policy: Annotated[str, cyclopts.Parameter(name=["--policy", "-p"])] = "threshold",
1409
+ # Literal mirrors tablassert.distill_reward.POLICIES (kept inline so the CLI stays
1410
+ # import-light); tests/test_cli_help_friendly.py enforces the two stay in lockstep.
1411
+ policy: Annotated[Literal["threshold", "best-of-n", "replication"], cyclopts.Parameter(name=["--policy", "-p"])] = "threshold",
1379
1412
  threshold: Annotated[float, cyclopts.Parameter(name=["--threshold", "-t"])] = 0.75,
1380
1413
  top_n: Annotated[int, cyclopts.Parameter(name=["--top-n", "-tn"])] = 2,
1381
1414
  replication_k: Annotated[int, cyclopts.Parameter(name=["--replication-k", "-rk"])] = 2,
@@ -1385,7 +1418,39 @@ def distill_weigh(
1385
1418
  final_call_only: Annotated[bool, cyclopts.Parameter(name="--final-call-only", negative="")] = False,
1386
1419
  manifest: Annotated[Path | None, cyclopts.Parameter(name="--manifest")] = None,
1387
1420
  ) -> None:
1388
- """Join distillation records to outcomes and write deterministic training rows."""
1421
+ """Join distillation records to outcomes and write deterministic training rows.
1422
+
1423
+ Data selection for LoRA/QLoRA supervised fine-tuning, not RLHF: one flat training row per
1424
+ input record, weighted by the deterministic reward, plus a reproducibility manifest.
1425
+ Malformed input, invalid knobs, unmatched records, or an ``--out`` inside
1426
+ ``--distill-dir`` fail with exit 2 and an actionable message.
1427
+
1428
+ Args:
1429
+ distill_dir: Input directory holding the raw ``records.ndjson``/``outcomes.ndjson``
1430
+ written by ``tablassert agent --distill``.
1431
+ out: Training NDJSON destination. Must be a file OUTSIDE ``--distill-dir``:
1432
+ ``distill-export`` loads every ``*.ndjson`` there, so an output inside it would
1433
+ duplicate raw records and mix schemas.
1434
+ policy: Row-selection policy. ``threshold`` keeps rows with ``weight >= --threshold``;
1435
+ ``best-of-n`` keeps the top ``--top-n`` rows per ``pmc_id`` group; ``replication``
1436
+ weights repeat occurrences by ``--replication-k``.
1437
+ threshold: Hard weight floor for the ``threshold`` policy, in ``[0, 1]``.
1438
+ top_n: Rows retained per ``pmc_id`` group for the ``best-of-n`` policy (at least 1).
1439
+ replication_k: Replication slope for the ``replication`` policy, bounded to ``[0, 3]``.
1440
+ reward_config: Optional YAML/JSON overriding the default deterministic
1441
+ :class:`tablassert.distill_reward.RewardConfig`.
1442
+ edge_ref: Breadth reference override (finite positive number). Defaults to the
1443
+ corpus median edge count; when the corpus has no comparable build the source is
1444
+ ``null`` and breadth contributes ``0.0`` (warned on stderr).
1445
+ purpose: Keep only records with this ``purpose`` (default ``agent``, the supervisor's
1446
+ own calls), or ``all`` to disable filtering. Any other value must match a purpose
1447
+ present in the corpus; an unknown value exits 2 naming the live purposes.
1448
+ final_call_only: Keep only the highest ``call_index`` record per ``run_id`` (the
1449
+ run's final LLM call). Records with no rankable ``run_id``/``call_index`` pair
1450
+ are retained and counted in a warning, never silently dropped.
1451
+ manifest: Reproducibility-manifest destination (default ``<out>.manifest.json``).
1452
+ Must differ from ``--out`` and stay outside ``--distill-dir``.
1453
+ """
1389
1454
  from tablassert.distill_reward import (
1390
1455
  OUTCOME_COLUMN_PREFIX,
1391
1456
  POLICIES,
@@ -1740,7 +1805,7 @@ def _prebuilt_fullmap_urls(babel_version: str) -> tuple[str, str]:
1740
1805
  installed-package metadata, never hardcoded, so a new release looks itself up.
1741
1806
 
1742
1807
  Args:
1743
- babel_version: BABEL snapshot label (the ``--version`` value), e.g. ``2026jul22``.
1808
+ babel_version: BABEL snapshot label (the ``--babel-version`` value), e.g. ``2026jul22``.
1744
1809
 
1745
1810
  Returns:
1746
1811
  ``(archive_url, checksum_url)`` for ``fullmap.tar.zst`` and ``sha256sum.txt``.
@@ -2017,7 +2082,10 @@ def fullmap_matches_allowlist(output: Path, taxon_allowlist: list[int]) -> bool:
2017
2082
  def build_fullmap(
2018
2083
  output: Annotated[Path, cyclopts.Parameter(name=["--output", "-o"])] = Path("./fullmap/data/fullmap.redb"),
2019
2084
  cache: Annotated[Path, cyclopts.Parameter(name=["--cache", "-c"])] = Path("./fullmap/downloads"),
2020
- version: Annotated[str, cyclopts.Parameter(name=["--version", "-v"])] = BABEL_VERSION,
2085
+ # Named --babel-version because the app-level --version flag shadowed the old --version
2086
+ # long form: `build-fullmap --version <snapshot>` printed the app version and exited 0
2087
+ # without ever running the command. -v keeps the short form.
2088
+ version: Annotated[str, cyclopts.Parameter(name=["--babel-version", "-v"])] = BABEL_VERSION,
2021
2089
  force: Annotated[bool, cyclopts.Parameter(name=["--force", "-f"], negative="")] = False,
2022
2090
  ) -> None:
2023
2091
  """Build an embedded fullmap redb database, or download a prebuilt one from RENCI.
@@ -2044,7 +2112,8 @@ def build_fullmap(
2044
2112
  Args:
2045
2113
  output: Path to write the redb file (prebuilt extraction or build output).
2046
2114
  cache: Directory for downloaded BABEL files when building from scratch.
2047
- version: BABEL snapshot date to fetch (a RENCI stamp, NOT Tablassert's version).
2115
+ version: BABEL snapshot date to fetch via ``--babel-version`` (a RENCI stamp, NOT
2116
+ Tablassert's version).
2048
2117
  force: Skip the prebuilt download and always rebuild from BABEL outputs.
2049
2118
  """
2050
2119
  allowlist_ids: list[int] = load_taxon_allowlist()
@@ -70,6 +70,7 @@ TablassertErrorCodes = Literal[
70
70
  "qualifier-unsatisfiable",
71
71
  "qualifier-nullable-literal",
72
72
  "qualifier-duplicated",
73
+ "qualifier-vocabulary-violation",
73
74
  "rig-bad-infores",
74
75
  "rig-bad-artifact-url",
75
76
  "rig-bad-access-location",
@@ -292,7 +292,10 @@ def distinct(lf: pl.LazyFrame, l1: str, l2: str, col: str = "term") -> pl.LazyFr
292
292
  t2: pl.LazyFrame = lf.select(pl.col(l2).alias(col)).unique()
293
293
  t2 = t2.with_columns(pl.lit(2).alias("nlp_level"))
294
294
 
295
- terms: pl.LazyFrame = pl.concat([t1, t2]).unique(subset=[col], keep="first")
295
+ # Level-1 wins per term. Group-by min instead of concat + unique(keep="first"):
296
+ # the streaming engine (polars 2.0 default) does not preserve row order for
297
+ # concat/unique, so first-seen tier selection would be scheduling-dependent.
298
+ terms: pl.LazyFrame = pl.concat([t1, t2]).group_by(col).agg(pl.col("nlp_level").min())
296
299
 
297
300
  bad: str = r"^\d+$|^(none|nan|na|null|unknown|not applicable|p_value|variable|result|exposure|expression|symbol)$|^$"
298
301
  return terms.filter(~pl.col(col).str.contains(bad))
@@ -45,6 +45,7 @@ from tablassert.coerce import (
45
45
  study_size_target,
46
46
  )
47
47
  from tablassert.enums import EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
48
+ from tablassert.errors import TablassertError
48
49
  from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
49
50
  from tablassert.log import cat
50
51
  from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
@@ -892,6 +893,16 @@ def idx(lf: pl.LazyFrame, col: str = "extracted_from_row_number") -> pl.LazyFram
892
893
  return lf.with_row_index(col, offset=1)
893
894
 
894
895
 
896
+ def _polars_auto_column_names(names: list[str]) -> bool:
897
+ """Return True when ``names`` is exactly polars' headerless auto-naming scheme.
898
+
899
+ Polars 2.0 autogenerates headerless column names as ``column_0``.. ``column_{n-1}``
900
+ (1.x used ``column_1``.. ``column_n``). Matching the full consecutive run keeps the
901
+ remap below strictly scoped to machine-generated names, never user data.
902
+ """
903
+ return bool(names) and all(name == f"column_{i}" for i, name in enumerate(names))
904
+
905
+
895
906
  def csv(p: Path, sep: str) -> pl.LazyFrame:
896
907
  """Read a CSV or TSV source as a LazyFrame.
897
908
 
@@ -901,8 +912,17 @@ def csv(p: Path, sep: str) -> pl.LazyFrame:
901
912
 
902
913
  Returns:
903
914
  LazyFrame over the file (no header inference; all columns raw).
915
+
916
+ Notes:
917
+ Headerless auto column names are remapped to the 1-based ``column_1``..
918
+ convention so existing table configs keep referencing the same names as
919
+ under polars 1.x (output parity).
904
920
  """
905
- return pl.scan_csv(source=p, separator=sep, has_header=False, infer_schema_length=None, truncate_ragged_lines=True)
921
+ lf: pl.LazyFrame = pl.scan_csv(source=p, separator=sep, has_header=False, infer_schema_length=None, truncate_ragged_lines=True)
922
+ schema_names: list[str] = list(lf.collect_schema().names())
923
+ if _polars_auto_column_names(schema_names):
924
+ lf = lf.rename(dict(zip(schema_names, (f"column_{i + 1}" for i in range(len(schema_names))), strict=True)))
925
+ return lf
906
926
 
907
927
 
908
928
  def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
@@ -915,6 +935,11 @@ def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
915
935
 
916
936
  Returns:
917
937
  LazyFrame over the sheet contents (no header inference).
938
+
939
+ Notes:
940
+ Headerless auto column names are remapped to the 1-based ``column_1``..
941
+ convention so existing table configs keep referencing the same names as
942
+ under polars 1.x (output parity).
918
943
  """
919
944
  df: pl.DataFrame = pl.read_excel(
920
945
  source=p,
@@ -923,6 +948,8 @@ def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
923
948
  has_header=False,
924
949
  infer_schema_length=None,
925
950
  )
951
+ if _polars_auto_column_names(df.columns):
952
+ df = df.rename(dict(zip(df.columns, (f"column_{i + 1}" for i in range(len(df.columns))), strict=True)))
926
953
  return df.lazy()
927
954
 
928
955
 
@@ -1116,6 +1143,51 @@ def trim(lf: pl.LazyFrame, regex: str = r"^column_\d+$") -> pl.LazyFrame:
1116
1143
  return lf.select(pl.exclude(regex))
1117
1144
 
1118
1145
 
1146
+ def qualifier_vocabulary_audit(lf: pl.LazyFrame, col: str, vocabulary: Iterable[str]) -> pl.LazyFrame:
1147
+ """Fail the build when an enum-ranged qualifier column holds off-vocabulary tokens.
1148
+
1149
+ Enum-ranged qualifiers (``ENUM_RANGED_QUALIFIERS``) bypass entity resolution, so nothing
1150
+ else checks that ``method: column`` cells -- or a ``method: value`` literal mutated by
1151
+ fill/regex/prefix/suffix -- carry tokens from the closed Biolink vocabulary. This op runs
1152
+ after the column's full encoding chain and raises a coded error naming the offending
1153
+ values instead of emitting KGX that fails validation much later, far from the cause.
1154
+
1155
+ Args:
1156
+ lf: Source LazyFrame.
1157
+ col: The enum-ranged qualifier column (named after the qualifier key).
1158
+ vocabulary: The closed set of permitted tokens (any iterable; sorted for the check).
1159
+
1160
+ Returns:
1161
+ The LazyFrame unchanged; the op exists for its check.
1162
+
1163
+ Notes:
1164
+ Nulls and null-like text (``""``, ``"NA"``, ...) are exempt: the Rust
1165
+ ``strip_nulls`` drops them at emission, so they never reach the output. Matching is
1166
+ exact -- the vocabularies are closed, and the config-time literal validator
1167
+ (``qualifier-bad-value``) applies the same rule.
1168
+ """
1169
+ values: pl.Series = lf.select(pl.col(col)).collect().get_column(col)
1170
+ if values.dtype.base_type() in (pl.List, pl.Array):
1171
+ values = values.explode()
1172
+ values = values.drop_nulls()
1173
+ if values.len() == 0:
1174
+ return lf
1175
+ allowed: list[str] = sorted(vocabulary)
1176
+ null_like: pl.Series = values.str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT))
1177
+ offending: pl.Series = values.filter(~null_like & ~values.is_in(allowed)).unique().sort()
1178
+ if offending.len() == 0:
1179
+ return lf
1180
+ shown: list[str] = offending.head(10).to_list()
1181
+ more: str = "..." if offending.len() > len(shown) else ""
1182
+ raise TablassertError(
1183
+ f"{col} has a closed Biolink vocabulary; {offending.len()} offending value(s) found in the built column: "
1184
+ f"{', '.join(repr(x) for x in shown)}{more}. Permitted values include: "
1185
+ f"{', '.join(repr(x) for x in allowed[:8])}... Null or blank cells are exempt "
1186
+ "(they are stripped from the output); fix the source table or the qualifier's encoding.",
1187
+ code="qualifier-vocabulary-violation",
1188
+ )
1189
+
1190
+
1119
1191
  def to_store(lf: pl.LazyFrame, p: Path, config_name: str) -> Path:
1120
1192
  """Collect and write a section LazyFrame to a parquet file.
1121
1193
 
@@ -1296,8 +1368,10 @@ class Tcode(Section):
1296
1368
  return [
1297
1369
  [self.node_prep(x, col) for x, col in node_columns],
1298
1370
  # Encode only: no pre-resolution snapshot and no NLP normalization, both of
1299
- # which exist to feed entity resolution these columns never undergo.
1300
- [self.encoding(x, x.qualifier) for x in literals],
1371
+ # which exist to feed entity resolution these columns never undergo. Each enum-
1372
+ # ranged column ends with a vocabulary audit: nothing else validates its cell
1373
+ # values against the closed Biolink range before KGX emission.
1374
+ [[*self.encoding(x, x.qualifier), (qualifier_vocabulary_audit, (str(x.qualifier), sorted(x.vocabulary or ())))] for x in literals],
1301
1375
  # Raw ``column_<n>`` columns are dead weight once the encodings above have
1302
1376
  # copied them into named slots; trim before resolution so the frame that
1303
1377
  # resolve_batch materializes and joins stays narrow.
@@ -1446,6 +1520,7 @@ PHASE_OF: dict[Callable, str] = {
1446
1520
  resolve_batch: "resolve",
1447
1521
  fullmap_audit: "qc",
1448
1522
  column: "encode",
1523
+ qualifier_vocabulary_audit: "encode",
1449
1524
  edge_category: "edge",
1450
1525
  publications: "provenance",
1451
1526
  retrieval_sources: "provenance",
@@ -1624,7 +1699,7 @@ def normalize(edges: pl.LazyFrame, col: str, names: list[str] | None = None, inf
1624
1699
  available: list[str] = edges.collect_schema().names()
1625
1700
  # `<col>_taxon_label` is not produced by every fullmap revision.
1626
1701
  pairs: list[tuple[str, str]] = [(c, n) for c, n in zip(cols, names, strict=True) if c in available]
1627
- nodes: pl.LazyFrame = edges.select([c for c, _ in pairs]).unique().rename(dict(pairs))
1702
+ nodes: pl.LazyFrame = edges.select([c for c, _ in pairs]).unique(maintain_order=True).rename(dict(pairs))
1628
1703
  # Ensures category has biolink: prefix.
1629
1704
  nodes = nodes.with_columns(
1630
1705
  pl.when(pl.col("category").str.starts_with("biolink:"))
@@ -1917,7 +1992,7 @@ def _write_ndjson(
1917
1992
  on_phase("write-nodes")
1918
1993
  with nodes_tmp.open("a", encoding="utf-8") as f:
1919
1994
  for subnode in subnodes:
1920
- eagernode: pl.DataFrame = subnode.collect().unique()
1995
+ eagernode: pl.DataFrame = subnode.collect().unique(maintain_order=True)
1921
1996
  eagernode.write_ndjson(f)
1922
1997
 
1923
1998
  # Phase: write-edges.
@@ -1925,7 +2000,7 @@ def _write_ndjson(
1925
2000
  on_phase("write-edges")
1926
2001
  with edges_tmp.open("a", encoding="utf-8") as f:
1927
2002
  for subedge in subedges:
1928
- eageredge: pl.DataFrame = subedge.collect().unique()
2003
+ eageredge: pl.DataFrame = subedge.collect().unique(maintain_order=True)
1929
2004
  eageredge.write_ndjson(f)
1930
2005
 
1931
2006
  # Phase: dedup.
@@ -269,6 +269,8 @@ OP_COST: dict[Callable, int] = {
269
269
  lib.suffix: 1,
270
270
  lib.math_op: 1,
271
271
  lib.split_list: 1,
272
+ # Vocabulary audit: one eager distinct-value pass over a single enum column.
273
+ lib.qualifier_vocabulary_audit: 1,
272
274
  # Clean: coerce_columns applies five coercion rules (schema resolve + renames + value
273
275
  # coercion) in one pass, so it outprices a single-expression clean op.
274
276
  lib.coerce_columns: 3,
File without changes
File without changes
File without changes
File without changes
File without changes