tablassert 20.0.0__tar.gz → 22.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-20.0.0 → tablassert-22.0.0}/PKG-INFO +12 -10
- {tablassert-20.0.0 → tablassert-22.0.0}/README.md +11 -9
- {tablassert-20.0.0 → tablassert-22.0.0}/pyproject.toml +1 -1
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/cli.py +113 -116
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/ingests.py +8 -3
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/lib.py +50 -34
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/models.py +19 -9
- {tablassert-20.0.0 → tablassert-22.0.0}/LICENSE +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/Cargo.lock +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/Cargo.toml +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/src/fullmap.rs +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/src/json.rs +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/src/lib.rs +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/src/ndjson.rs +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/src/nlp.rs +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/src/uuid.rs +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/__init__.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/biolink.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/cli_entry.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/coerce.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/data/infores_catalog.yaml +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/enums.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/errors.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/extras.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/fullmap.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/infores_registry.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/log.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/net.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/nlp.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/progress.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/qc.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/rig.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/runcache.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/study.py +0 -0
- {tablassert-20.0.0 → tablassert-22.0.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version:
|
|
3
|
+
Version: 22.0.0
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -163,15 +163,15 @@ Build the knowledge graph:
|
|
|
163
163
|
tablassert build-kg graph.yaml
|
|
164
164
|
```
|
|
165
165
|
|
|
166
|
-
Output is one JSON object per line: nodes with Biolink categories, edges with annotations.
|
|
166
|
+
Output is one JSON object per line: nodes with Biolink categories, edges with annotations. The edge sample below is abridged; real records also carry provenance fields such as `knowledge_level`, `agent_type`, `sources`, and `original_subject`/`original_object`.
|
|
167
167
|
|
|
168
168
|
```json
|
|
169
|
-
{"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"
|
|
170
|
-
{"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"]}
|
|
169
|
+
{"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
|
|
170
|
+
{"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
|
|
171
171
|
```
|
|
172
172
|
|
|
173
173
|
```json
|
|
174
|
-
{"id":"
|
|
174
|
+
{"id":"8f41c2d7-9b3e-35c1-a4e8-6d2f9a0c5b71","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"very_strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.yaml SECTION 1","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
|
|
175
175
|
```
|
|
176
176
|
|
|
177
177
|
See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
|
|
@@ -181,8 +181,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
181
181
|
- **Declarative YAML configuration**: define data transformations without writing code
|
|
182
182
|
- **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
|
|
183
183
|
CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
|
|
184
|
-
- **Optional quality control**: a four-stage audit (exact
|
|
185
|
-
low-confidence mappings
|
|
184
|
+
- **Optional quality control**: a four-stage audit (exact match, fuzzy match, abbreviation
|
|
185
|
+
expansion, then SapBERT embeddings) flags low-confidence mappings
|
|
186
186
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
187
187
|
and predicates
|
|
188
188
|
- **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
|
|
@@ -194,15 +194,17 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
194
194
|
pip install tablassert
|
|
195
195
|
```
|
|
196
196
|
|
|
197
|
-
|
|
198
|
-
|
|
197
|
+
To use the Python API, install Tablassert into your project's environment with `pip install
|
|
198
|
+
"tablassert"` (or `uv add tablassert`). To get the `tablassert` command as a standalone tool, use
|
|
199
|
+
`uv tool install "tablassert[cli]"`; note that a tool install is isolated and does not expose the
|
|
200
|
+
Python API to your project. Other extras are opt-in:
|
|
199
201
|
|
|
200
202
|
| Extra | Adds | Install |
|
|
201
203
|
| ----- | ---- | ------- |
|
|
202
204
|
| `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
|
|
203
205
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
204
206
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
205
|
-
| `qc` | four-stage QC audit (exact
|
|
207
|
+
| `qc` | four-stage QC audit (exact match, fuzzy match, abbreviation expansion, and SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
206
208
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
207
209
|
|
|
208
210
|
Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
|
|
@@ -109,15 +109,15 @@ Build the knowledge graph:
|
|
|
109
109
|
tablassert build-kg graph.yaml
|
|
110
110
|
```
|
|
111
111
|
|
|
112
|
-
Output is one JSON object per line: nodes with Biolink categories, edges with annotations.
|
|
112
|
+
Output is one JSON object per line: nodes with Biolink categories, edges with annotations. The edge sample below is abridged; real records also carry provenance fields such as `knowledge_level`, `agent_type`, `sources`, and `original_subject`/`original_object`.
|
|
113
113
|
|
|
114
114
|
```json
|
|
115
|
-
{"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"
|
|
116
|
-
{"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"]}
|
|
115
|
+
{"id":"HGNC:11998","name":"TP53","category":["biolink:Gene"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
|
|
116
|
+
{"id":"MONDO:0008903","name":"lung cancer","category":["biolink:Disease"],"in_taxon":["NCBITaxon:9606"],"provided_by":["infores:my-kg"]}
|
|
117
117
|
```
|
|
118
118
|
|
|
119
119
|
```json
|
|
120
|
-
{"id":"
|
|
120
|
+
{"id":"8f41c2d7-9b3e-35c1-a4e8-6d2f9a0c5b71","subject":"HGNC:11998","predicate":"biolink:associated_with","object":"MONDO:0008903","p_value":"1.0000e-03","statistical_significance_qualifier":"very_strongly_significant","has_supporting_studies":{"PMID:12345678":{"id":"PMID:12345678","name":"gene-disease.yaml SECTION 1","study_size":450,"has_study_results":[{"id":"row:2"}]}},"publications":["PMID:12345678"]}
|
|
121
121
|
```
|
|
122
122
|
|
|
123
123
|
See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full walkthrough.
|
|
@@ -127,8 +127,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
127
127
|
- **Declarative YAML configuration**: define data transformations without writing code
|
|
128
128
|
- **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
|
|
129
129
|
CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
|
|
130
|
-
- **Optional quality control**: a four-stage audit (exact
|
|
131
|
-
low-confidence mappings
|
|
130
|
+
- **Optional quality control**: a four-stage audit (exact match, fuzzy match, abbreviation
|
|
131
|
+
expansion, then SapBERT embeddings) flags low-confidence mappings
|
|
132
132
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
133
133
|
and predicates
|
|
134
134
|
- **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
|
|
@@ -140,15 +140,17 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
140
140
|
pip install tablassert
|
|
141
141
|
```
|
|
142
142
|
|
|
143
|
-
|
|
144
|
-
|
|
143
|
+
To use the Python API, install Tablassert into your project's environment with `pip install
|
|
144
|
+
"tablassert"` (or `uv add tablassert`). To get the `tablassert` command as a standalone tool, use
|
|
145
|
+
`uv tool install "tablassert[cli]"`; note that a tool install is isolated and does not expose the
|
|
146
|
+
Python API to your project. Other extras are opt-in:
|
|
145
147
|
|
|
146
148
|
| Extra | Adds | Install |
|
|
147
149
|
| ----- | ---- | ------- |
|
|
148
150
|
| `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
|
|
149
151
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
150
152
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
151
|
-
| `qc` | four-stage QC audit (exact
|
|
153
|
+
| `qc` | four-stage QC audit (exact match, fuzzy match, abbreviation expansion, and SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
152
154
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
153
155
|
|
|
154
156
|
Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "
|
|
3
|
+
version = "22.0.0"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -420,12 +420,12 @@ def build_graph_pipeline(
|
|
|
420
420
|
advance()
|
|
421
421
|
with RunCache() as cache:
|
|
422
422
|
for planned in plan.execution_order:
|
|
423
|
-
|
|
423
|
+
flat_index, x, op = pending[planned]
|
|
424
424
|
entry: PlanEntry = plan.entries[planned]
|
|
425
425
|
start(format_section_compact(x))
|
|
426
426
|
# on_phase drives the per-op sub-step indicator (cache → resolve → write ...);
|
|
427
427
|
# resume/snapshot are the producer/consumer seams the plan assigned this section.
|
|
428
|
-
results[
|
|
428
|
+
results[flat_index] = compile_subgraph(
|
|
429
429
|
op, on_phase=sub_step, resume=_run_cache_resume(cache, entry), snapshot=_run_cache_snapshot(cache, entry)
|
|
430
430
|
)
|
|
431
431
|
advance()
|
|
@@ -984,19 +984,52 @@ def _print_kgx_section(label: str, section: dict[str, Any], path: Path) -> None:
|
|
|
984
984
|
logger.info(f"validate-kgx {label}: {section['valid']}/{section['total']} valid")
|
|
985
985
|
|
|
986
986
|
|
|
987
|
+
def _print_infores_report(report: dict[str, Any], paths: dict[str, Path | None]) -> None:
|
|
988
|
+
"""Print one identity-half report: absent inputs, registry provenance, counts, examples."""
|
|
989
|
+
for label, absent in report["missing"].items():
|
|
990
|
+
if absent:
|
|
991
|
+
print(f"{label}: file not found ({paths.get(label)})", file=sys.stderr)
|
|
992
|
+
print(f"registry: {report['registry']['source']} ({report['registry']['entries']} entries)", file=sys.stderr)
|
|
993
|
+
counts: str = (
|
|
994
|
+
f"curies: {report['registered']} registered, {report['allowed']} allowed, "
|
|
995
|
+
f"{report['unregistered']} unregistered, {report['malformed']} malformed"
|
|
996
|
+
)
|
|
997
|
+
print(counts, file=sys.stderr)
|
|
998
|
+
logger.info(f"validate-kgx infores: {counts}")
|
|
999
|
+
for example in report["examples"][:3]:
|
|
1000
|
+
print(f" e.g. {example['curie']} ({example['where']}.{example['field']}): {example['problem']}", file=sys.stderr)
|
|
1001
|
+
|
|
1002
|
+
|
|
987
1003
|
@APP.command(name="validate-kgx")
|
|
988
1004
|
def validate_kgx_command(
|
|
989
1005
|
nodes: Annotated[Path, cyclopts.Parameter(name=["--nodes", "-n"])],
|
|
990
1006
|
edges: Annotated[Path, cyclopts.Parameter(name=["--edges", "-e"])],
|
|
991
1007
|
limit: Annotated[int, cyclopts.Parameter(name=["--limit"])] = 20,
|
|
992
1008
|
prune: Annotated[bool, cyclopts.Parameter(name=["--prune", "-p"])] = False,
|
|
1009
|
+
rig: Annotated[Path | None, cyclopts.Parameter(name=["--rig"])] = None,
|
|
1010
|
+
allow_infores: Annotated[list[str] | None, cyclopts.Parameter(name=["--allow-infores"], negative="", allow_leading_hyphen=False)] = None,
|
|
1011
|
+
registry: Annotated[Literal["snapshot", "refresh", "off"], cyclopts.Parameter(name=["--registry"])] = "snapshot",
|
|
1012
|
+
strict: Annotated[bool, cyclopts.Parameter(name=["--strict"])] = False,
|
|
993
1013
|
) -> None:
|
|
994
|
-
"""Validate built KGX NDJSON against the Biolink Model.
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
``category`` -- the same classes ``NCATSTranslator/translator-
|
|
998
|
-
and reports failures grouped by field and error type.
|
|
999
|
-
|
|
1014
|
+
"""Validate built KGX NDJSON against the Biolink Model and the Translator registry.
|
|
1015
|
+
|
|
1016
|
+
The structural half constructs every node and edge as the Biolink Pydantic class
|
|
1017
|
+
named by its own ``category`` -- the same classes ``NCATSTranslator/translator-
|
|
1018
|
+
ingests`` builds -- and reports failures grouped by field and error type. The
|
|
1019
|
+
identity half classifies every emitted knowledge-source identifier (edge
|
|
1020
|
+
``primary_knowledge_source``, ``sources[].resource_id`` and
|
|
1021
|
+
``sources[].upstream_resource_ids``, node ``provided_by``, and, with ``--rig``, the
|
|
1022
|
+
RIG document's infores values) against ``biolink/information-resource-registry``'s
|
|
1023
|
+
``infores_catalog.yaml`` (vendored as the bundled snapshot, so the default path is
|
|
1024
|
+
offline). Values without the ``infores:`` prefix are malformed, not unregistered.
|
|
1025
|
+
|
|
1026
|
+
The structural half exits non-zero when any record fails, so a build can be gated
|
|
1027
|
+
in CI. The identity half is advisory by default -- unregistered CURIEs are reported
|
|
1028
|
+
and the command exits 0, because Tablassert legitimately mints graph-local CURIEs
|
|
1029
|
+
that will never sit in the registry -- and ``--strict`` (CI gates) exits 1 while any
|
|
1030
|
+
unregistered CURIE remains; allowlist legitimate local ids with repeatable
|
|
1031
|
+
``--allow-infores``. Missing input files fail in every mode: a typo'd path must
|
|
1032
|
+
never read as a clean bill of health.
|
|
1000
1033
|
|
|
1001
1034
|
``--prune`` rewrites the EDGES file in place, keeping only edges that validate
|
|
1002
1035
|
strictly. Real defects (non-coercible values, extras the model will never declare,
|
|
@@ -1007,137 +1040,101 @@ def validate_kgx_command(
|
|
|
1007
1040
|
stay byte-identical; a clean file is not rewritten at all, and a missing edges file
|
|
1008
1041
|
is reported without ever being pruned. Nodes are validated and reported, never
|
|
1009
1042
|
pruned. The exit code stays strict over the post-prune state: after a prune, every
|
|
1010
|
-
kept edge is strictly valid, so the run is compliant exactly when the nodes are
|
|
1043
|
+
kept edge is strictly valid, so the run is compliant exactly when the nodes are,
|
|
1044
|
+
and the identity half classifies the PRUNED file -- what ships is what is checked.
|
|
1011
1045
|
|
|
1012
1046
|
Args:
|
|
1013
1047
|
nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
|
|
1014
1048
|
edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
|
|
1015
|
-
limit: Maximum example failures retained per file for the ``e.g.`` report lines
|
|
1016
|
-
every record is still validated
|
|
1017
|
-
reported and fails the run rather
|
|
1049
|
+
limit: Maximum example failures retained per file for the ``e.g.`` report lines
|
|
1050
|
+
(structural and identity examples alike); every record is still validated
|
|
1051
|
+
regardless of this cap. A missing file is reported and fails the run rather
|
|
1052
|
+
than reading as a pass.
|
|
1018
1053
|
prune: Rewrite the edges file in place, removing every edge record that is not
|
|
1019
1054
|
strictly KGX/Biolink-compliant, including the deliberate pending carryovers.
|
|
1020
1055
|
The nodes file is never modified; a clean edges file is left untouched.
|
|
1056
|
+
rig: Optional RIG yaml (``<name>_<version>.RIG.yaml``); its infores values are
|
|
1057
|
+
classified too.
|
|
1058
|
+
allow-infores: Registry-exempt CURIE, repeatable; use for the locally-minted
|
|
1059
|
+
graph identifier (``rig.infores()``) and other deliberate locals.
|
|
1060
|
+
registry: Registry source. ``snapshot`` (default) validates offline against the
|
|
1061
|
+
bundled copy; ``refresh`` fetches the live upstream catalog (network) and
|
|
1062
|
+
caches it under the user cache dir; ``off`` skips the identity half
|
|
1063
|
+
entirely (structure-only validation).
|
|
1064
|
+
strict: Exit non-zero when any unregistered CURIE remains. Without it the
|
|
1065
|
+
identity half is advisory and never fails a run once inputs exist.
|
|
1021
1066
|
"""
|
|
1022
1067
|
from tablassert.biolink import _validate_file, prune_kgx_edges, validate_kgx
|
|
1068
|
+
from tablassert.infores_registry import REGISTRY_RAW_URL, refresh_registry, validate_infores
|
|
1069
|
+
|
|
1070
|
+
ids: frozenset[str] | None = refresh_registry() if registry == "refresh" else None
|
|
1023
1071
|
|
|
1024
1072
|
if not prune:
|
|
1025
1073
|
report: dict[str, Any] = validate_kgx(nodes, edges, limit=limit)
|
|
1026
1074
|
print(f"biolink-model {report['biolink_version']}", file=sys.stderr)
|
|
1027
1075
|
_print_kgx_section("nodes", report["nodes"], nodes)
|
|
1028
1076
|
_print_kgx_section("edges", report["edges"], edges)
|
|
1029
|
-
|
|
1077
|
+
structural_ok: bool = report["ok"]
|
|
1078
|
+
structural_missing: bool = report["nodes"]["missing"] or report["edges"]["missing"]
|
|
1079
|
+
else:
|
|
1080
|
+
# The prune path validates nodes-only itself and drives prune_kgx_edges' own single
|
|
1081
|
+
# pass for both edges sections, so the edges file is never read and written twice.
|
|
1082
|
+
nodes_section: dict[str, Any] = _validate_file(nodes, edge=False, limit=limit)
|
|
1083
|
+
pruned_report: dict[str, Any] = prune_kgx_edges(edges, limit=limit)
|
|
1084
|
+
print(f"biolink-model {pruned_report['biolink_version']}", file=sys.stderr)
|
|
1085
|
+
_print_kgx_section("nodes", nodes_section, nodes)
|
|
1086
|
+
_print_kgx_section("edges", pruned_report["before"], edges)
|
|
1087
|
+
if pruned_report["before"]["missing"]:
|
|
1030
1088
|
print("KGX output is not Biolink-compliant.", file=sys.stderr)
|
|
1031
1089
|
raise SystemExit(1)
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1090
|
+
malformed_note: str = f" ({pruned_report['malformed']} malformed lines)" if pruned_report["malformed"] else ""
|
|
1091
|
+
print(f"edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} non-compliant records{malformed_note}", file=sys.stderr)
|
|
1092
|
+
logger.info(
|
|
1093
|
+
f"validate-kgx edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} (rewritten={pruned_report['rewritten']})"
|
|
1094
|
+
)
|
|
1095
|
+
for problem, count in pruned_report["dropped"]["problems"].items():
|
|
1096
|
+
print(f" {count:>9} {problem}", file=sys.stderr)
|
|
1097
|
+
for example in pruned_report["dropped"]["examples"][:3]:
|
|
1098
|
+
print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
|
|
1099
|
+
_print_kgx_section("edges", pruned_report["after"], edges)
|
|
1100
|
+
nodes_ok: bool = not nodes_section["missing"] and nodes_section["total"] == nodes_section["valid"]
|
|
1101
|
+
structural_ok = nodes_ok and pruned_report["ok"]
|
|
1102
|
+
structural_missing = nodes_section["missing"]
|
|
1103
|
+
|
|
1104
|
+
# The identity half classifies what ships: in prune mode the edges file has already
|
|
1105
|
+
# been rewritten, so the scan reads the post-prune state. When the structural half
|
|
1106
|
+
# already failed on absent inputs and no RIG was given, a second scan would only
|
|
1107
|
+
# echo the same file-not-found lines.
|
|
1108
|
+
infores_ok: bool = True
|
|
1109
|
+
if registry != "off" and (rig is not None or not structural_missing):
|
|
1110
|
+
infores_report: dict[str, Any] = validate_infores(nodes, edges, rig_path=rig, allow=tuple(allow_infores or ()), registry=ids, limit=limit)
|
|
1111
|
+
if ids is not None:
|
|
1112
|
+
infores_report["registry"]["source"] = f"{REGISTRY_RAW_URL} (refreshed)"
|
|
1113
|
+
_print_infores_report(infores_report, {"nodes": nodes, "edges": edges, "rig": rig})
|
|
1114
|
+
if any(infores_report["missing"].values()):
|
|
1115
|
+
print("infores validation failed: missing input files.", file=sys.stderr)
|
|
1116
|
+
infores_ok = False
|
|
1117
|
+
else:
|
|
1118
|
+
clean: bool = not infores_report["unregistered"] and not infores_report["malformed"]
|
|
1119
|
+
if strict:
|
|
1120
|
+
if infores_report["ok_strict"]:
|
|
1121
|
+
print("All emitted infores CURIEs are registered." if clean else "infores validation passed (strict).", file=sys.stderr)
|
|
1122
|
+
else:
|
|
1123
|
+
print("infores validation failed (strict): unregistered CURIEs remain.", file=sys.stderr)
|
|
1124
|
+
infores_ok = False
|
|
1125
|
+
elif not clean:
|
|
1126
|
+
print("infores warnings present; pass --strict to fail on them.", file=sys.stderr)
|
|
1127
|
+
else:
|
|
1128
|
+
print("All emitted infores CURIEs are registered.", file=sys.stderr)
|
|
1129
|
+
|
|
1130
|
+
if not structural_ok:
|
|
1043
1131
|
print("KGX output is not Biolink-compliant.", file=sys.stderr)
|
|
1044
1132
|
raise SystemExit(1)
|
|
1045
|
-
|
|
1046
|
-
print(f"edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} non-compliant records{malformed_note}", file=sys.stderr)
|
|
1047
|
-
logger.info(f"validate-kgx edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} (rewritten={pruned_report['rewritten']})")
|
|
1048
|
-
for problem, count in pruned_report["dropped"]["problems"].items():
|
|
1049
|
-
print(f" {count:>9} {problem}", file=sys.stderr)
|
|
1050
|
-
for example in pruned_report["dropped"]["examples"][:3]:
|
|
1051
|
-
print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
|
|
1052
|
-
_print_kgx_section("edges", pruned_report["after"], edges)
|
|
1053
|
-
nodes_ok: bool = not nodes_section["missing"] and nodes_section["total"] == nodes_section["valid"]
|
|
1054
|
-
if not (nodes_ok and pruned_report["ok"]):
|
|
1055
|
-
print("KGX output is not Biolink-compliant.", file=sys.stderr)
|
|
1133
|
+
if not infores_ok:
|
|
1056
1134
|
raise SystemExit(1)
|
|
1057
1135
|
print("KGX output is Biolink-compliant.", file=sys.stderr)
|
|
1058
1136
|
|
|
1059
1137
|
|
|
1060
|
-
@APP.command(name="validate-infores")
|
|
1061
|
-
def validate_infores_command(
|
|
1062
|
-
nodes: Annotated[Path, cyclopts.Parameter(name=["--nodes", "-n"])],
|
|
1063
|
-
edges: Annotated[Path, cyclopts.Parameter(name=["--edges", "-e"])],
|
|
1064
|
-
rig: Annotated[Path | None, cyclopts.Parameter(name=["--rig"])] = None,
|
|
1065
|
-
allow_infores: Annotated[list[str] | None, cyclopts.Parameter(name=["--allow-infores"], negative="", allow_leading_hyphen=False)] = None,
|
|
1066
|
-
registry: Annotated[Literal["snapshot", "refresh", "off"], cyclopts.Parameter(name=["--registry"])] = "snapshot",
|
|
1067
|
-
strict: Annotated[bool, cyclopts.Parameter(name=["--strict"])] = False,
|
|
1068
|
-
limit: Annotated[int, cyclopts.Parameter(name=["--limit"])] = 20,
|
|
1069
|
-
) -> None:
|
|
1070
|
-
"""Validate emitted infores CURIEs against the NCATS Translator registry.
|
|
1071
|
-
|
|
1072
|
-
Every knowledge-source identifier the build emits -- edge ``primary_knowledge_source``,
|
|
1073
|
-
``sources[].resource_id`` and ``sources[].upstream_resource_ids``, node ``provided_by``,
|
|
1074
|
-
and, with ``--rig``, the RIG document's infores values -- is classified against
|
|
1075
|
-
``biolink/information-resource-registry``'s ``infores_catalog.yaml`` (vendored as the
|
|
1076
|
-
bundled snapshot, so the default path is offline). Values without the ``infores:``
|
|
1077
|
-
prefix are malformed, not unregistered.
|
|
1078
|
-
|
|
1079
|
-
The default posture is advisory: unregistered CURIEs are reported and the command
|
|
1080
|
-
exits 0, because Tablassert legitimately mints graph-local CURIEs that will never sit
|
|
1081
|
-
in the registry. ``--strict`` (CI gates) exits 1 when any unregistered CURIE remains;
|
|
1082
|
-
allowlist legitimate local ids with repeatable ``--allow-infores``. Missing input
|
|
1083
|
-
files fail in every mode: a typo'd path must never read as a clean bill of health.
|
|
1084
|
-
|
|
1085
|
-
Args:
|
|
1086
|
-
nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
|
|
1087
|
-
edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
|
|
1088
|
-
rig: Optional RIG yaml (``<name>_<version>.RIG.yaml``); its infores values are
|
|
1089
|
-
classified too.
|
|
1090
|
-
allow-infores: Registry-exempt CURIE, repeatable; use for the locally-minted
|
|
1091
|
-
graph identifier (``rig.infores()``) and other deliberate locals.
|
|
1092
|
-
registry: Registry source. ``snapshot`` (default) validates offline against the
|
|
1093
|
-
bundled copy; ``refresh`` fetches the live upstream catalog (network) and
|
|
1094
|
-
caches it under the user cache dir; ``off`` skips membership classification
|
|
1095
|
-
and reports structure only.
|
|
1096
|
-
strict: Exit non-zero when any unregistered CURIE remains. Without it the run
|
|
1097
|
-
is advisory and always exits 0 once inputs exist.
|
|
1098
|
-
limit: Maximum number of example unregistered/malformed CURIEs retained for the
|
|
1099
|
-
``e.g.`` report lines; every observation is still classified regardless.
|
|
1100
|
-
"""
|
|
1101
|
-
from tablassert.infores_registry import REGISTRY_RAW_URL, refresh_registry, validate_infores
|
|
1102
|
-
|
|
1103
|
-
ids: frozenset[str] | None = None
|
|
1104
|
-
if registry == "refresh":
|
|
1105
|
-
ids = refresh_registry()
|
|
1106
|
-
report: dict[str, Any] = validate_infores(
|
|
1107
|
-
nodes, edges, rig_path=rig, allow=tuple(allow_infores or ()), registry=ids, check_registry=registry != "off", limit=limit
|
|
1108
|
-
)
|
|
1109
|
-
if ids is not None:
|
|
1110
|
-
report["registry"]["source"] = f"{REGISTRY_RAW_URL} (refreshed)"
|
|
1111
|
-
for label, absent in report["missing"].items():
|
|
1112
|
-
if absent:
|
|
1113
|
-
path_label = {"nodes": nodes, "edges": edges, "rig": rig}.get(label)
|
|
1114
|
-
print(f"{label}: file not found ({path_label})", file=sys.stderr)
|
|
1115
|
-
if any(report["missing"].values()):
|
|
1116
|
-
print("infores validation failed: missing input files.", file=sys.stderr)
|
|
1117
|
-
raise SystemExit(1)
|
|
1118
|
-
print(f"registry: {report['registry']['source']} ({report['registry']['entries']} entries)", file=sys.stderr)
|
|
1119
|
-
counts: str = (
|
|
1120
|
-
f"curies: {report['registered']} registered, {report['allowed']} allowed, "
|
|
1121
|
-
f"{report['unregistered']} unregistered, {report['malformed']} malformed"
|
|
1122
|
-
)
|
|
1123
|
-
print(counts, file=sys.stderr)
|
|
1124
|
-
logger.info(f"validate-infores: {counts}")
|
|
1125
|
-
for example in report["examples"][:3]:
|
|
1126
|
-
print(f" e.g. {example['curie']} ({example['where']}.{example['field']}): {example['problem']}", file=sys.stderr)
|
|
1127
|
-
clean: bool = not report["unregistered"] and not report["malformed"]
|
|
1128
|
-
passed: str = "infores membership check skipped (--registry off)." if registry == "off" else "All emitted infores CURIEs are registered."
|
|
1129
|
-
if strict:
|
|
1130
|
-
if report["ok_strict"]:
|
|
1131
|
-
print(passed if clean else "infores validation passed (strict).", file=sys.stderr)
|
|
1132
|
-
return
|
|
1133
|
-
print("infores validation failed (strict): unregistered CURIEs remain.", file=sys.stderr)
|
|
1134
|
-
raise SystemExit(1)
|
|
1135
|
-
if not clean:
|
|
1136
|
-
print("infores warnings present; pass --strict to fail on them.", file=sys.stderr)
|
|
1137
|
-
return
|
|
1138
|
-
print(passed, file=sys.stderr)
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
1138
|
@APP.command(name="quick-map")
|
|
1142
1139
|
def quick_map_command(
|
|
1143
1140
|
# negative="" suppresses cyclopts' auto-generated --empty-terms help row: the reset
|
|
@@ -82,12 +82,14 @@ def to_yaml(p: Path, data: object) -> None:
|
|
|
82
82
|
yaml.dump(data, f, Dumper=_IndentedSafeDumper, sort_keys=False, default_flow_style=False, allow_unicode=True, width=120)
|
|
83
83
|
|
|
84
84
|
|
|
85
|
-
def to_sections(instructions: dict[str, Any], table: Path) -> list[
|
|
85
|
+
def to_sections(instructions: dict[str, Any], table: Path) -> list[dict[str, Any]]:
|
|
86
86
|
"""Expand a table config dict into a list of section dicts.
|
|
87
87
|
|
|
88
88
|
Each section is the deep merge of the table-level ``template`` over a
|
|
89
89
|
per-section entry. The originating ``table`` path is stamped into each
|
|
90
|
-
section as ``config
|
|
90
|
+
section as ``config``, and the section's 1-based position in the file
|
|
91
|
+
(document order) as ``section_index`` so provenance can name the inlined
|
|
92
|
+
Study after its origin ("<table config> SECTION <n>").
|
|
91
93
|
|
|
92
94
|
Args:
|
|
93
95
|
instructions: Parsed YAML config with optional ``template`` and
|
|
@@ -100,4 +102,7 @@ def to_sections(instructions: dict[str, Any], table: Path) -> list[list[dict[str
|
|
|
100
102
|
template: dict[str, Any] = instructions.get("template", {})
|
|
101
103
|
template["config"] = table
|
|
102
104
|
sections: list[dict[str, Any]] = instructions.get("sections", [{}])
|
|
103
|
-
|
|
105
|
+
merged: list[dict[str, Any]] = [fastmerge(deepcopy(template), x) for x in sections]
|
|
106
|
+
for index, section in enumerate(merged, start=1):
|
|
107
|
+
section["section_index"] = index
|
|
108
|
+
return merged
|
|
@@ -471,8 +471,8 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, study_name: str | N
|
|
|
471
471
|
The struct is shaped by the current Biolink Model (4.4.4+):
|
|
472
472
|
|
|
473
473
|
* ``Study.id`` is a real identifier -- the section's publication CURIE -- and
|
|
474
|
-
``Study.name`` a DISJOINT human label (
|
|
475
|
-
are never composed into one string.
|
|
474
|
+
``Study.name`` a DISJOINT human label (the table config file plus the section's
|
|
475
|
+
1-based index); the two are never composed into one string.
|
|
476
476
|
* Study-level metadata declared as annotations (``study_size``, ``study_cohort``,
|
|
477
477
|
``study_context``, ``study_date_range``, ``study_method_description``,
|
|
478
478
|
``study_method_types`` -- see :data:`biolink.STUDY_METADATA_FIELDS`) lands as
|
|
@@ -504,8 +504,8 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, study_name: str | N
|
|
|
504
504
|
lf: Edges LazyFrame after annotation and provenance ops.
|
|
505
505
|
study_id: Stable study identifier (publication CURIE, or the config stem when
|
|
506
506
|
unidentified).
|
|
507
|
-
study_name: Human-readable study label (
|
|
508
|
-
``None`` to omit it.
|
|
507
|
+
study_name: Human-readable study label (table config file plus 1-based section
|
|
508
|
+
index), or ``None`` to omit it.
|
|
509
509
|
identified: Whether ``study_id`` names a real publication rather than falling
|
|
510
510
|
back to the config stem. ``False`` lets a contentless study be skipped.
|
|
511
511
|
|
|
@@ -608,11 +608,14 @@ def retrieval_sources(
|
|
|
608
608
|
Mirrors ``build_association_knowledge_sources()`` from
|
|
609
609
|
``translator-ingests/util/biolink.py``: the primary knowledge source lists the
|
|
610
610
|
upstream resources, and each upstream resource additionally appears as its own
|
|
611
|
-
``supporting_data_source`` entry.
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
611
|
+
``supporting_data_source`` entry. URL placement is determined by ``upstream_urls``
|
|
612
|
+
whenever it is given (the derived build path passes it unconditionally): each
|
|
613
|
+
listed upstream entry carries its own ``source_record_urls``, because the section's
|
|
614
|
+
record URLs describe the upstream repository's downloadable record, not the
|
|
615
|
+
transforming primary. The primary entry still emits ``urls`` that are not source
|
|
616
|
+
records -- the per-edge EdgeLinkouts template when ``--linkouts`` is set. Without
|
|
617
|
+
``upstream_urls`` (override sections that set no explicit mapping), the primary
|
|
618
|
+
entry carries ``urls`` verbatim and the supporting entries stay bare.
|
|
616
619
|
|
|
617
620
|
When ``explicit`` is given, exactly those entries are emitted, in order, and
|
|
618
621
|
``primary``/``upstream``/``urls``/``upstream_urls`` are ignored. Each entry
|
|
@@ -626,7 +629,9 @@ def retrieval_sources(
|
|
|
626
629
|
lf: Source LazyFrame.
|
|
627
630
|
primary: Infores CURIE of the primary knowledge source.
|
|
628
631
|
upstream: Infores CURIEs of upstream/supporting data sources.
|
|
629
|
-
urls:
|
|
632
|
+
urls: URLs for the primary entry (when ``upstream_urls`` is set these must not be
|
|
633
|
+
source records -- e.g. the per-edge EdgeLinkouts template -- since record URLs
|
|
634
|
+
ride the supporting entries in that mode).
|
|
630
635
|
upstream_urls: Optional per-upstream source record URLs keyed by infores CURIE.
|
|
631
636
|
explicit: Optional explicit entry templates emitted verbatim, in order.
|
|
632
637
|
|
|
@@ -639,7 +644,7 @@ def retrieval_sources(
|
|
|
639
644
|
for entry in explicit
|
|
640
645
|
]
|
|
641
646
|
elif upstream_urls is not None:
|
|
642
|
-
entries = [_retrieval_source(primary, "primary_knowledge_source", upstream)]
|
|
647
|
+
entries = [_retrieval_source(primary, "primary_knowledge_source", upstream, urls)]
|
|
643
648
|
entries.extend(_retrieval_source(x, "supporting_data_source", urls=upstream_urls.get(x)) for x in upstream)
|
|
644
649
|
else:
|
|
645
650
|
entries = [_retrieval_source(primary, "primary_knowledge_source", upstream, urls)]
|
|
@@ -1250,6 +1255,10 @@ class Tcode(Section):
|
|
|
1250
1255
|
name: str | None = Field(None)
|
|
1251
1256
|
infores: str | None = Field(None)
|
|
1252
1257
|
linkouts: bool = Field(False)
|
|
1258
|
+
section_index: int = Field(
|
|
1259
|
+
1,
|
|
1260
|
+
description="1-based position of this section within its table config file (document order); stamped by ingests.to_sections and used to name the inlined Study.",
|
|
1261
|
+
)
|
|
1253
1262
|
qc_similarity_threshold: float = Field(0.5)
|
|
1254
1263
|
qc_fuzzy_ratio_threshold: float = Field(70.0)
|
|
1255
1264
|
qc_fuzzy_partial_threshold: float = Field(80.0)
|
|
@@ -1454,19 +1463,28 @@ class Tcode(Section):
|
|
|
1454
1463
|
# (rig.source_info.infores_id); there is no implicit derivation from the graph
|
|
1455
1464
|
# name, so a RIG and its edges can never disagree about the source identity.
|
|
1456
1465
|
primary_knowledge_source: str | None = self.infores
|
|
1457
|
-
#
|
|
1458
|
-
#
|
|
1459
|
-
#
|
|
1460
|
-
#
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1466
|
+
# Record URLs: the section's declared source URLs. On the derived path they
|
|
1467
|
+
# describe the UPSTREAM record the data was extracted from, so they are emitted
|
|
1468
|
+
# on the supporting entry of the publication repository (infores:pubmed-central
|
|
1469
|
+
# for PMC sections) and the primary entry -- the transforming resource --
|
|
1470
|
+
# carries only the EdgeLinkouts template when --linkouts is set. A section with
|
|
1471
|
+
# ANY provenance.override is author-controlled (DAKP hand-writes the same
|
|
1472
|
+
# template into override.sources): an upstream_source_record_urls mapping
|
|
1473
|
+
# re-homes URLs exactly as specified (primary gets nothing), any other override
|
|
1474
|
+
# keeps the legacy primary-entry emission, and the flag injects nothing there.
|
|
1475
|
+
record_urls: list[str] = [str(u) for u in self.source.url]
|
|
1464
1476
|
upstream_ids = override.upstream_resource_ids if override else upstream_resource_ids(self.provenance.repo)
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1477
|
+
primary_urls: list[str]
|
|
1478
|
+
upstream_urls: dict[str, list[str]] | None
|
|
1479
|
+
if override is None:
|
|
1480
|
+
upstream_urls = {upstream_ids[0]: record_urls}
|
|
1481
|
+
primary_urls = [EDGE_LINKOUT_URL_TEMPLATE] if self.linkouts else []
|
|
1482
|
+
elif override.upstream_source_record_urls is not None:
|
|
1483
|
+
upstream_urls = {key: [str(u) for u in urls] for key, urls in override.upstream_source_record_urls.items()}
|
|
1484
|
+
primary_urls = []
|
|
1485
|
+
else:
|
|
1486
|
+
upstream_urls = None
|
|
1487
|
+
primary_urls = record_urls
|
|
1470
1488
|
# An explicit `sources` template replaces the derived primary/upstream
|
|
1471
1489
|
# emission entirely; the model already forbids combining it with
|
|
1472
1490
|
# `upstream_resource_ids`/`upstream_source_record_urls`.
|
|
@@ -1476,19 +1494,17 @@ class Tcode(Section):
|
|
|
1476
1494
|
knowledge_level = override.knowledge_level if override else self.provenance.knowledge_level
|
|
1477
1495
|
agent_type = override.agent_type if override else self.provenance.agent_type
|
|
1478
1496
|
publication_values = override.publications if override else [publication_curie(self.provenance.repo, self.provenance.publication or "")]
|
|
1479
|
-
# The study is the table itself: one publication, one
|
|
1480
|
-
# publication CURIE is the study's IDENTIFIER; the
|
|
1481
|
-
# is its human-readable NAME.
|
|
1497
|
+
# The study is the table section itself: one publication, one config section.
|
|
1498
|
+
# The publication CURIE is the study's IDENTIFIER; the originating table config
|
|
1499
|
+
# file plus the section's 1-based index is its human-readable NAME (e.g.
|
|
1500
|
+
# "KANG12.yaml SECTION 2"). The two stay disjoint fields and are never composed
|
|
1482
1501
|
# into one string, so neither ever duplicates the other.
|
|
1483
|
-
sheet: str | None = self.source.sheet if self.source.kind == Files.EXCEL else None # pyright: ignore
|
|
1484
1502
|
# No publication means no study to name: the fallback id is the config stem (the
|
|
1485
1503
|
# YAML filename without its extension), not an identifier, so
|
|
1486
1504
|
# `inline_supporting_study` skips the struct unless it has content to carry.
|
|
1487
1505
|
identified: bool = bool(publication_values)
|
|
1488
1506
|
study_id: str = publication_values[0] if publication_values else self.config.stem
|
|
1489
|
-
study_name: str
|
|
1490
|
-
if study_name == study_id:
|
|
1491
|
-
study_name = None
|
|
1507
|
+
study_name: str = f"{self.config.name} SECTION {self.section_index}"
|
|
1492
1508
|
# `use_enum_values` stores the override's enum keys/values as plain strings.
|
|
1493
1509
|
category_override: dict[str, str] | None = (
|
|
1494
1510
|
{obj: f"biolink:{pinned}" for obj, pinned in self.statement.category_override.items()} if self.statement.category_override else None
|
|
@@ -1500,10 +1516,10 @@ class Tcode(Section):
|
|
|
1500
1516
|
(value, ("agent_type", agent_type)),
|
|
1501
1517
|
# Retrieval provenance lives only in the nested `sources` list (Biolink
|
|
1502
1518
|
# RetrievalSource); current translator-ingests emits no flat
|
|
1503
|
-
# `primary_knowledge_source` scalar, so neither do we.
|
|
1504
|
-
#
|
|
1505
|
-
#
|
|
1506
|
-
#
|
|
1519
|
+
# `primary_knowledge_source` scalar, so neither do we. The derived path maps
|
|
1520
|
+
# the section's record URLs onto the upstream repository's supporting entry
|
|
1521
|
+
# (see above); an explicit `sources` template (override.sources) replaces
|
|
1522
|
+
# the whole derivation.
|
|
1507
1523
|
(retrieval_sources, (primary_knowledge_source, upstream_ids, primary_urls, upstream_urls, explicit_sources))
|
|
1508
1524
|
if primary_knowledge_source
|
|
1509
1525
|
else None,
|
|
@@ -47,11 +47,20 @@ else:
|
|
|
47
47
|
pl = LazyModule("polars")
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
# Deprecated config keys -> guidance.
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
DEPRECATED_KEYS: dict[str, str] = {
|
|
50
|
+
# Deprecated config keys -> guidance. The before-hook below only warns and returns the data
|
|
51
|
+
# unchanged: a key that is no longer a valid field is STILL rejected by extra="forbid" unless the
|
|
52
|
+
# hook also pops/translates it, so registering a key here supplies the warning half only. That is
|
|
53
|
+
# the right half for a removed feature: the config still fails, but it says why and what to do.
|
|
54
|
+
DEPRECATED_KEYS: dict[str, str] = {
|
|
55
|
+
"no_map": (
|
|
56
|
+
"`no_map` is removed: subject, object, and CURIE-ranged qualifier values resolve against the fullmap again. "
|
|
57
|
+
"The mode emitted node ids verbatim and pinned each one to `biolink:NamedThing`, which flattened every derived "
|
|
58
|
+
"edge to `biolink:Association`. Tables that already carry CURIEs need nothing else, because the fullmap indexes "
|
|
59
|
+
"CURIEs and their equivalent identifiers as terms, so a CURIE cell resolves back to its own CURIE. The usual "
|
|
60
|
+
"resolution rules still apply: `taxon`, `prioritize`/`avoid`, `exclude_prefixes`/`exclude_regex`, and a blank "
|
|
61
|
+
"cell still drops the row."
|
|
62
|
+
)
|
|
63
|
+
}
|
|
55
64
|
|
|
56
65
|
|
|
57
66
|
def _shown(name: str, target: str) -> str:
|
|
@@ -94,10 +103,11 @@ class TablaBase(BaseModel):
|
|
|
94
103
|
"""Soft-warn when a raw input key is registered in ``DEPRECATED_KEYS``.
|
|
95
104
|
|
|
96
105
|
Pure observation: returns ``data`` unchanged and never rejects (``extra="forbid"``
|
|
97
|
-
still governs
|
|
98
|
-
|
|
106
|
+
still governs keys that name no live field, so a removed feature's key warns and then
|
|
107
|
+
still fails; that pairing is the point, since silently dropping the key would build the
|
|
108
|
+
output the removal exists to prevent). ``no_map`` is the registered entry today.
|
|
99
109
|
Note: with ``validate_assignment=True`` pydantic also re-runs this on attribute set,
|
|
100
|
-
so a registered key warns on assignment too
|
|
110
|
+
so a registered key warns on assignment too.
|
|
101
111
|
"""
|
|
102
112
|
if isinstance(data, dict):
|
|
103
113
|
for key in data:
|
|
@@ -138,7 +148,7 @@ class BaseSource(TablaBase):
|
|
|
138
148
|
url: list[HttpUrl] = Field(
|
|
139
149
|
...,
|
|
140
150
|
min_length=1,
|
|
141
|
-
description="One or more remote source URL(s) recorded as provenance
|
|
151
|
+
description="One or more remote source URL(s) recorded as provenance. Derived sections emit them in the edge `sources` list under the upstream repository supporting entry's `source_record_urls` list (infores:pubmed-central for PMC sections) and in the RIG. When `provenance.override.upstream_source_record_urls` is set, these URLs serve the RIG only and the per-upstream mapping determines edge placement instead; any other provenance.override keeps the primary-entry emission. Format-validated only; not fetched.",
|
|
142
152
|
)
|
|
143
153
|
|
|
144
154
|
rows: list[NonNegativeInt] | None = Field(None, description="Zero-based row indices kept after any row_slice crop.", examples=[[0, 2, 5]])
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|