tablassert 20.0.0__tar.gz → 21.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-20.0.0 → tablassert-21.0.0}/PKG-INFO +4 -4
- {tablassert-20.0.0 → tablassert-21.0.0}/README.md +3 -3
- {tablassert-20.0.0 → tablassert-21.0.0}/pyproject.toml +1 -1
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/cli.py +113 -116
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/errors.py +3 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/ingests.py +8 -3
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/lib.py +138 -41
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/models.py +111 -4
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/runcache.py +3 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/LICENSE +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/Cargo.lock +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/Cargo.toml +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/fullmap.rs +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/json.rs +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/lib.rs +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/ndjson.rs +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/nlp.rs +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/uuid.rs +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/__init__.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/biolink.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/cli_entry.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/coerce.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/data/infores_catalog.yaml +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/enums.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/extras.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/fullmap.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/infores_registry.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/log.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/net.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/nlp.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/progress.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/qc.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/rig.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/study.py +0 -0
- {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version:
|
|
3
|
+
Version: 21.0.0
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -181,8 +181,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
181
181
|
- **Declarative YAML configuration**: define data transformations without writing code
|
|
182
182
|
- **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
|
|
183
183
|
CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
|
|
184
|
-
- **Optional quality control**: a four-stage audit (exact
|
|
185
|
-
low-confidence mappings
|
|
184
|
+
- **Optional quality control**: a four-stage audit (exact match, fuzzy match, abbreviation
|
|
185
|
+
expansion, then SapBERT embeddings) flags low-confidence mappings
|
|
186
186
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
187
187
|
and predicates
|
|
188
188
|
- **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
|
|
@@ -202,7 +202,7 @@ provides the `tablassert` command. Other extras are opt-in:
|
|
|
202
202
|
| `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
|
|
203
203
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
204
204
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
205
|
-
| `qc` | four-stage QC audit (exact
|
|
205
|
+
| `qc` | four-stage QC audit (exact match, fuzzy match, abbreviation expansion, and SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
206
206
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
207
207
|
|
|
208
208
|
Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
|
|
@@ -127,8 +127,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
|
|
|
127
127
|
- **Declarative YAML configuration**: define data transformations without writing code
|
|
128
128
|
- **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
|
|
129
129
|
CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
|
|
130
|
-
- **Optional quality control**: a four-stage audit (exact
|
|
131
|
-
low-confidence mappings
|
|
130
|
+
- **Optional quality control**: a four-stage audit (exact match, fuzzy match, abbreviation
|
|
131
|
+
expansion, then SapBERT embeddings) flags low-confidence mappings
|
|
132
132
|
- **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
|
|
133
133
|
and predicates
|
|
134
134
|
- **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
|
|
@@ -148,7 +148,7 @@ provides the `tablassert` command. Other extras are opt-in:
|
|
|
148
148
|
| `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
|
|
149
149
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
150
150
|
| `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
151
|
-
| `qc` | four-stage QC audit (exact
|
|
151
|
+
| `qc` | four-stage QC audit (exact match, fuzzy match, abbreviation expansion, and SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
152
152
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
153
153
|
|
|
154
154
|
Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "
|
|
3
|
+
version = "21.0.0"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -420,12 +420,12 @@ def build_graph_pipeline(
|
|
|
420
420
|
advance()
|
|
421
421
|
with RunCache() as cache:
|
|
422
422
|
for planned in plan.execution_order:
|
|
423
|
-
|
|
423
|
+
flat_index, x, op = pending[planned]
|
|
424
424
|
entry: PlanEntry = plan.entries[planned]
|
|
425
425
|
start(format_section_compact(x))
|
|
426
426
|
# on_phase drives the per-op sub-step indicator (cache → resolve → write ...);
|
|
427
427
|
# resume/snapshot are the producer/consumer seams the plan assigned this section.
|
|
428
|
-
results[
|
|
428
|
+
results[flat_index] = compile_subgraph(
|
|
429
429
|
op, on_phase=sub_step, resume=_run_cache_resume(cache, entry), snapshot=_run_cache_snapshot(cache, entry)
|
|
430
430
|
)
|
|
431
431
|
advance()
|
|
@@ -984,19 +984,52 @@ def _print_kgx_section(label: str, section: dict[str, Any], path: Path) -> None:
|
|
|
984
984
|
logger.info(f"validate-kgx {label}: {section['valid']}/{section['total']} valid")
|
|
985
985
|
|
|
986
986
|
|
|
987
|
+
def _print_infores_report(report: dict[str, Any], paths: dict[str, Path | None]) -> None:
|
|
988
|
+
"""Print one identity-half report: absent inputs, registry provenance, counts, examples."""
|
|
989
|
+
for label, absent in report["missing"].items():
|
|
990
|
+
if absent:
|
|
991
|
+
print(f"{label}: file not found ({paths.get(label)})", file=sys.stderr)
|
|
992
|
+
print(f"registry: {report['registry']['source']} ({report['registry']['entries']} entries)", file=sys.stderr)
|
|
993
|
+
counts: str = (
|
|
994
|
+
f"curies: {report['registered']} registered, {report['allowed']} allowed, "
|
|
995
|
+
f"{report['unregistered']} unregistered, {report['malformed']} malformed"
|
|
996
|
+
)
|
|
997
|
+
print(counts, file=sys.stderr)
|
|
998
|
+
logger.info(f"validate-kgx infores: {counts}")
|
|
999
|
+
for example in report["examples"][:3]:
|
|
1000
|
+
print(f" e.g. {example['curie']} ({example['where']}.{example['field']}): {example['problem']}", file=sys.stderr)
|
|
1001
|
+
|
|
1002
|
+
|
|
987
1003
|
@APP.command(name="validate-kgx")
|
|
988
1004
|
def validate_kgx_command(
|
|
989
1005
|
nodes: Annotated[Path, cyclopts.Parameter(name=["--nodes", "-n"])],
|
|
990
1006
|
edges: Annotated[Path, cyclopts.Parameter(name=["--edges", "-e"])],
|
|
991
1007
|
limit: Annotated[int, cyclopts.Parameter(name=["--limit"])] = 20,
|
|
992
1008
|
prune: Annotated[bool, cyclopts.Parameter(name=["--prune", "-p"])] = False,
|
|
1009
|
+
rig: Annotated[Path | None, cyclopts.Parameter(name=["--rig"])] = None,
|
|
1010
|
+
allow_infores: Annotated[list[str] | None, cyclopts.Parameter(name=["--allow-infores"], negative="", allow_leading_hyphen=False)] = None,
|
|
1011
|
+
registry: Annotated[Literal["snapshot", "refresh", "off"], cyclopts.Parameter(name=["--registry"])] = "snapshot",
|
|
1012
|
+
strict: Annotated[bool, cyclopts.Parameter(name=["--strict"])] = False,
|
|
993
1013
|
) -> None:
|
|
994
|
-
"""Validate built KGX NDJSON against the Biolink Model.
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
``category`` -- the same classes ``NCATSTranslator/translator-
|
|
998
|
-
and reports failures grouped by field and error type.
|
|
999
|
-
|
|
1014
|
+
"""Validate built KGX NDJSON against the Biolink Model and the Translator registry.
|
|
1015
|
+
|
|
1016
|
+
The structural half constructs every node and edge as the Biolink Pydantic class
|
|
1017
|
+
named by its own ``category`` -- the same classes ``NCATSTranslator/translator-
|
|
1018
|
+
ingests`` builds -- and reports failures grouped by field and error type. The
|
|
1019
|
+
identity half classifies every emitted knowledge-source identifier (edge
|
|
1020
|
+
``primary_knowledge_source``, ``sources[].resource_id`` and
|
|
1021
|
+
``sources[].upstream_resource_ids``, node ``provided_by``, and, with ``--rig``, the
|
|
1022
|
+
RIG document's infores values) against ``biolink/information-resource-registry``'s
|
|
1023
|
+
``infores_catalog.yaml`` (vendored as the bundled snapshot, so the default path is
|
|
1024
|
+
offline). Values without the ``infores:`` prefix are malformed, not unregistered.
|
|
1025
|
+
|
|
1026
|
+
The structural half exits non-zero when any record fails, so a build can be gated
|
|
1027
|
+
in CI. The identity half is advisory by default -- unregistered CURIEs are reported
|
|
1028
|
+
and the command exits 0, because Tablassert legitimately mints graph-local CURIEs
|
|
1029
|
+
that will never sit in the registry -- and ``--strict`` (CI gates) exits 1 while any
|
|
1030
|
+
unregistered CURIE remains; allowlist legitimate local ids with repeatable
|
|
1031
|
+
``--allow-infores``. Missing input files fail in every mode: a typo'd path must
|
|
1032
|
+
never read as a clean bill of health.
|
|
1000
1033
|
|
|
1001
1034
|
``--prune`` rewrites the EDGES file in place, keeping only edges that validate
|
|
1002
1035
|
strictly. Real defects (non-coercible values, extras the model will never declare,
|
|
@@ -1007,137 +1040,101 @@ def validate_kgx_command(
|
|
|
1007
1040
|
stay byte-identical; a clean file is not rewritten at all, and a missing edges file
|
|
1008
1041
|
is reported without ever being pruned. Nodes are validated and reported, never
|
|
1009
1042
|
pruned. The exit code stays strict over the post-prune state: after a prune, every
|
|
1010
|
-
kept edge is strictly valid, so the run is compliant exactly when the nodes are
|
|
1043
|
+
kept edge is strictly valid, so the run is compliant exactly when the nodes are,
|
|
1044
|
+
and the identity half classifies the PRUNED file -- what ships is what is checked.
|
|
1011
1045
|
|
|
1012
1046
|
Args:
|
|
1013
1047
|
nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
|
|
1014
1048
|
edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
|
|
1015
|
-
limit: Maximum example failures retained per file for the ``e.g.`` report lines
|
|
1016
|
-
every record is still validated
|
|
1017
|
-
reported and fails the run rather
|
|
1049
|
+
limit: Maximum example failures retained per file for the ``e.g.`` report lines
|
|
1050
|
+
(structural and identity examples alike); every record is still validated
|
|
1051
|
+
regardless of this cap. A missing file is reported and fails the run rather
|
|
1052
|
+
than reading as a pass.
|
|
1018
1053
|
prune: Rewrite the edges file in place, removing every edge record that is not
|
|
1019
1054
|
strictly KGX/Biolink-compliant, including the deliberate pending carryovers.
|
|
1020
1055
|
The nodes file is never modified; a clean edges file is left untouched.
|
|
1056
|
+
rig: Optional RIG yaml (``<name>_<version>.RIG.yaml``); its infores values are
|
|
1057
|
+
classified too.
|
|
1058
|
+
allow-infores: Registry-exempt CURIE, repeatable; use for the locally-minted
|
|
1059
|
+
graph identifier (``rig.infores()``) and other deliberate locals.
|
|
1060
|
+
registry: Registry source. ``snapshot`` (default) validates offline against the
|
|
1061
|
+
bundled copy; ``refresh`` fetches the live upstream catalog (network) and
|
|
1062
|
+
caches it under the user cache dir; ``off`` skips the identity half
|
|
1063
|
+
entirely (structure-only validation).
|
|
1064
|
+
strict: Exit non-zero when any unregistered CURIE remains. Without it the
|
|
1065
|
+
identity half is advisory and never fails a run once inputs exist.
|
|
1021
1066
|
"""
|
|
1022
1067
|
from tablassert.biolink import _validate_file, prune_kgx_edges, validate_kgx
|
|
1068
|
+
from tablassert.infores_registry import REGISTRY_RAW_URL, refresh_registry, validate_infores
|
|
1069
|
+
|
|
1070
|
+
ids: frozenset[str] | None = refresh_registry() if registry == "refresh" else None
|
|
1023
1071
|
|
|
1024
1072
|
if not prune:
|
|
1025
1073
|
report: dict[str, Any] = validate_kgx(nodes, edges, limit=limit)
|
|
1026
1074
|
print(f"biolink-model {report['biolink_version']}", file=sys.stderr)
|
|
1027
1075
|
_print_kgx_section("nodes", report["nodes"], nodes)
|
|
1028
1076
|
_print_kgx_section("edges", report["edges"], edges)
|
|
1029
|
-
|
|
1077
|
+
structural_ok: bool = report["ok"]
|
|
1078
|
+
structural_missing: bool = report["nodes"]["missing"] or report["edges"]["missing"]
|
|
1079
|
+
else:
|
|
1080
|
+
# The prune path validates nodes-only itself and drives prune_kgx_edges' own single
|
|
1081
|
+
# pass for both edges sections, so the edges file is never read and written twice.
|
|
1082
|
+
nodes_section: dict[str, Any] = _validate_file(nodes, edge=False, limit=limit)
|
|
1083
|
+
pruned_report: dict[str, Any] = prune_kgx_edges(edges, limit=limit)
|
|
1084
|
+
print(f"biolink-model {pruned_report['biolink_version']}", file=sys.stderr)
|
|
1085
|
+
_print_kgx_section("nodes", nodes_section, nodes)
|
|
1086
|
+
_print_kgx_section("edges", pruned_report["before"], edges)
|
|
1087
|
+
if pruned_report["before"]["missing"]:
|
|
1030
1088
|
print("KGX output is not Biolink-compliant.", file=sys.stderr)
|
|
1031
1089
|
raise SystemExit(1)
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1090
|
+
malformed_note: str = f" ({pruned_report['malformed']} malformed lines)" if pruned_report["malformed"] else ""
|
|
1091
|
+
print(f"edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} non-compliant records{malformed_note}", file=sys.stderr)
|
|
1092
|
+
logger.info(
|
|
1093
|
+
f"validate-kgx edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} (rewritten={pruned_report['rewritten']})"
|
|
1094
|
+
)
|
|
1095
|
+
for problem, count in pruned_report["dropped"]["problems"].items():
|
|
1096
|
+
print(f" {count:>9} {problem}", file=sys.stderr)
|
|
1097
|
+
for example in pruned_report["dropped"]["examples"][:3]:
|
|
1098
|
+
print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
|
|
1099
|
+
_print_kgx_section("edges", pruned_report["after"], edges)
|
|
1100
|
+
nodes_ok: bool = not nodes_section["missing"] and nodes_section["total"] == nodes_section["valid"]
|
|
1101
|
+
structural_ok = nodes_ok and pruned_report["ok"]
|
|
1102
|
+
structural_missing = nodes_section["missing"]
|
|
1103
|
+
|
|
1104
|
+
# The identity half classifies what ships: in prune mode the edges file has already
|
|
1105
|
+
# been rewritten, so the scan reads the post-prune state. When the structural half
|
|
1106
|
+
# already failed on absent inputs and no RIG was given, a second scan would only
|
|
1107
|
+
# echo the same file-not-found lines.
|
|
1108
|
+
infores_ok: bool = True
|
|
1109
|
+
if registry != "off" and (rig is not None or not structural_missing):
|
|
1110
|
+
infores_report: dict[str, Any] = validate_infores(nodes, edges, rig_path=rig, allow=tuple(allow_infores or ()), registry=ids, limit=limit)
|
|
1111
|
+
if ids is not None:
|
|
1112
|
+
infores_report["registry"]["source"] = f"{REGISTRY_RAW_URL} (refreshed)"
|
|
1113
|
+
_print_infores_report(infores_report, {"nodes": nodes, "edges": edges, "rig": rig})
|
|
1114
|
+
if any(infores_report["missing"].values()):
|
|
1115
|
+
print("infores validation failed: missing input files.", file=sys.stderr)
|
|
1116
|
+
infores_ok = False
|
|
1117
|
+
else:
|
|
1118
|
+
clean: bool = not infores_report["unregistered"] and not infores_report["malformed"]
|
|
1119
|
+
if strict:
|
|
1120
|
+
if infores_report["ok_strict"]:
|
|
1121
|
+
print("All emitted infores CURIEs are registered." if clean else "infores validation passed (strict).", file=sys.stderr)
|
|
1122
|
+
else:
|
|
1123
|
+
print("infores validation failed (strict): unregistered CURIEs remain.", file=sys.stderr)
|
|
1124
|
+
infores_ok = False
|
|
1125
|
+
elif not clean:
|
|
1126
|
+
print("infores warnings present; pass --strict to fail on them.", file=sys.stderr)
|
|
1127
|
+
else:
|
|
1128
|
+
print("All emitted infores CURIEs are registered.", file=sys.stderr)
|
|
1129
|
+
|
|
1130
|
+
if not structural_ok:
|
|
1043
1131
|
print("KGX output is not Biolink-compliant.", file=sys.stderr)
|
|
1044
1132
|
raise SystemExit(1)
|
|
1045
|
-
|
|
1046
|
-
print(f"edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} non-compliant records{malformed_note}", file=sys.stderr)
|
|
1047
|
-
logger.info(f"validate-kgx edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} (rewritten={pruned_report['rewritten']})")
|
|
1048
|
-
for problem, count in pruned_report["dropped"]["problems"].items():
|
|
1049
|
-
print(f" {count:>9} {problem}", file=sys.stderr)
|
|
1050
|
-
for example in pruned_report["dropped"]["examples"][:3]:
|
|
1051
|
-
print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
|
|
1052
|
-
_print_kgx_section("edges", pruned_report["after"], edges)
|
|
1053
|
-
nodes_ok: bool = not nodes_section["missing"] and nodes_section["total"] == nodes_section["valid"]
|
|
1054
|
-
if not (nodes_ok and pruned_report["ok"]):
|
|
1055
|
-
print("KGX output is not Biolink-compliant.", file=sys.stderr)
|
|
1133
|
+
if not infores_ok:
|
|
1056
1134
|
raise SystemExit(1)
|
|
1057
1135
|
print("KGX output is Biolink-compliant.", file=sys.stderr)
|
|
1058
1136
|
|
|
1059
1137
|
|
|
1060
|
-
@APP.command(name="validate-infores")
|
|
1061
|
-
def validate_infores_command(
|
|
1062
|
-
nodes: Annotated[Path, cyclopts.Parameter(name=["--nodes", "-n"])],
|
|
1063
|
-
edges: Annotated[Path, cyclopts.Parameter(name=["--edges", "-e"])],
|
|
1064
|
-
rig: Annotated[Path | None, cyclopts.Parameter(name=["--rig"])] = None,
|
|
1065
|
-
allow_infores: Annotated[list[str] | None, cyclopts.Parameter(name=["--allow-infores"], negative="", allow_leading_hyphen=False)] = None,
|
|
1066
|
-
registry: Annotated[Literal["snapshot", "refresh", "off"], cyclopts.Parameter(name=["--registry"])] = "snapshot",
|
|
1067
|
-
strict: Annotated[bool, cyclopts.Parameter(name=["--strict"])] = False,
|
|
1068
|
-
limit: Annotated[int, cyclopts.Parameter(name=["--limit"])] = 20,
|
|
1069
|
-
) -> None:
|
|
1070
|
-
"""Validate emitted infores CURIEs against the NCATS Translator registry.
|
|
1071
|
-
|
|
1072
|
-
Every knowledge-source identifier the build emits -- edge ``primary_knowledge_source``,
|
|
1073
|
-
``sources[].resource_id`` and ``sources[].upstream_resource_ids``, node ``provided_by``,
|
|
1074
|
-
and, with ``--rig``, the RIG document's infores values -- is classified against
|
|
1075
|
-
``biolink/information-resource-registry``'s ``infores_catalog.yaml`` (vendored as the
|
|
1076
|
-
bundled snapshot, so the default path is offline). Values without the ``infores:``
|
|
1077
|
-
prefix are malformed, not unregistered.
|
|
1078
|
-
|
|
1079
|
-
The default posture is advisory: unregistered CURIEs are reported and the command
|
|
1080
|
-
exits 0, because Tablassert legitimately mints graph-local CURIEs that will never sit
|
|
1081
|
-
in the registry. ``--strict`` (CI gates) exits 1 when any unregistered CURIE remains;
|
|
1082
|
-
allowlist legitimate local ids with repeatable ``--allow-infores``. Missing input
|
|
1083
|
-
files fail in every mode: a typo'd path must never read as a clean bill of health.
|
|
1084
|
-
|
|
1085
|
-
Args:
|
|
1086
|
-
nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
|
|
1087
|
-
edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
|
|
1088
|
-
rig: Optional RIG yaml (``<name>_<version>.RIG.yaml``); its infores values are
|
|
1089
|
-
classified too.
|
|
1090
|
-
allow-infores: Registry-exempt CURIE, repeatable; use for the locally-minted
|
|
1091
|
-
graph identifier (``rig.infores()``) and other deliberate locals.
|
|
1092
|
-
registry: Registry source. ``snapshot`` (default) validates offline against the
|
|
1093
|
-
bundled copy; ``refresh`` fetches the live upstream catalog (network) and
|
|
1094
|
-
caches it under the user cache dir; ``off`` skips membership classification
|
|
1095
|
-
and reports structure only.
|
|
1096
|
-
strict: Exit non-zero when any unregistered CURIE remains. Without it the run
|
|
1097
|
-
is advisory and always exits 0 once inputs exist.
|
|
1098
|
-
limit: Maximum number of example unregistered/malformed CURIEs retained for the
|
|
1099
|
-
``e.g.`` report lines; every observation is still classified regardless.
|
|
1100
|
-
"""
|
|
1101
|
-
from tablassert.infores_registry import REGISTRY_RAW_URL, refresh_registry, validate_infores
|
|
1102
|
-
|
|
1103
|
-
ids: frozenset[str] | None = None
|
|
1104
|
-
if registry == "refresh":
|
|
1105
|
-
ids = refresh_registry()
|
|
1106
|
-
report: dict[str, Any] = validate_infores(
|
|
1107
|
-
nodes, edges, rig_path=rig, allow=tuple(allow_infores or ()), registry=ids, check_registry=registry != "off", limit=limit
|
|
1108
|
-
)
|
|
1109
|
-
if ids is not None:
|
|
1110
|
-
report["registry"]["source"] = f"{REGISTRY_RAW_URL} (refreshed)"
|
|
1111
|
-
for label, absent in report["missing"].items():
|
|
1112
|
-
if absent:
|
|
1113
|
-
path_label = {"nodes": nodes, "edges": edges, "rig": rig}.get(label)
|
|
1114
|
-
print(f"{label}: file not found ({path_label})", file=sys.stderr)
|
|
1115
|
-
if any(report["missing"].values()):
|
|
1116
|
-
print("infores validation failed: missing input files.", file=sys.stderr)
|
|
1117
|
-
raise SystemExit(1)
|
|
1118
|
-
print(f"registry: {report['registry']['source']} ({report['registry']['entries']} entries)", file=sys.stderr)
|
|
1119
|
-
counts: str = (
|
|
1120
|
-
f"curies: {report['registered']} registered, {report['allowed']} allowed, "
|
|
1121
|
-
f"{report['unregistered']} unregistered, {report['malformed']} malformed"
|
|
1122
|
-
)
|
|
1123
|
-
print(counts, file=sys.stderr)
|
|
1124
|
-
logger.info(f"validate-infores: {counts}")
|
|
1125
|
-
for example in report["examples"][:3]:
|
|
1126
|
-
print(f" e.g. {example['curie']} ({example['where']}.{example['field']}): {example['problem']}", file=sys.stderr)
|
|
1127
|
-
clean: bool = not report["unregistered"] and not report["malformed"]
|
|
1128
|
-
passed: str = "infores membership check skipped (--registry off)." if registry == "off" else "All emitted infores CURIEs are registered."
|
|
1129
|
-
if strict:
|
|
1130
|
-
if report["ok_strict"]:
|
|
1131
|
-
print(passed if clean else "infores validation passed (strict).", file=sys.stderr)
|
|
1132
|
-
return
|
|
1133
|
-
print("infores validation failed (strict): unregistered CURIEs remain.", file=sys.stderr)
|
|
1134
|
-
raise SystemExit(1)
|
|
1135
|
-
if not clean:
|
|
1136
|
-
print("infores warnings present; pass --strict to fail on them.", file=sys.stderr)
|
|
1137
|
-
return
|
|
1138
|
-
print(passed, file=sys.stderr)
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
1138
|
@APP.command(name="quick-map")
|
|
1142
1139
|
def quick_map_command(
|
|
1143
1140
|
# negative="" suppresses cyclopts' auto-generated --empty-terms help row: the reset
|
|
@@ -63,6 +63,9 @@ TablassertErrorCodes = Literal[
|
|
|
63
63
|
"provenance-missing-publication",
|
|
64
64
|
"provenance-publication-and-override",
|
|
65
65
|
"encoding-list-method-removed",
|
|
66
|
+
"no-map-dead-config",
|
|
67
|
+
"no-map-bad-curie",
|
|
68
|
+
"no-map-curie-violation",
|
|
66
69
|
"annotation-split-by-requires-column",
|
|
67
70
|
"annotation-split-by-empty",
|
|
68
71
|
"field-disabled",
|
|
@@ -82,12 +82,14 @@ def to_yaml(p: Path, data: object) -> None:
|
|
|
82
82
|
yaml.dump(data, f, Dumper=_IndentedSafeDumper, sort_keys=False, default_flow_style=False, allow_unicode=True, width=120)
|
|
83
83
|
|
|
84
84
|
|
|
85
|
-
def to_sections(instructions: dict[str, Any], table: Path) -> list[
|
|
85
|
+
def to_sections(instructions: dict[str, Any], table: Path) -> list[dict[str, Any]]:
|
|
86
86
|
"""Expand a table config dict into a list of section dicts.
|
|
87
87
|
|
|
88
88
|
Each section is the deep merge of the table-level ``template`` over a
|
|
89
89
|
per-section entry. The originating ``table`` path is stamped into each
|
|
90
|
-
section as ``config
|
|
90
|
+
section as ``config``, and the section's 1-based position in the file
|
|
91
|
+
(document order) as ``section_index`` so provenance can name the inlined
|
|
92
|
+
Study after its origin ("<table config> SECTION <n>").
|
|
91
93
|
|
|
92
94
|
Args:
|
|
93
95
|
instructions: Parsed YAML config with optional ``template`` and
|
|
@@ -100,4 +102,7 @@ def to_sections(instructions: dict[str, Any], table: Path) -> list[list[dict[str
|
|
|
100
102
|
template: dict[str, Any] = instructions.get("template", {})
|
|
101
103
|
template["config"] = table
|
|
102
104
|
sections: list[dict[str, Any]] = instructions.get("sections", [{}])
|
|
103
|
-
|
|
105
|
+
merged: list[dict[str, Any]] = [fastmerge(deepcopy(template), x) for x in sections]
|
|
106
|
+
for index, section in enumerate(merged, start=1):
|
|
107
|
+
section["section_index"] = index
|
|
108
|
+
return merged
|
|
@@ -48,7 +48,7 @@ from tablassert.enums import EncodingMethods, Files, Functions, InformationResou
|
|
|
48
48
|
from tablassert.errors import TablassertError
|
|
49
49
|
from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
|
|
50
50
|
from tablassert.log import cat
|
|
51
|
-
from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
|
|
51
|
+
from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section, is_curie
|
|
52
52
|
from tablassert.nlp import level_one, level_two
|
|
53
53
|
from tablassert.qc import fullmap_audit
|
|
54
54
|
from tablassert.rig import (
|
|
@@ -471,8 +471,8 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, study_name: str | N
|
|
|
471
471
|
The struct is shaped by the current Biolink Model (4.4.4+):
|
|
472
472
|
|
|
473
473
|
* ``Study.id`` is a real identifier -- the section's publication CURIE -- and
|
|
474
|
-
``Study.name`` a DISJOINT human label (
|
|
475
|
-
are never composed into one string.
|
|
474
|
+
``Study.name`` a DISJOINT human label (the table config file plus the section's
|
|
475
|
+
1-based index); the two are never composed into one string.
|
|
476
476
|
* Study-level metadata declared as annotations (``study_size``, ``study_cohort``,
|
|
477
477
|
``study_context``, ``study_date_range``, ``study_method_description``,
|
|
478
478
|
``study_method_types`` -- see :data:`biolink.STUDY_METADATA_FIELDS`) lands as
|
|
@@ -504,8 +504,8 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, study_name: str | N
|
|
|
504
504
|
lf: Edges LazyFrame after annotation and provenance ops.
|
|
505
505
|
study_id: Stable study identifier (publication CURIE, or the config stem when
|
|
506
506
|
unidentified).
|
|
507
|
-
study_name: Human-readable study label (
|
|
508
|
-
``None`` to omit it.
|
|
507
|
+
study_name: Human-readable study label (table config file plus 1-based section
|
|
508
|
+
index), or ``None`` to omit it.
|
|
509
509
|
identified: Whether ``study_id`` names a real publication rather than falling
|
|
510
510
|
back to the config stem. ``False`` lets a contentless study be skipped.
|
|
511
511
|
|
|
@@ -608,11 +608,14 @@ def retrieval_sources(
|
|
|
608
608
|
Mirrors ``build_association_knowledge_sources()`` from
|
|
609
609
|
``translator-ingests/util/biolink.py``: the primary knowledge source lists the
|
|
610
610
|
upstream resources, and each upstream resource additionally appears as its own
|
|
611
|
-
``supporting_data_source`` entry.
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
611
|
+
``supporting_data_source`` entry. URL placement is determined by ``upstream_urls``
|
|
612
|
+
whenever it is given (the derived build path passes it unconditionally): each
|
|
613
|
+
listed upstream entry carries its own ``source_record_urls``, because the section's
|
|
614
|
+
record URLs describe the upstream repository's downloadable record, not the
|
|
615
|
+
transforming primary. The primary entry still emits ``urls`` that are not source
|
|
616
|
+
records -- the per-edge EdgeLinkouts template when ``--linkouts`` is set. Without
|
|
617
|
+
``upstream_urls`` (override sections that set no explicit mapping), the primary
|
|
618
|
+
entry carries ``urls`` verbatim and the supporting entries stay bare.
|
|
616
619
|
|
|
617
620
|
When ``explicit`` is given, exactly those entries are emitted, in order, and
|
|
618
621
|
``primary``/``upstream``/``urls``/``upstream_urls`` are ignored. Each entry
|
|
@@ -626,7 +629,9 @@ def retrieval_sources(
|
|
|
626
629
|
lf: Source LazyFrame.
|
|
627
630
|
primary: Infores CURIE of the primary knowledge source.
|
|
628
631
|
upstream: Infores CURIEs of upstream/supporting data sources.
|
|
629
|
-
urls:
|
|
632
|
+
urls: URLs for the primary entry (when ``upstream_urls`` is set these must not be
|
|
633
|
+
source records -- e.g. the per-edge EdgeLinkouts template -- since record URLs
|
|
634
|
+
ride the supporting entries in that mode).
|
|
630
635
|
upstream_urls: Optional per-upstream source record URLs keyed by infores CURIE.
|
|
631
636
|
explicit: Optional explicit entry templates emitted verbatim, in order.
|
|
632
637
|
|
|
@@ -639,7 +644,7 @@ def retrieval_sources(
|
|
|
639
644
|
for entry in explicit
|
|
640
645
|
]
|
|
641
646
|
elif upstream_urls is not None:
|
|
642
|
-
entries = [_retrieval_source(primary, "primary_knowledge_source", upstream)]
|
|
647
|
+
entries = [_retrieval_source(primary, "primary_knowledge_source", upstream, urls)]
|
|
643
648
|
entries.extend(_retrieval_source(x, "supporting_data_source", urls=upstream_urls.get(x)) for x in upstream)
|
|
644
649
|
else:
|
|
645
650
|
entries = [_retrieval_source(primary, "primary_knowledge_source", upstream, urls)]
|
|
@@ -1215,6 +1220,65 @@ def qualifier_vocabulary_audit(lf: pl.LazyFrame, col: str, vocabulary: Iterable[
|
|
|
1215
1220
|
)
|
|
1216
1221
|
|
|
1217
1222
|
|
|
1223
|
+
def curie_audit(lf: pl.LazyFrame, col: str, section: str, config_file: str) -> pl.LazyFrame:
|
|
1224
|
+
"""Finalize a ``no_map`` node column: blank rows drop, non-CURIE values fail, category is synthesized.
|
|
1225
|
+
|
|
1226
|
+
Columns configured with ``no_map: true`` bypass entity resolution, so nothing else
|
|
1227
|
+
(a) drops rows whose cell is blank (a missing node id can never form an edge, exactly
|
|
1228
|
+
like an unresolved subject/object), (b) rejects cells that are not CURIE-shaped -- the
|
|
1229
|
+
encoded value becomes the KGX node id verbatim, so garbage would ship as an identifier --
|
|
1230
|
+
or (c) produces the ``<col>_category`` column downstream stages require: without a
|
|
1231
|
+
fullmap lookup there is no role to derive, so the column is pinned to
|
|
1232
|
+
``biolink:NamedThing`` and edge derivation degrades to ``biolink:Association`` exactly
|
|
1233
|
+
like the documented no-role-lookup fallback in :func:`edge_category`.
|
|
1234
|
+
|
|
1235
|
+
Args:
|
|
1236
|
+
lf: Source LazyFrame.
|
|
1237
|
+
col: The no_map node column (``subject``/``object`` or a CURIE-ranged qualifier key).
|
|
1238
|
+
section: Short section hash (for log context).
|
|
1239
|
+
config_file: Originating config file (for log context).
|
|
1240
|
+
|
|
1241
|
+
Returns:
|
|
1242
|
+
LazyFrame with blank/null-like rows dropped and ``<col>_category`` added.
|
|
1243
|
+
|
|
1244
|
+
Notes:
|
|
1245
|
+
A non-blank value that is not CURIE-shaped raises a coded error naming the
|
|
1246
|
+
offending values (mirroring ``qualifier_vocabulary_audit``); blank and null-like
|
|
1247
|
+
cells (``""``, ``"NA"``, ...) are dropped with a loud per-column count instead,
|
|
1248
|
+
mirroring how an unresolvable subject/object row is dropped, never emitted.
|
|
1249
|
+
"""
|
|
1250
|
+
values: pl.Series = lf.select(pl.col(col)).collect().get_column(col)
|
|
1251
|
+
if values.dtype.base_type() in (pl.List, pl.Array):
|
|
1252
|
+
values = values.explode()
|
|
1253
|
+
blank_mask: pl.Series = values.is_null() | values.str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT))
|
|
1254
|
+
# Judge CURIE shape on the distinct non-blank values only: the audit runs once per
|
|
1255
|
+
# section, and map_elements is a per-element Python call that must not scale with
|
|
1256
|
+
# row count when one bad cell repeats a million times.
|
|
1257
|
+
distinct: pl.Series = values.filter(~blank_mask).unique()
|
|
1258
|
+
offending: pl.Series = distinct.filter(~distinct.str.strip_chars().map_elements(is_curie, return_dtype=pl.Boolean)).sort()
|
|
1259
|
+
if offending.len() > 0:
|
|
1260
|
+
shown: list[str] = offending.head(10).to_list()
|
|
1261
|
+
more: str = "..." if offending.len() > len(shown) else ""
|
|
1262
|
+
raise TablassertError(
|
|
1263
|
+
f"{col} is configured with `no_map: true` so its values are used verbatim as node CURIEs, but "
|
|
1264
|
+
f"{offending.len()} non-blank value(s) in the built column are not CURIE-shaped: "
|
|
1265
|
+
f"{', '.join(repr(x) for x in shown)}{more}. Fix the source table or the encoding "
|
|
1266
|
+
"(prefix/suffix can form a CURIE from raw cells).",
|
|
1267
|
+
code="no-map-curie-violation",
|
|
1268
|
+
)
|
|
1269
|
+
dropped: int = int(blank_mask.sum())
|
|
1270
|
+
if dropped:
|
|
1271
|
+
logger.info(
|
|
1272
|
+
"no_map column {col} in {config} ({hash}) dropped {n} blank row(s): a missing CURIE cannot form a node",
|
|
1273
|
+
col=col,
|
|
1274
|
+
config=config_file,
|
|
1275
|
+
hash=section,
|
|
1276
|
+
n=dropped,
|
|
1277
|
+
)
|
|
1278
|
+
present: pl.Expr = pl.col(col).is_not_null() & ~pl.col(col).str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT))
|
|
1279
|
+
return lf.filter(present).with_columns(pl.lit("biolink:NamedThing").alias(f"{col}_category"))
|
|
1280
|
+
|
|
1281
|
+
|
|
1218
1282
|
def to_store(lf: pl.LazyFrame, p: Path, config_name: str) -> Path:
|
|
1219
1283
|
"""Collect and write a section LazyFrame to a parquet file.
|
|
1220
1284
|
|
|
@@ -1250,6 +1314,10 @@ class Tcode(Section):
|
|
|
1250
1314
|
name: str | None = Field(None)
|
|
1251
1315
|
infores: str | None = Field(None)
|
|
1252
1316
|
linkouts: bool = Field(False)
|
|
1317
|
+
section_index: int = Field(
|
|
1318
|
+
1,
|
|
1319
|
+
description="1-based position of this section within its table config file (document order); stamped by ingests.to_sections and used to name the inlined Study.",
|
|
1320
|
+
)
|
|
1253
1321
|
qc_similarity_threshold: float = Field(0.5)
|
|
1254
1322
|
qc_fuzzy_ratio_threshold: float = Field(70.0)
|
|
1255
1323
|
qc_fuzzy_partial_threshold: float = Field(80.0)
|
|
@@ -1369,16 +1437,25 @@ class Tcode(Section):
|
|
|
1369
1437
|
enabled.
|
|
1370
1438
|
"""
|
|
1371
1439
|
# Subject/object/qualifiers share one resolve_batch call instead of one per column.
|
|
1372
|
-
#
|
|
1373
|
-
# Biolink vocabulary
|
|
1374
|
-
# token `increased` into the CURIE `UMLS:C0205217`, which the slot
|
|
1440
|
+
# Two kinds of columns never reach it: enum-ranged qualifiers, whose range is a
|
|
1441
|
+
# closed Biolink vocabulary (sending them through the fullmap would turn the
|
|
1442
|
+
# required token `increased` into the CURIE `UMLS:C0205217`, which the slot
|
|
1443
|
+
# rejects), and no_map nodes, whose values are already CURIEs used verbatim.
|
|
1375
1444
|
qualifiers: list[Qualifier] = self.statement.qualifiers or []
|
|
1445
|
+
subj_obj: list[tuple[NodeEncoding, str]] = [(self.statement.subject, "subject"), (self.statement.object, "object")]
|
|
1376
1446
|
node_columns: list[tuple[NodeEncoding, str]] = [
|
|
1377
|
-
(
|
|
1378
|
-
(self.statement.object, "object"),
|
|
1447
|
+
*[(x, col) for x, col in subj_obj if not x.no_map],
|
|
1379
1448
|
*[(x, x.qualifier) for x in qualifiers if x.resolved],
|
|
1380
1449
|
]
|
|
1381
|
-
|
|
1450
|
+
# no_map subject/object plus CURIE-ranged qualifiers that opted out (a no_map
|
|
1451
|
+
# enum-ranged qualifier is rejected at config time). Their encoded values become
|
|
1452
|
+
# node ids verbatim: encode only, then curie_audit drops blank rows, fails on
|
|
1453
|
+
# non-CURIE cells, and synthesizes the category column edge derivation needs.
|
|
1454
|
+
verbatim: list[tuple[NodeEncoding, str]] = [
|
|
1455
|
+
*[(x, col) for x, col in subj_obj if x.no_map],
|
|
1456
|
+
*[(x, x.qualifier) for x in qualifiers if not x.resolved and x.no_map],
|
|
1457
|
+
]
|
|
1458
|
+
literals: list[Qualifier] = [x for x in qualifiers if not x.resolved and not x.no_map]
|
|
1382
1459
|
# A nullable qualifier keeps its edge when the cell is blank or unresolvable (the
|
|
1383
1460
|
# column stays null and the null-stripper omits the key); subject/object and
|
|
1384
1461
|
# strict qualifiers drop the row as before.
|
|
@@ -1401,6 +1478,18 @@ class Tcode(Section):
|
|
|
1401
1478
|
# ranged column ends with a vocabulary audit: nothing else validates its cell
|
|
1402
1479
|
# values against the closed Biolink range before KGX emission.
|
|
1403
1480
|
[[*self.encoding(x, x.qualifier), (qualifier_vocabulary_audit, (str(x.qualifier), sorted(x.vocabulary or ())))] for x in literals],
|
|
1481
|
+
# no_map columns are encoded exactly like enum-ranged literals (no NLP
|
|
1482
|
+
# normalization -- it exists only to feed entity resolution these columns
|
|
1483
|
+
# never undergo), then finalized by curie_audit: blank rows dropped,
|
|
1484
|
+
# non-CURIE cells failed loudly, and the pinned `biolink:NamedThing`
|
|
1485
|
+
# category added for downstream edge derivation. The `<col>_pre_resolution`
|
|
1486
|
+
# snapshot rides along because `_collect_subframes` discovers node columns
|
|
1487
|
+
# through it at graph-compile time; without it a verbatim subject/object
|
|
1488
|
+
# would never be split out of the edges into nodes.
|
|
1489
|
+
[
|
|
1490
|
+
[*self.encoding(x, col), (column, (f"{col}_pre_resolution", col)), (curie_audit, (col, self.store.stem, self.config.name))]
|
|
1491
|
+
for x, col in verbatim
|
|
1492
|
+
],
|
|
1404
1493
|
# Raw ``column_<n>`` columns are dead weight once the encodings above have
|
|
1405
1494
|
# copied them into named slots; trim before resolution so the frame that
|
|
1406
1495
|
# resolve_batch materializes and joins stays narrow.
|
|
@@ -1454,19 +1543,28 @@ class Tcode(Section):
|
|
|
1454
1543
|
# (rig.source_info.infores_id); there is no implicit derivation from the graph
|
|
1455
1544
|
# name, so a RIG and its edges can never disagree about the source identity.
|
|
1456
1545
|
primary_knowledge_source: str | None = self.infores
|
|
1457
|
-
#
|
|
1458
|
-
#
|
|
1459
|
-
#
|
|
1460
|
-
#
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1546
|
+
# Record URLs: the section's declared source URLs. On the derived path they
|
|
1547
|
+
# describe the UPSTREAM record the data was extracted from, so they are emitted
|
|
1548
|
+
# on the supporting entry of the publication repository (infores:pubmed-central
|
|
1549
|
+
# for PMC sections) and the primary entry -- the transforming resource --
|
|
1550
|
+
# carries only the EdgeLinkouts template when --linkouts is set. A section with
|
|
1551
|
+
# ANY provenance.override is author-controlled (DAKP hand-writes the same
|
|
1552
|
+
# template into override.sources): an upstream_source_record_urls mapping
|
|
1553
|
+
# re-homes URLs exactly as specified (primary gets nothing), any other override
|
|
1554
|
+
# keeps the legacy primary-entry emission, and the flag injects nothing there.
|
|
1555
|
+
record_urls: list[str] = [str(u) for u in self.source.url]
|
|
1464
1556
|
upstream_ids = override.upstream_resource_ids if override else upstream_resource_ids(self.provenance.repo)
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1557
|
+
primary_urls: list[str]
|
|
1558
|
+
upstream_urls: dict[str, list[str]] | None
|
|
1559
|
+
if override is None:
|
|
1560
|
+
upstream_urls = {upstream_ids[0]: record_urls}
|
|
1561
|
+
primary_urls = [EDGE_LINKOUT_URL_TEMPLATE] if self.linkouts else []
|
|
1562
|
+
elif override.upstream_source_record_urls is not None:
|
|
1563
|
+
upstream_urls = {key: [str(u) for u in urls] for key, urls in override.upstream_source_record_urls.items()}
|
|
1564
|
+
primary_urls = []
|
|
1565
|
+
else:
|
|
1566
|
+
upstream_urls = None
|
|
1567
|
+
primary_urls = record_urls
|
|
1470
1568
|
# An explicit `sources` template replaces the derived primary/upstream
|
|
1471
1569
|
# emission entirely; the model already forbids combining it with
|
|
1472
1570
|
# `upstream_resource_ids`/`upstream_source_record_urls`.
|
|
@@ -1476,19 +1574,17 @@ class Tcode(Section):
|
|
|
1476
1574
|
knowledge_level = override.knowledge_level if override else self.provenance.knowledge_level
|
|
1477
1575
|
agent_type = override.agent_type if override else self.provenance.agent_type
|
|
1478
1576
|
publication_values = override.publications if override else [publication_curie(self.provenance.repo, self.provenance.publication or "")]
|
|
1479
|
-
# The study is the table itself: one publication, one
|
|
1480
|
-
# publication CURIE is the study's IDENTIFIER; the
|
|
1481
|
-
# is its human-readable NAME.
|
|
1577
|
+
# The study is the table section itself: one publication, one config section.
|
|
1578
|
+
# The publication CURIE is the study's IDENTIFIER; the originating table config
|
|
1579
|
+
# file plus the section's 1-based index is its human-readable NAME (e.g.
|
|
1580
|
+
# "KANG12.yaml SECTION 2"). The two stay disjoint fields and are never composed
|
|
1482
1581
|
# into one string, so neither ever duplicates the other.
|
|
1483
|
-
sheet: str | None = self.source.sheet if self.source.kind == Files.EXCEL else None # pyright: ignore
|
|
1484
1582
|
# No publication means no study to name: the fallback id is the config stem (the
|
|
1485
1583
|
# YAML filename without its extension), not an identifier, so
|
|
1486
1584
|
# `inline_supporting_study` skips the struct unless it has content to carry.
|
|
1487
1585
|
identified: bool = bool(publication_values)
|
|
1488
1586
|
study_id: str = publication_values[0] if publication_values else self.config.stem
|
|
1489
|
-
study_name: str
|
|
1490
|
-
if study_name == study_id:
|
|
1491
|
-
study_name = None
|
|
1587
|
+
study_name: str = f"{self.config.name} SECTION {self.section_index}"
|
|
1492
1588
|
# `use_enum_values` stores the override's enum keys/values as plain strings.
|
|
1493
1589
|
category_override: dict[str, str] | None = (
|
|
1494
1590
|
{obj: f"biolink:{pinned}" for obj, pinned in self.statement.category_override.items()} if self.statement.category_override else None
|
|
@@ -1500,10 +1596,10 @@ class Tcode(Section):
|
|
|
1500
1596
|
(value, ("agent_type", agent_type)),
|
|
1501
1597
|
# Retrieval provenance lives only in the nested `sources` list (Biolink
|
|
1502
1598
|
# RetrievalSource); current translator-ingests emits no flat
|
|
1503
|
-
# `primary_knowledge_source` scalar, so neither do we.
|
|
1504
|
-
#
|
|
1505
|
-
#
|
|
1506
|
-
#
|
|
1599
|
+
# `primary_knowledge_source` scalar, so neither do we. The derived path maps
|
|
1600
|
+
# the section's record URLs onto the upstream repository's supporting entry
|
|
1601
|
+
# (see above); an explicit `sources` template (override.sources) replaces
|
|
1602
|
+
# the whole derivation.
|
|
1507
1603
|
(retrieval_sources, (primary_knowledge_source, upstream_ids, primary_urls, upstream_urls, explicit_sources))
|
|
1508
1604
|
if primary_knowledge_source
|
|
1509
1605
|
else None,
|
|
@@ -1558,6 +1654,7 @@ PHASE_OF: dict[Callable, str] = {
|
|
|
1558
1654
|
fullmap_audit: "qc",
|
|
1559
1655
|
column: "encode",
|
|
1560
1656
|
qualifier_vocabulary_audit: "encode",
|
|
1657
|
+
curie_audit: "encode",
|
|
1561
1658
|
edge_category: "edge",
|
|
1562
1659
|
publications: "provenance",
|
|
1563
1660
|
retrieval_sources: "provenance",
|
|
@@ -138,7 +138,7 @@ class BaseSource(TablaBase):
|
|
|
138
138
|
url: list[HttpUrl] = Field(
|
|
139
139
|
...,
|
|
140
140
|
min_length=1,
|
|
141
|
-
description="One or more remote source URL(s) recorded as provenance
|
|
141
|
+
description="One or more remote source URL(s) recorded as provenance. Derived sections emit them in the edge `sources` list under the upstream repository supporting entry's `source_record_urls` list (infores:pubmed-central for PMC sections) and in the RIG. When `provenance.override.upstream_source_record_urls` is set, these URLs serve the RIG only and the per-upstream mapping determines edge placement instead; any other provenance.override keeps the primary-entry emission. Format-validated only; not fetched.",
|
|
142
142
|
)
|
|
143
143
|
|
|
144
144
|
rows: list[NonNegativeInt] | None = Field(None, description="Zero-based row indices kept after any row_slice crop.", examples=[[0, 2, 5]])
|
|
@@ -288,6 +288,16 @@ class Encoding(TablaBase):
|
|
|
288
288
|
)
|
|
289
289
|
|
|
290
290
|
|
|
291
|
+
#: CURIE shape enforced on verbatim (``no_map``) node values: non-empty ``PREFIX:local-id``.
|
|
292
|
+
#: Shared by config-time literal validation (models) and build-time cell audits (lib).
|
|
293
|
+
CURIE_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]*:[A-Za-z0-9_.\-/]+$")
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def is_curie(value: object) -> bool:
|
|
297
|
+
"""Whether ``value`` is a CURIE-shaped string (``PREFIX:local-id``, both halves non-empty)."""
|
|
298
|
+
return isinstance(value, str) and CURIE_PATTERN.match(value.strip()) is not None
|
|
299
|
+
|
|
300
|
+
|
|
291
301
|
class NodeEncoding(Encoding):
|
|
292
302
|
taxon: PositiveInt | None = Field(
|
|
293
303
|
9606,
|
|
@@ -312,6 +322,87 @@ class NodeEncoding(Encoding):
|
|
|
312
322
|
examples=[["^OMIM:\\d+$"]],
|
|
313
323
|
)
|
|
314
324
|
|
|
325
|
+
no_map: bool = Field(
|
|
326
|
+
False,
|
|
327
|
+
description=(
|
|
328
|
+
"When True, skip fullmap entity resolution: the encoded value is used verbatim as the node CURIE, "
|
|
329
|
+
"for data that already carries its own CURIEs. No NLP normalization, taxon constraint, or category "
|
|
330
|
+
"lookup is applied; blank cells are dropped like unresolved nodes and non-CURIE cells fail the build."
|
|
331
|
+
),
|
|
332
|
+
examples=[True],
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
@model_validator(mode="before")
|
|
336
|
+
@classmethod
|
|
337
|
+
def no_map_rejects_explicit_taxon(cls, data: Any) -> Any:
|
|
338
|
+
"""Reject ``no_map`` with an explicit non-default ``taxon`` (dead resolution config).
|
|
339
|
+
|
|
340
|
+
``taxon`` defaults to 9606 and only steers fullmap ranking, so an explicit
|
|
341
|
+
non-default taxon alongside ``no_map: true`` is config that can never take
|
|
342
|
+
effect. The default itself stays indistinguishable from an explicit 9606 and
|
|
343
|
+
is therefore allowed. Checked ``before`` precisely so "explicitly set" is
|
|
344
|
+
detectable from the raw input dict.
|
|
345
|
+
"""
|
|
346
|
+
if isinstance(data, dict) and data.get("no_map") and "taxon" in data and data["taxon"] not in (None, 9606):
|
|
347
|
+
raise TablassertValidationError(
|
|
348
|
+
f"`no_map: true` skips entity resolution, so `taxon: {data['taxon']}` can never take effect; remove the taxon constraint.",
|
|
349
|
+
code="no-map-dead-config",
|
|
350
|
+
)
|
|
351
|
+
return data
|
|
352
|
+
|
|
353
|
+
@model_validator(mode="after")
|
|
354
|
+
def no_map_rejects_resolution_knobs(self: Self) -> Self:
|
|
355
|
+
"""Reject ``no_map`` combined with resolution-only knobs (dead config).
|
|
356
|
+
|
|
357
|
+
``prioritize``, ``avoid``, ``exclude_prefixes``, and ``exclude_regex`` only
|
|
358
|
+
steer fullmap ranking and filtering; with ``no_map: true`` no lookup happens,
|
|
359
|
+
so these fields mislead the reader. Fail loudly at config time, like the
|
|
360
|
+
``nullable``-on-literal check on :class:`Qualifier`.
|
|
361
|
+
"""
|
|
362
|
+
if not self.no_map:
|
|
363
|
+
return self
|
|
364
|
+
dead: list[str] = [
|
|
365
|
+
name
|
|
366
|
+
for name, value in (
|
|
367
|
+
("prioritize", self.prioritize),
|
|
368
|
+
("avoid", self.avoid),
|
|
369
|
+
("exclude_prefixes", self.exclude_prefixes),
|
|
370
|
+
("exclude_regex", self.exclude_regex),
|
|
371
|
+
)
|
|
372
|
+
if value
|
|
373
|
+
]
|
|
374
|
+
if dead:
|
|
375
|
+
raise TablassertValidationError(
|
|
376
|
+
f"`no_map: true` skips entity resolution, so {', '.join(f'`{name}`' for name in dead)} can never take effect; remove them.",
|
|
377
|
+
code="no-map-dead-config",
|
|
378
|
+
)
|
|
379
|
+
return self
|
|
380
|
+
|
|
381
|
+
@model_validator(mode="after")
|
|
382
|
+
def no_map_literal_must_be_curie(self: Self) -> Self:
|
|
383
|
+
"""Validate that a ``no_map`` literal encodes a CURIE at config time.
|
|
384
|
+
|
|
385
|
+
A ``method: value`` node with ``no_map: true`` becomes the node id verbatim,
|
|
386
|
+
so the literal (after ``prefix``/``suffix`` are applied, matching the build's
|
|
387
|
+
``encoding()``) must already be CURIE-shaped; otherwise every row would be
|
|
388
|
+
dropped at build time. ``method: column`` cells are audited per row at build
|
|
389
|
+
time instead.
|
|
390
|
+
"""
|
|
391
|
+
if not self.no_map or self.method != EncodingMethods.VALUE:
|
|
392
|
+
return self
|
|
393
|
+
# Enum-ranged qualifier literals are closed-vocabulary tokens (audited against
|
|
394
|
+
# the vocabulary, and `no_map` on them is rejected as dead config), not CURIEs.
|
|
395
|
+
if getattr(self, "vocabulary", None) is not None:
|
|
396
|
+
return self
|
|
397
|
+
literal: str = f"{self.prefix or ''}{self.encoding}{self.suffix or ''}".strip()
|
|
398
|
+
if not is_curie(literal):
|
|
399
|
+
raise TablassertValidationError(
|
|
400
|
+
f"`no_map: true` uses the encoded value verbatim as the node CURIE, but the literal {literal!r} is not CURIE-shaped "
|
|
401
|
+
"(expected `PREFIX:local-id`). Fix the literal, or add `prefix`/`suffix` to form a CURIE.",
|
|
402
|
+
code="no-map-bad-curie",
|
|
403
|
+
)
|
|
404
|
+
return self
|
|
405
|
+
|
|
315
406
|
@field_validator("exclude_regex", mode="after")
|
|
316
407
|
@classmethod
|
|
317
408
|
def polars_compatible_exclude_regex(cls, exclude_regex: list[str] | None) -> list[str] | None:
|
|
@@ -359,10 +450,26 @@ class Qualifier(NodeEncoding):
|
|
|
359
450
|
def resolved(self: Self) -> bool:
|
|
360
451
|
"""Whether this qualifier's values go through fullmap entity resolution.
|
|
361
452
|
|
|
362
|
-
|
|
363
|
-
|
|
453
|
+
Enum-ranged qualifiers carry a literal token from a closed Biolink vocabulary
|
|
454
|
+
and must be passed through verbatim. CURIE-ranged qualifiers are resolved
|
|
455
|
+
unless they opt out with ``no_map: true`` (their values are already CURIEs).
|
|
456
|
+
"""
|
|
457
|
+
return self.vocabulary is None and not self.no_map
|
|
458
|
+
|
|
459
|
+
@model_validator(mode="after")
|
|
460
|
+
def reject_no_map_on_enum_ranged(self: Self) -> Self:
|
|
461
|
+
"""Reject ``no_map: true`` on enum-ranged qualifiers as dead config.
|
|
462
|
+
|
|
463
|
+
Enum-ranged qualifiers already pass their literal tokens through verbatim, so
|
|
464
|
+
``no_map`` would be a no-op that misleads the reader; fail loudly at config
|
|
465
|
+
time, like the ``nullable``-on-literal check below.
|
|
364
466
|
"""
|
|
365
|
-
|
|
467
|
+
if self.no_map and self.vocabulary is not None:
|
|
468
|
+
raise TablassertValidationError(
|
|
469
|
+
f"{self.qualifier} has a closed Biolink vocabulary and already passes values through verbatim, so `no_map` is redundant; remove it.",
|
|
470
|
+
code="no-map-dead-config",
|
|
471
|
+
)
|
|
472
|
+
return self
|
|
366
473
|
|
|
367
474
|
@model_validator(mode="after")
|
|
368
475
|
def reject_disabled_qualifiers(self: Self) -> Self:
|
|
@@ -272,6 +272,9 @@ OP_COST: dict[Callable, int] = {
|
|
|
272
272
|
lib.split_list: 1,
|
|
273
273
|
# Vocabulary audit: one eager distinct-value pass over a single enum column.
|
|
274
274
|
lib.qualifier_vocabulary_audit: 1,
|
|
275
|
+
# CURIE audit: one eager pass over a single no_map column (distinct-value check,
|
|
276
|
+
# blank-row filter, category column synthesis).
|
|
277
|
+
lib.curie_audit: 1,
|
|
275
278
|
# Clean: coerce_columns applies five coercion rules (schema resolve + renames + value
|
|
276
279
|
# coercion) in one pass, so it outprices a single-expression clean op.
|
|
277
280
|
lib.coerce_columns: 3,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|