tablassert 20.0.0__tar.gz → 21.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {tablassert-20.0.0 → tablassert-21.0.0}/PKG-INFO +4 -4
  2. {tablassert-20.0.0 → tablassert-21.0.0}/README.md +3 -3
  3. {tablassert-20.0.0 → tablassert-21.0.0}/pyproject.toml +1 -1
  4. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/cli.py +113 -116
  5. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/errors.py +3 -0
  6. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/ingests.py +8 -3
  7. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/lib.py +138 -41
  8. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/models.py +111 -4
  9. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/runcache.py +3 -0
  10. {tablassert-20.0.0 → tablassert-21.0.0}/LICENSE +0 -0
  11. {tablassert-20.0.0 → tablassert-21.0.0}/rust/Cargo.lock +0 -0
  12. {tablassert-20.0.0 → tablassert-21.0.0}/rust/Cargo.toml +0 -0
  13. {tablassert-20.0.0 → tablassert-21.0.0}/rust/examples/count_tables.rs +0 -0
  14. {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/fullmap.rs +0 -0
  15. {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/json.rs +0 -0
  16. {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/lib.rs +0 -0
  17. {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/ndjson.rs +0 -0
  18. {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/nlp.rs +0 -0
  19. {tablassert-20.0.0 → tablassert-21.0.0}/rust/src/uuid.rs +0 -0
  20. {tablassert-20.0.0 → tablassert-21.0.0}/rust/tests/build_golden.rs +0 -0
  21. {tablassert-20.0.0 → tablassert-21.0.0}/rust/tests/common/mod.rs +0 -0
  22. {tablassert-20.0.0 → tablassert-21.0.0}/rust/tests/extract_prebuilt.rs +0 -0
  23. {tablassert-20.0.0 → tablassert-21.0.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
  24. {tablassert-20.0.0 → tablassert-21.0.0}/rust/tests/nlp_golden.rs +0 -0
  25. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/__init__.py +0 -0
  26. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/_lazy.py +0 -0
  27. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/biolink.py +0 -0
  28. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/cli_entry.py +0 -0
  29. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/coerce.py +0 -0
  30. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
  31. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/data/infores_catalog.yaml +0 -0
  32. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/enums.py +0 -0
  33. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/extras.py +0 -0
  34. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/fullmap.py +0 -0
  35. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/infores_registry.py +0 -0
  36. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/log.py +0 -0
  37. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/net.py +0 -0
  38. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/nlp.py +0 -0
  39. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/progress.py +0 -0
  40. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/qc.py +0 -0
  41. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/rig.py +0 -0
  42. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/rs.pyi +0 -0
  43. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/study.py +0 -0
  44. {tablassert-20.0.0 → tablassert-21.0.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 20.0.0
3
+ Version: 21.0.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -181,8 +181,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
181
181
  - **Declarative YAML configuration**: define data transformations without writing code
182
182
  - **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
183
183
  CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
184
- - **Optional quality control**: a four-stage audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) flags
185
- low-confidence mappings
184
+ - **Optional quality control**: a four-stage audit (exact match, fuzzy match, abbreviation
185
+ expansion, then SapBERT embeddings) flags low-confidence mappings
186
186
  - **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
187
187
  and predicates
188
188
  - **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
@@ -202,7 +202,7 @@ provides the `tablassert` command. Other extras are opt-in:
202
202
  | `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
203
203
  | `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
204
204
  | `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
205
- | `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
205
+ | `qc` | four-stage QC audit (exact match, fuzzy match, abbreviation expansion, and SapBERT embeddings) | `pip install "tablassert[qc]"` |
206
206
  | `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
207
207
 
208
208
  Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
@@ -127,8 +127,8 @@ See the [Tutorial](https://skyeav.github.io/Tablassert/tutorial/) for the full w
127
127
  - **Declarative YAML configuration**: define data transformations without writing code
128
128
  - **Built-in entity resolution**: map free text to genes, diseases, and chemicals with standard
129
129
  CURIEs, taxonomic filtering, and provenance, backed by an embedded redb database
130
- - **Optional quality control**: a four-stage audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) flags
131
- low-confidence mappings
130
+ - **Optional quality control**: a four-stage audit (exact match, fuzzy match, abbreviation
131
+ expansion, then SapBERT embeddings) flags low-confidence mappings
132
132
  - **KGX compliance**: emits NCATS Translator-compatible node/edge NDJSON with Biolink categories
133
133
  and predicates
134
134
  - **Performance & reproducibility**: lazy Polars pipelines and a deterministic UV-based
@@ -148,7 +148,7 @@ provides the `tablassert` command. Other extras are opt-in:
148
148
  | `cli` | `tablassert` command and rich terminal progress | `pip install "tablassert[cli]"` |
149
149
  | `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
150
150
  | `aria2` | bundled aria2c downloader, used automatically by `build-fullmap` when installed (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
151
- | `qc` | four-stage QC audit (exact -> fuzzy -> abbreviation -> SapBERT embeddings) | `pip install "tablassert[qc]"` |
151
+ | `qc` | four-stage QC audit (exact match, fuzzy match, abbreviation expansion, and SapBERT embeddings) | `pip install "tablassert[qc]"` |
152
152
  | `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
153
153
 
154
154
  Reaching a feature whose extra is missing never produces a bare `ModuleNotFoundError`: the failure
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "20.0.0"
3
+ version = "21.0.0"
4
4
  description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -420,12 +420,12 @@ def build_graph_pipeline(
420
420
  advance()
421
421
  with RunCache() as cache:
422
422
  for planned in plan.execution_order:
423
- section_index, x, op = pending[planned]
423
+ flat_index, x, op = pending[planned]
424
424
  entry: PlanEntry = plan.entries[planned]
425
425
  start(format_section_compact(x))
426
426
  # on_phase drives the per-op sub-step indicator (cache → resolve → write ...);
427
427
  # resume/snapshot are the producer/consumer seams the plan assigned this section.
428
- results[section_index] = compile_subgraph(
428
+ results[flat_index] = compile_subgraph(
429
429
  op, on_phase=sub_step, resume=_run_cache_resume(cache, entry), snapshot=_run_cache_snapshot(cache, entry)
430
430
  )
431
431
  advance()
@@ -984,19 +984,52 @@ def _print_kgx_section(label: str, section: dict[str, Any], path: Path) -> None:
984
984
  logger.info(f"validate-kgx {label}: {section['valid']}/{section['total']} valid")
985
985
 
986
986
 
987
+ def _print_infores_report(report: dict[str, Any], paths: dict[str, Path | None]) -> None:
988
+ """Print one identity-half report: absent inputs, registry provenance, counts, examples."""
989
+ for label, absent in report["missing"].items():
990
+ if absent:
991
+ print(f"{label}: file not found ({paths.get(label)})", file=sys.stderr)
992
+ print(f"registry: {report['registry']['source']} ({report['registry']['entries']} entries)", file=sys.stderr)
993
+ counts: str = (
994
+ f"curies: {report['registered']} registered, {report['allowed']} allowed, "
995
+ f"{report['unregistered']} unregistered, {report['malformed']} malformed"
996
+ )
997
+ print(counts, file=sys.stderr)
998
+ logger.info(f"validate-kgx infores: {counts}")
999
+ for example in report["examples"][:3]:
1000
+ print(f" e.g. {example['curie']} ({example['where']}.{example['field']}): {example['problem']}", file=sys.stderr)
1001
+
1002
+
987
1003
  @APP.command(name="validate-kgx")
988
1004
  def validate_kgx_command(
989
1005
  nodes: Annotated[Path, cyclopts.Parameter(name=["--nodes", "-n"])],
990
1006
  edges: Annotated[Path, cyclopts.Parameter(name=["--edges", "-e"])],
991
1007
  limit: Annotated[int, cyclopts.Parameter(name=["--limit"])] = 20,
992
1008
  prune: Annotated[bool, cyclopts.Parameter(name=["--prune", "-p"])] = False,
1009
+ rig: Annotated[Path | None, cyclopts.Parameter(name=["--rig"])] = None,
1010
+ allow_infores: Annotated[list[str] | None, cyclopts.Parameter(name=["--allow-infores"], negative="", allow_leading_hyphen=False)] = None,
1011
+ registry: Annotated[Literal["snapshot", "refresh", "off"], cyclopts.Parameter(name=["--registry"])] = "snapshot",
1012
+ strict: Annotated[bool, cyclopts.Parameter(name=["--strict"])] = False,
993
1013
  ) -> None:
994
- """Validate built KGX NDJSON against the Biolink Model.
995
-
996
- Constructs every node and edge as the Biolink Pydantic class named by its own
997
- ``category`` -- the same classes ``NCATSTranslator/translator-ingests`` builds --
998
- and reports failures grouped by field and error type. Exits non-zero when any
999
- record fails, so a build can be gated in CI.
1014
+ """Validate built KGX NDJSON against the Biolink Model and the Translator registry.
1015
+
1016
+ The structural half constructs every node and edge as the Biolink Pydantic class
1017
+ named by its own ``category`` -- the same classes ``NCATSTranslator/translator-
1018
+ ingests`` builds -- and reports failures grouped by field and error type. The
1019
+ identity half classifies every emitted knowledge-source identifier (edge
1020
+ ``primary_knowledge_source``, ``sources[].resource_id`` and
1021
+ ``sources[].upstream_resource_ids``, node ``provided_by``, and, with ``--rig``, the
1022
+ RIG document's infores values) against ``biolink/information-resource-registry``'s
1023
+ ``infores_catalog.yaml`` (vendored as the bundled snapshot, so the default path is
1024
+ offline). Values without the ``infores:`` prefix are malformed, not unregistered.
1025
+
1026
+ The structural half exits non-zero when any record fails, so a build can be gated
1027
+ in CI. The identity half is advisory by default -- unregistered CURIEs are reported
1028
+ and the command exits 0, because Tablassert legitimately mints graph-local CURIEs
1029
+ that will never sit in the registry -- and ``--strict`` (CI gates) exits 1 while any
1030
+ unregistered CURIE remains; allowlist legitimate local ids with repeatable
1031
+ ``--allow-infores``. Missing input files fail in every mode: a typo'd path must
1032
+ never read as a clean bill of health.
1000
1033
 
1001
1034
  ``--prune`` rewrites the EDGES file in place, keeping only edges that validate
1002
1035
  strictly. Real defects (non-coercible values, extras the model will never declare,
@@ -1007,137 +1040,101 @@ def validate_kgx_command(
1007
1040
  stay byte-identical; a clean file is not rewritten at all, and a missing edges file
1008
1041
  is reported without ever being pruned. Nodes are validated and reported, never
1009
1042
  pruned. The exit code stays strict over the post-prune state: after a prune, every
1010
- kept edge is strictly valid, so the run is compliant exactly when the nodes are.
1043
+ kept edge is strictly valid, so the run is compliant exactly when the nodes are,
1044
+ and the identity half classifies the PRUNED file -- what ships is what is checked.
1011
1045
 
1012
1046
  Args:
1013
1047
  nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
1014
1048
  edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
1015
- limit: Maximum example failures retained per file for the ``e.g.`` report lines;
1016
- every record is still validated regardless of this cap. A missing file is
1017
- reported and fails the run rather than reading as a pass.
1049
+ limit: Maximum example failures retained per file for the ``e.g.`` report lines
1050
+ (structural and identity examples alike); every record is still validated
1051
+ regardless of this cap. A missing file is reported and fails the run rather
1052
+ than reading as a pass.
1018
1053
  prune: Rewrite the edges file in place, removing every edge record that is not
1019
1054
  strictly KGX/Biolink-compliant, including the deliberate pending carryovers.
1020
1055
  The nodes file is never modified; a clean edges file is left untouched.
1056
+ rig: Optional RIG yaml (``<name>_<version>.RIG.yaml``); its infores values are
1057
+ classified too.
1058
+ allow-infores: Registry-exempt CURIE, repeatable; use for the locally-minted
1059
+ graph identifier (``rig.infores()``) and other deliberate locals.
1060
+ registry: Registry source. ``snapshot`` (default) validates offline against the
1061
+ bundled copy; ``refresh`` fetches the live upstream catalog (network) and
1062
+ caches it under the user cache dir; ``off`` skips the identity half
1063
+ entirely (structure-only validation).
1064
+ strict: Exit non-zero when any unregistered CURIE remains. Without it the
1065
+ identity half is advisory and never fails a run once inputs exist.
1021
1066
  """
1022
1067
  from tablassert.biolink import _validate_file, prune_kgx_edges, validate_kgx
1068
+ from tablassert.infores_registry import REGISTRY_RAW_URL, refresh_registry, validate_infores
1069
+
1070
+ ids: frozenset[str] | None = refresh_registry() if registry == "refresh" else None
1023
1071
 
1024
1072
  if not prune:
1025
1073
  report: dict[str, Any] = validate_kgx(nodes, edges, limit=limit)
1026
1074
  print(f"biolink-model {report['biolink_version']}", file=sys.stderr)
1027
1075
  _print_kgx_section("nodes", report["nodes"], nodes)
1028
1076
  _print_kgx_section("edges", report["edges"], edges)
1029
- if not report["ok"]:
1077
+ structural_ok: bool = report["ok"]
1078
+ structural_missing: bool = report["nodes"]["missing"] or report["edges"]["missing"]
1079
+ else:
1080
+ # The prune path validates nodes-only itself and drives prune_kgx_edges' own single
1081
+ # pass for both edges sections, so the edges file is never read and written twice.
1082
+ nodes_section: dict[str, Any] = _validate_file(nodes, edge=False, limit=limit)
1083
+ pruned_report: dict[str, Any] = prune_kgx_edges(edges, limit=limit)
1084
+ print(f"biolink-model {pruned_report['biolink_version']}", file=sys.stderr)
1085
+ _print_kgx_section("nodes", nodes_section, nodes)
1086
+ _print_kgx_section("edges", pruned_report["before"], edges)
1087
+ if pruned_report["before"]["missing"]:
1030
1088
  print("KGX output is not Biolink-compliant.", file=sys.stderr)
1031
1089
  raise SystemExit(1)
1032
- print("KGX output is Biolink-compliant.", file=sys.stderr)
1033
- return
1034
-
1035
- # The prune path validates nodes-only itself and drives prune_kgx_edges' own single
1036
- # pass for both edges sections, so the edges file is never read and written twice.
1037
- nodes_section: dict[str, Any] = _validate_file(nodes, edge=False, limit=limit)
1038
- pruned_report: dict[str, Any] = prune_kgx_edges(edges, limit=limit)
1039
- print(f"biolink-model {pruned_report['biolink_version']}", file=sys.stderr)
1040
- _print_kgx_section("nodes", nodes_section, nodes)
1041
- _print_kgx_section("edges", pruned_report["before"], edges)
1042
- if pruned_report["before"]["missing"]:
1090
+ malformed_note: str = f" ({pruned_report['malformed']} malformed lines)" if pruned_report["malformed"] else ""
1091
+ print(f"edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} non-compliant records{malformed_note}", file=sys.stderr)
1092
+ logger.info(
1093
+ f"validate-kgx edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} (rewritten={pruned_report['rewritten']})"
1094
+ )
1095
+ for problem, count in pruned_report["dropped"]["problems"].items():
1096
+ print(f" {count:>9} {problem}", file=sys.stderr)
1097
+ for example in pruned_report["dropped"]["examples"][:3]:
1098
+ print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
1099
+ _print_kgx_section("edges", pruned_report["after"], edges)
1100
+ nodes_ok: bool = not nodes_section["missing"] and nodes_section["total"] == nodes_section["valid"]
1101
+ structural_ok = nodes_ok and pruned_report["ok"]
1102
+ structural_missing = nodes_section["missing"]
1103
+
1104
+ # The identity half classifies what ships: in prune mode the edges file has already
1105
+ # been rewritten, so the scan reads the post-prune state. When the structural half
1106
+ # already failed on absent inputs and no RIG was given, a second scan would only
1107
+ # echo the same file-not-found lines.
1108
+ infores_ok: bool = True
1109
+ if registry != "off" and (rig is not None or not structural_missing):
1110
+ infores_report: dict[str, Any] = validate_infores(nodes, edges, rig_path=rig, allow=tuple(allow_infores or ()), registry=ids, limit=limit)
1111
+ if ids is not None:
1112
+ infores_report["registry"]["source"] = f"{REGISTRY_RAW_URL} (refreshed)"
1113
+ _print_infores_report(infores_report, {"nodes": nodes, "edges": edges, "rig": rig})
1114
+ if any(infores_report["missing"].values()):
1115
+ print("infores validation failed: missing input files.", file=sys.stderr)
1116
+ infores_ok = False
1117
+ else:
1118
+ clean: bool = not infores_report["unregistered"] and not infores_report["malformed"]
1119
+ if strict:
1120
+ if infores_report["ok_strict"]:
1121
+ print("All emitted infores CURIEs are registered." if clean else "infores validation passed (strict).", file=sys.stderr)
1122
+ else:
1123
+ print("infores validation failed (strict): unregistered CURIEs remain.", file=sys.stderr)
1124
+ infores_ok = False
1125
+ elif not clean:
1126
+ print("infores warnings present; pass --strict to fail on them.", file=sys.stderr)
1127
+ else:
1128
+ print("All emitted infores CURIEs are registered.", file=sys.stderr)
1129
+
1130
+ if not structural_ok:
1043
1131
  print("KGX output is not Biolink-compliant.", file=sys.stderr)
1044
1132
  raise SystemExit(1)
1045
- malformed_note: str = f" ({pruned_report['malformed']} malformed lines)" if pruned_report["malformed"] else ""
1046
- print(f"edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} non-compliant records{malformed_note}", file=sys.stderr)
1047
- logger.info(f"validate-kgx edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} (rewritten={pruned_report['rewritten']})")
1048
- for problem, count in pruned_report["dropped"]["problems"].items():
1049
- print(f" {count:>9} {problem}", file=sys.stderr)
1050
- for example in pruned_report["dropped"]["examples"][:3]:
1051
- print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
1052
- _print_kgx_section("edges", pruned_report["after"], edges)
1053
- nodes_ok: bool = not nodes_section["missing"] and nodes_section["total"] == nodes_section["valid"]
1054
- if not (nodes_ok and pruned_report["ok"]):
1055
- print("KGX output is not Biolink-compliant.", file=sys.stderr)
1133
+ if not infores_ok:
1056
1134
  raise SystemExit(1)
1057
1135
  print("KGX output is Biolink-compliant.", file=sys.stderr)
1058
1136
 
1059
1137
 
1060
- @APP.command(name="validate-infores")
1061
- def validate_infores_command(
1062
- nodes: Annotated[Path, cyclopts.Parameter(name=["--nodes", "-n"])],
1063
- edges: Annotated[Path, cyclopts.Parameter(name=["--edges", "-e"])],
1064
- rig: Annotated[Path | None, cyclopts.Parameter(name=["--rig"])] = None,
1065
- allow_infores: Annotated[list[str] | None, cyclopts.Parameter(name=["--allow-infores"], negative="", allow_leading_hyphen=False)] = None,
1066
- registry: Annotated[Literal["snapshot", "refresh", "off"], cyclopts.Parameter(name=["--registry"])] = "snapshot",
1067
- strict: Annotated[bool, cyclopts.Parameter(name=["--strict"])] = False,
1068
- limit: Annotated[int, cyclopts.Parameter(name=["--limit"])] = 20,
1069
- ) -> None:
1070
- """Validate emitted infores CURIEs against the NCATS Translator registry.
1071
-
1072
- Every knowledge-source identifier the build emits -- edge ``primary_knowledge_source``,
1073
- ``sources[].resource_id`` and ``sources[].upstream_resource_ids``, node ``provided_by``,
1074
- and, with ``--rig``, the RIG document's infores values -- is classified against
1075
- ``biolink/information-resource-registry``'s ``infores_catalog.yaml`` (vendored as the
1076
- bundled snapshot, so the default path is offline). Values without the ``infores:``
1077
- prefix are malformed, not unregistered.
1078
-
1079
- The default posture is advisory: unregistered CURIEs are reported and the command
1080
- exits 0, because Tablassert legitimately mints graph-local CURIEs that will never sit
1081
- in the registry. ``--strict`` (CI gates) exits 1 when any unregistered CURIE remains;
1082
- allowlist legitimate local ids with repeatable ``--allow-infores``. Missing input
1083
- files fail in every mode: a typo'd path must never read as a clean bill of health.
1084
-
1085
- Args:
1086
- nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
1087
- edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
1088
- rig: Optional RIG yaml (``<name>_<version>.RIG.yaml``); its infores values are
1089
- classified too.
1090
- allow-infores: Registry-exempt CURIE, repeatable; use for the locally-minted
1091
- graph identifier (``rig.infores()``) and other deliberate locals.
1092
- registry: Registry source. ``snapshot`` (default) validates offline against the
1093
- bundled copy; ``refresh`` fetches the live upstream catalog (network) and
1094
- caches it under the user cache dir; ``off`` skips membership classification
1095
- and reports structure only.
1096
- strict: Exit non-zero when any unregistered CURIE remains. Without it the run
1097
- is advisory and always exits 0 once inputs exist.
1098
- limit: Maximum number of example unregistered/malformed CURIEs retained for the
1099
- ``e.g.`` report lines; every observation is still classified regardless.
1100
- """
1101
- from tablassert.infores_registry import REGISTRY_RAW_URL, refresh_registry, validate_infores
1102
-
1103
- ids: frozenset[str] | None = None
1104
- if registry == "refresh":
1105
- ids = refresh_registry()
1106
- report: dict[str, Any] = validate_infores(
1107
- nodes, edges, rig_path=rig, allow=tuple(allow_infores or ()), registry=ids, check_registry=registry != "off", limit=limit
1108
- )
1109
- if ids is not None:
1110
- report["registry"]["source"] = f"{REGISTRY_RAW_URL} (refreshed)"
1111
- for label, absent in report["missing"].items():
1112
- if absent:
1113
- path_label = {"nodes": nodes, "edges": edges, "rig": rig}.get(label)
1114
- print(f"{label}: file not found ({path_label})", file=sys.stderr)
1115
- if any(report["missing"].values()):
1116
- print("infores validation failed: missing input files.", file=sys.stderr)
1117
- raise SystemExit(1)
1118
- print(f"registry: {report['registry']['source']} ({report['registry']['entries']} entries)", file=sys.stderr)
1119
- counts: str = (
1120
- f"curies: {report['registered']} registered, {report['allowed']} allowed, "
1121
- f"{report['unregistered']} unregistered, {report['malformed']} malformed"
1122
- )
1123
- print(counts, file=sys.stderr)
1124
- logger.info(f"validate-infores: {counts}")
1125
- for example in report["examples"][:3]:
1126
- print(f" e.g. {example['curie']} ({example['where']}.{example['field']}): {example['problem']}", file=sys.stderr)
1127
- clean: bool = not report["unregistered"] and not report["malformed"]
1128
- passed: str = "infores membership check skipped (--registry off)." if registry == "off" else "All emitted infores CURIEs are registered."
1129
- if strict:
1130
- if report["ok_strict"]:
1131
- print(passed if clean else "infores validation passed (strict).", file=sys.stderr)
1132
- return
1133
- print("infores validation failed (strict): unregistered CURIEs remain.", file=sys.stderr)
1134
- raise SystemExit(1)
1135
- if not clean:
1136
- print("infores warnings present; pass --strict to fail on them.", file=sys.stderr)
1137
- return
1138
- print(passed, file=sys.stderr)
1139
-
1140
-
1141
1138
  @APP.command(name="quick-map")
1142
1139
  def quick_map_command(
1143
1140
  # negative="" suppresses cyclopts' auto-generated --empty-terms help row: the reset
@@ -63,6 +63,9 @@ TablassertErrorCodes = Literal[
63
63
  "provenance-missing-publication",
64
64
  "provenance-publication-and-override",
65
65
  "encoding-list-method-removed",
66
+ "no-map-dead-config",
67
+ "no-map-bad-curie",
68
+ "no-map-curie-violation",
66
69
  "annotation-split-by-requires-column",
67
70
  "annotation-split-by-empty",
68
71
  "field-disabled",
@@ -82,12 +82,14 @@ def to_yaml(p: Path, data: object) -> None:
82
82
  yaml.dump(data, f, Dumper=_IndentedSafeDumper, sort_keys=False, default_flow_style=False, allow_unicode=True, width=120)
83
83
 
84
84
 
85
- def to_sections(instructions: dict[str, Any], table: Path) -> list[list[dict[str, Any]]]:
85
+ def to_sections(instructions: dict[str, Any], table: Path) -> list[dict[str, Any]]:
86
86
  """Expand a table config dict into a list of section dicts.
87
87
 
88
88
  Each section is the deep merge of the table-level ``template`` over a
89
89
  per-section entry. The originating ``table`` path is stamped into each
90
- section as ``config``.
90
+ section as ``config``, and the section's 1-based position in the file
91
+ (document order) as ``section_index`` so provenance can name the inlined
92
+ Study after its origin ("<table config> SECTION <n>").
91
93
 
92
94
  Args:
93
95
  instructions: Parsed YAML config with optional ``template`` and
@@ -100,4 +102,7 @@ def to_sections(instructions: dict[str, Any], table: Path) -> list[list[dict[str
100
102
  template: dict[str, Any] = instructions.get("template", {})
101
103
  template["config"] = table
102
104
  sections: list[dict[str, Any]] = instructions.get("sections", [{}])
103
- return [fastmerge(deepcopy(template), x) for x in sections]
105
+ merged: list[dict[str, Any]] = [fastmerge(deepcopy(template), x) for x in sections]
106
+ for index, section in enumerate(merged, start=1):
107
+ section["section_index"] = index
108
+ return merged
@@ -48,7 +48,7 @@ from tablassert.enums import EncodingMethods, Files, Functions, InformationResou
48
48
  from tablassert.errors import TablassertError
49
49
  from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
50
50
  from tablassert.log import cat
51
- from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
51
+ from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section, is_curie
52
52
  from tablassert.nlp import level_one, level_two
53
53
  from tablassert.qc import fullmap_audit
54
54
  from tablassert.rig import (
@@ -471,8 +471,8 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, study_name: str | N
471
471
  The struct is shaped by the current Biolink Model (4.4.4+):
472
472
 
473
473
  * ``Study.id`` is a real identifier -- the section's publication CURIE -- and
474
- ``Study.name`` a DISJOINT human label (worksheet or source filename); the two
475
- are never composed into one string.
474
+ ``Study.name`` a DISJOINT human label (the table config file plus the section's
475
+ 1-based index); the two are never composed into one string.
476
476
  * Study-level metadata declared as annotations (``study_size``, ``study_cohort``,
477
477
  ``study_context``, ``study_date_range``, ``study_method_description``,
478
478
  ``study_method_types`` -- see :data:`biolink.STUDY_METADATA_FIELDS`) lands as
@@ -504,8 +504,8 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, study_name: str | N
504
504
  lf: Edges LazyFrame after annotation and provenance ops.
505
505
  study_id: Stable study identifier (publication CURIE, or the config stem when
506
506
  unidentified).
507
- study_name: Human-readable study label (worksheet or source filename), or
508
- ``None`` to omit it.
507
+ study_name: Human-readable study label (table config file plus 1-based section
508
+ index), or ``None`` to omit it.
509
509
  identified: Whether ``study_id`` names a real publication rather than falling
510
510
  back to the config stem. ``False`` lets a contentless study be skipped.
511
511
 
@@ -608,11 +608,14 @@ def retrieval_sources(
608
608
  Mirrors ``build_association_knowledge_sources()`` from
609
609
  ``translator-ingests/util/biolink.py``: the primary knowledge source lists the
610
610
  upstream resources, and each upstream resource additionally appears as its own
611
- ``supporting_data_source`` entry. By default the primary entry carries the
612
- section's source record URLs; when ``upstream_urls`` is given, URL placement is
613
- fully determined by that mapping instead -- each listed upstream entry carries
614
- its own ``source_record_urls`` and the primary entry emits none (the primary is
615
- the transforming resource, not a downloadable record).
611
+ ``supporting_data_source`` entry. URL placement is determined by ``upstream_urls``
612
+ whenever it is given (the derived build path passes it unconditionally): each
613
+ listed upstream entry carries its own ``source_record_urls``, because the section's
614
+ record URLs describe the upstream repository's downloadable record, not the
615
+ transforming primary. The primary entry still emits ``urls`` that are not source
616
+ records -- the per-edge EdgeLinkouts template when ``--linkouts`` is set. Without
617
+ ``upstream_urls`` (override sections that set no explicit mapping), the primary
618
+ entry carries ``urls`` verbatim and the supporting entries stay bare.
616
619
 
617
620
  When ``explicit`` is given, exactly those entries are emitted, in order, and
618
621
  ``primary``/``upstream``/``urls``/``upstream_urls`` are ignored. Each entry
@@ -626,7 +629,9 @@ def retrieval_sources(
626
629
  lf: Source LazyFrame.
627
630
  primary: Infores CURIE of the primary knowledge source.
628
631
  upstream: Infores CURIEs of upstream/supporting data sources.
629
- urls: Source record URLs for the primary entry (ignored when ``upstream_urls`` is set).
632
+ urls: URLs for the primary entry (when ``upstream_urls`` is set these must not be
633
+ source records -- e.g. the per-edge EdgeLinkouts template -- since record URLs
634
+ ride the supporting entries in that mode).
630
635
  upstream_urls: Optional per-upstream source record URLs keyed by infores CURIE.
631
636
  explicit: Optional explicit entry templates emitted verbatim, in order.
632
637
 
@@ -639,7 +644,7 @@ def retrieval_sources(
639
644
  for entry in explicit
640
645
  ]
641
646
  elif upstream_urls is not None:
642
- entries = [_retrieval_source(primary, "primary_knowledge_source", upstream)]
647
+ entries = [_retrieval_source(primary, "primary_knowledge_source", upstream, urls)]
643
648
  entries.extend(_retrieval_source(x, "supporting_data_source", urls=upstream_urls.get(x)) for x in upstream)
644
649
  else:
645
650
  entries = [_retrieval_source(primary, "primary_knowledge_source", upstream, urls)]
@@ -1215,6 +1220,65 @@ def qualifier_vocabulary_audit(lf: pl.LazyFrame, col: str, vocabulary: Iterable[
1215
1220
  )
1216
1221
 
1217
1222
 
1223
+ def curie_audit(lf: pl.LazyFrame, col: str, section: str, config_file: str) -> pl.LazyFrame:
1224
+ """Finalize a ``no_map`` node column: blank rows drop, non-CURIE values fail, category is synthesized.
1225
+
1226
+ Columns configured with ``no_map: true`` bypass entity resolution, so nothing else
1227
+ (a) drops rows whose cell is blank (a missing node id can never form an edge, exactly
1228
+ like an unresolved subject/object), (b) rejects cells that are not CURIE-shaped -- the
1229
+ encoded value becomes the KGX node id verbatim, so garbage would ship as an identifier --
1230
+ or (c) produces the ``<col>_category`` column downstream stages require: without a
1231
+ fullmap lookup there is no role to derive, so the column is pinned to
1232
+ ``biolink:NamedThing`` and edge derivation degrades to ``biolink:Association`` exactly
1233
+ like the documented no-role-lookup fallback in :func:`edge_category`.
1234
+
1235
+ Args:
1236
+ lf: Source LazyFrame.
1237
+ col: The no_map node column (``subject``/``object`` or a CURIE-ranged qualifier key).
1238
+ section: Short section hash (for log context).
1239
+ config_file: Originating config file (for log context).
1240
+
1241
+ Returns:
1242
+ LazyFrame with blank/null-like rows dropped and ``<col>_category`` added.
1243
+
1244
+ Notes:
1245
+ A non-blank value that is not CURIE-shaped raises a coded error naming the
1246
+ offending values (mirroring ``qualifier_vocabulary_audit``); blank and null-like
1247
+ cells (``""``, ``"NA"``, ...) are dropped with a loud per-column count instead,
1248
+ mirroring how an unresolvable subject/object row is dropped, never emitted.
1249
+ """
1250
+ values: pl.Series = lf.select(pl.col(col)).collect().get_column(col)
1251
+ if values.dtype.base_type() in (pl.List, pl.Array):
1252
+ values = values.explode()
1253
+ blank_mask: pl.Series = values.is_null() | values.str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT))
1254
+ # Judge CURIE shape on the distinct non-blank values only: the audit runs once per
1255
+ # section, and map_elements is a per-element Python call that must not scale with
1256
+ # row count when one bad cell repeats a million times.
1257
+ distinct: pl.Series = values.filter(~blank_mask).unique()
1258
+ offending: pl.Series = distinct.filter(~distinct.str.strip_chars().map_elements(is_curie, return_dtype=pl.Boolean)).sort()
1259
+ if offending.len() > 0:
1260
+ shown: list[str] = offending.head(10).to_list()
1261
+ more: str = "..." if offending.len() > len(shown) else ""
1262
+ raise TablassertError(
1263
+ f"{col} is configured with `no_map: true` so its values are used verbatim as node CURIEs, but "
1264
+ f"{offending.len()} non-blank value(s) in the built column are not CURIE-shaped: "
1265
+ f"{', '.join(repr(x) for x in shown)}{more}. Fix the source table or the encoding "
1266
+ "(prefix/suffix can form a CURIE from raw cells).",
1267
+ code="no-map-curie-violation",
1268
+ )
1269
+ dropped: int = int(blank_mask.sum())
1270
+ if dropped:
1271
+ logger.info(
1272
+ "no_map column {col} in {config} ({hash}) dropped {n} blank row(s): a missing CURIE cannot form a node",
1273
+ col=col,
1274
+ config=config_file,
1275
+ hash=section,
1276
+ n=dropped,
1277
+ )
1278
+ present: pl.Expr = pl.col(col).is_not_null() & ~pl.col(col).str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT))
1279
+ return lf.filter(present).with_columns(pl.lit("biolink:NamedThing").alias(f"{col}_category"))
1280
+
1281
+
1218
1282
  def to_store(lf: pl.LazyFrame, p: Path, config_name: str) -> Path:
1219
1283
  """Collect and write a section LazyFrame to a parquet file.
1220
1284
 
@@ -1250,6 +1314,10 @@ class Tcode(Section):
1250
1314
  name: str | None = Field(None)
1251
1315
  infores: str | None = Field(None)
1252
1316
  linkouts: bool = Field(False)
1317
+ section_index: int = Field(
1318
+ 1,
1319
+ description="1-based position of this section within its table config file (document order); stamped by ingests.to_sections and used to name the inlined Study.",
1320
+ )
1253
1321
  qc_similarity_threshold: float = Field(0.5)
1254
1322
  qc_fuzzy_ratio_threshold: float = Field(70.0)
1255
1323
  qc_fuzzy_partial_threshold: float = Field(80.0)
@@ -1369,16 +1437,25 @@ class Tcode(Section):
1369
1437
  enabled.
1370
1438
  """
1371
1439
  # Subject/object/qualifiers share one resolve_batch call instead of one per column.
1372
- # Enum-ranged qualifiers are excluded from resolution: their range is a closed
1373
- # Biolink vocabulary, so sending them through the fullmap would turn the required
1374
- # token `increased` into the CURIE `UMLS:C0205217`, which the slot rejects.
1440
+ # Two kinds of columns never reach it: enum-ranged qualifiers, whose range is a
1441
+ # closed Biolink vocabulary (sending them through the fullmap would turn the
1442
+ # required token `increased` into the CURIE `UMLS:C0205217`, which the slot
1443
+ # rejects), and no_map nodes, whose values are already CURIEs used verbatim.
1375
1444
  qualifiers: list[Qualifier] = self.statement.qualifiers or []
1445
+ subj_obj: list[tuple[NodeEncoding, str]] = [(self.statement.subject, "subject"), (self.statement.object, "object")]
1376
1446
  node_columns: list[tuple[NodeEncoding, str]] = [
1377
- (self.statement.subject, "subject"),
1378
- (self.statement.object, "object"),
1447
+ *[(x, col) for x, col in subj_obj if not x.no_map],
1379
1448
  *[(x, x.qualifier) for x in qualifiers if x.resolved],
1380
1449
  ]
1381
- literals: list[Qualifier] = [x for x in qualifiers if not x.resolved]
1450
+ # no_map subject/object plus CURIE-ranged qualifiers that opted out (a no_map
1451
+ # enum-ranged qualifier is rejected at config time). Their encoded values become
1452
+ # node ids verbatim: encode only, then curie_audit drops blank rows, fails on
1453
+ # non-CURIE cells, and synthesizes the category column edge derivation needs.
1454
+ verbatim: list[tuple[NodeEncoding, str]] = [
1455
+ *[(x, col) for x, col in subj_obj if x.no_map],
1456
+ *[(x, x.qualifier) for x in qualifiers if not x.resolved and x.no_map],
1457
+ ]
1458
+ literals: list[Qualifier] = [x for x in qualifiers if not x.resolved and not x.no_map]
1382
1459
  # A nullable qualifier keeps its edge when the cell is blank or unresolvable (the
1383
1460
  # column stays null and the null-stripper omits the key); subject/object and
1384
1461
  # strict qualifiers drop the row as before.
@@ -1401,6 +1478,18 @@ class Tcode(Section):
1401
1478
  # ranged column ends with a vocabulary audit: nothing else validates its cell
1402
1479
  # values against the closed Biolink range before KGX emission.
1403
1480
  [[*self.encoding(x, x.qualifier), (qualifier_vocabulary_audit, (str(x.qualifier), sorted(x.vocabulary or ())))] for x in literals],
1481
+ # no_map columns are encoded exactly like enum-ranged literals (no NLP
1482
+ # normalization -- it exists only to feed entity resolution these columns
1483
+ # never undergo), then finalized by curie_audit: blank rows dropped,
1484
+ # non-CURIE cells failed loudly, and the pinned `biolink:NamedThing`
1485
+ # category added for downstream edge derivation. The `<col>_pre_resolution`
1486
+ # snapshot rides along because `_collect_subframes` discovers node columns
1487
+ # through it at graph-compile time; without it a verbatim subject/object
1488
+ # would never be split out of the edges into nodes.
1489
+ [
1490
+ [*self.encoding(x, col), (column, (f"{col}_pre_resolution", col)), (curie_audit, (col, self.store.stem, self.config.name))]
1491
+ for x, col in verbatim
1492
+ ],
1404
1493
  # Raw ``column_<n>`` columns are dead weight once the encodings above have
1405
1494
  # copied them into named slots; trim before resolution so the frame that
1406
1495
  # resolve_batch materializes and joins stays narrow.
@@ -1454,19 +1543,28 @@ class Tcode(Section):
1454
1543
  # (rig.source_info.infores_id); there is no implicit derivation from the graph
1455
1544
  # name, so a RIG and its edges can never disagree about the source identity.
1456
1545
  primary_knowledge_source: str | None = self.infores
1457
- # Per-edge record URLs for the primary entry: the section's declared source URLs,
1458
- # plus the EdgeLinkouts template when --linkouts is set. A section with ANY
1459
- # provenance.override is author-controlled (DAKP hand-writes the same template
1460
- # into override.sources), so the flag injects nothing there.
1461
- primary_urls: list[str] = [str(u) for u in self.source.url]
1462
- if self.linkouts and override is None:
1463
- primary_urls.append(EDGE_LINKOUT_URL_TEMPLATE)
1546
+ # Record URLs: the section's declared source URLs. On the derived path they
1547
+ # describe the UPSTREAM record the data was extracted from, so they are emitted
1548
+ # on the supporting entry of the publication repository (infores:pubmed-central
1549
+ # for PMC sections) and the primary entry -- the transforming resource --
1550
+ # carries only the EdgeLinkouts template when --linkouts is set. A section with
1551
+ # ANY provenance.override is author-controlled (DAKP hand-writes the same
1552
+ # template into override.sources): an upstream_source_record_urls mapping
1553
+ # re-homes URLs exactly as specified (primary gets nothing), any other override
1554
+ # keeps the legacy primary-entry emission, and the flag injects nothing there.
1555
+ record_urls: list[str] = [str(u) for u in self.source.url]
1464
1556
  upstream_ids = override.upstream_resource_ids if override else upstream_resource_ids(self.provenance.repo)
1465
- upstream_urls: dict[str, list[str]] | None = (
1466
- {key: [str(u) for u in urls] for key, urls in override.upstream_source_record_urls.items()}
1467
- if override and override.upstream_source_record_urls is not None
1468
- else None
1469
- )
1557
+ primary_urls: list[str]
1558
+ upstream_urls: dict[str, list[str]] | None
1559
+ if override is None:
1560
+ upstream_urls = {upstream_ids[0]: record_urls}
1561
+ primary_urls = [EDGE_LINKOUT_URL_TEMPLATE] if self.linkouts else []
1562
+ elif override.upstream_source_record_urls is not None:
1563
+ upstream_urls = {key: [str(u) for u in urls] for key, urls in override.upstream_source_record_urls.items()}
1564
+ primary_urls = []
1565
+ else:
1566
+ upstream_urls = None
1567
+ primary_urls = record_urls
1470
1568
  # An explicit `sources` template replaces the derived primary/upstream
1471
1569
  # emission entirely; the model already forbids combining it with
1472
1570
  # `upstream_resource_ids`/`upstream_source_record_urls`.
@@ -1476,19 +1574,17 @@ class Tcode(Section):
1476
1574
  knowledge_level = override.knowledge_level if override else self.provenance.knowledge_level
1477
1575
  agent_type = override.agent_type if override else self.provenance.agent_type
1478
1576
  publication_values = override.publications if override else [publication_curie(self.provenance.repo, self.provenance.publication or "")]
1479
- # The study is the table itself: one publication, one worksheet/table. The
1480
- # publication CURIE is the study's IDENTIFIER; the worksheet or source filename
1481
- # is its human-readable NAME. The two stay disjoint fields and are never composed
1577
+ # The study is the table section itself: one publication, one config section.
1578
+ # The publication CURIE is the study's IDENTIFIER; the originating table config
1579
+ # file plus the section's 1-based index is its human-readable NAME (e.g.
1580
+ # "KANG12.yaml SECTION 2"). The two stay disjoint fields and are never composed
1482
1581
  # into one string, so neither ever duplicates the other.
1483
- sheet: str | None = self.source.sheet if self.source.kind == Files.EXCEL else None # pyright: ignore
1484
1582
  # No publication means no study to name: the fallback id is the config stem (the
1485
1583
  # YAML filename without its extension), not an identifier, so
1486
1584
  # `inline_supporting_study` skips the struct unless it has content to carry.
1487
1585
  identified: bool = bool(publication_values)
1488
1586
  study_id: str = publication_values[0] if publication_values else self.config.stem
1489
- study_name: str | None = sheet if sheet is not None else Path(self.source.local).name
1490
- if study_name == study_id:
1491
- study_name = None
1587
+ study_name: str = f"{self.config.name} SECTION {self.section_index}"
1492
1588
  # `use_enum_values` stores the override's enum keys/values as plain strings.
1493
1589
  category_override: dict[str, str] | None = (
1494
1590
  {obj: f"biolink:{pinned}" for obj, pinned in self.statement.category_override.items()} if self.statement.category_override else None
@@ -1500,10 +1596,10 @@ class Tcode(Section):
1500
1596
  (value, ("agent_type", agent_type)),
1501
1597
  # Retrieval provenance lives only in the nested `sources` list (Biolink
1502
1598
  # RetrievalSource); current translator-ingests emits no flat
1503
- # `primary_knowledge_source` scalar, so neither do we. A per-upstream URL
1504
- # mapping (override.upstream_source_record_urls) re-homes the record URLs
1505
- # from the primary entry onto the matching supporting entries; an explicit
1506
- # `sources` template (override.sources) replaces the whole derivation.
1599
+ # `primary_knowledge_source` scalar, so neither do we. The derived path maps
1600
+ # the section's record URLs onto the upstream repository's supporting entry
1601
+ # (see above); an explicit `sources` template (override.sources) replaces
1602
+ # the whole derivation.
1507
1603
  (retrieval_sources, (primary_knowledge_source, upstream_ids, primary_urls, upstream_urls, explicit_sources))
1508
1604
  if primary_knowledge_source
1509
1605
  else None,
@@ -1558,6 +1654,7 @@ PHASE_OF: dict[Callable, str] = {
1558
1654
  fullmap_audit: "qc",
1559
1655
  column: "encode",
1560
1656
  qualifier_vocabulary_audit: "encode",
1657
+ curie_audit: "encode",
1561
1658
  edge_category: "edge",
1562
1659
  publications: "provenance",
1563
1660
  retrieval_sources: "provenance",
@@ -138,7 +138,7 @@ class BaseSource(TablaBase):
138
138
  url: list[HttpUrl] = Field(
139
139
  ...,
140
140
  min_length=1,
141
- description="One or more remote source URL(s) recorded as provenance; emitted in the edge `sources` list under the primary entry's `source_record_urls` list and in the RIG. When `provenance.override.upstream_source_record_urls` is set, these URLs serve the RIG only and the per-upstream mapping determines edge placement instead. Format-validated only; not fetched.",
141
+ description="One or more remote source URL(s) recorded as provenance. Derived sections emit them in the edge `sources` list under the upstream repository supporting entry's `source_record_urls` list (infores:pubmed-central for PMC sections) and in the RIG. When `provenance.override.upstream_source_record_urls` is set, these URLs serve the RIG only and the per-upstream mapping determines edge placement instead; any other provenance.override keeps the primary-entry emission. Format-validated only; not fetched.",
142
142
  )
143
143
 
144
144
  rows: list[NonNegativeInt] | None = Field(None, description="Zero-based row indices kept after any row_slice crop.", examples=[[0, 2, 5]])
@@ -288,6 +288,16 @@ class Encoding(TablaBase):
288
288
  )
289
289
 
290
290
 
291
+ #: CURIE shape enforced on verbatim (``no_map``) node values: non-empty ``PREFIX:local-id``.
292
+ #: Shared by config-time literal validation (models) and build-time cell audits (lib).
293
+ CURIE_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]*:[A-Za-z0-9_.\-/]+$")
294
+
295
+
296
+ def is_curie(value: object) -> bool:
297
+ """Whether ``value`` is a CURIE-shaped string (``PREFIX:local-id``, both halves non-empty)."""
298
+ return isinstance(value, str) and CURIE_PATTERN.match(value.strip()) is not None
299
+
300
+
291
301
  class NodeEncoding(Encoding):
292
302
  taxon: PositiveInt | None = Field(
293
303
  9606,
@@ -312,6 +322,87 @@ class NodeEncoding(Encoding):
312
322
  examples=[["^OMIM:\\d+$"]],
313
323
  )
314
324
 
325
+ no_map: bool = Field(
326
+ False,
327
+ description=(
328
+ "When True, skip fullmap entity resolution: the encoded value is used verbatim as the node CURIE, "
329
+ "for data that already carries its own CURIEs. No NLP normalization, taxon constraint, or category "
330
+ "lookup is applied; blank cells are dropped like unresolved nodes and non-CURIE cells fail the build."
331
+ ),
332
+ examples=[True],
333
+ )
334
+
335
+ @model_validator(mode="before")
336
+ @classmethod
337
+ def no_map_rejects_explicit_taxon(cls, data: Any) -> Any:
338
+ """Reject ``no_map`` with an explicit non-default ``taxon`` (dead resolution config).
339
+
340
+ ``taxon`` defaults to 9606 and only steers fullmap ranking, so an explicit
341
+ non-default taxon alongside ``no_map: true`` is config that can never take
342
+ effect. The default itself stays indistinguishable from an explicit 9606 and
343
+ is therefore allowed. Checked ``before`` precisely so "explicitly set" is
344
+ detectable from the raw input dict.
345
+ """
346
+ if isinstance(data, dict) and data.get("no_map") and "taxon" in data and data["taxon"] not in (None, 9606):
347
+ raise TablassertValidationError(
348
+ f"`no_map: true` skips entity resolution, so `taxon: {data['taxon']}` can never take effect; remove the taxon constraint.",
349
+ code="no-map-dead-config",
350
+ )
351
+ return data
352
+
353
+ @model_validator(mode="after")
354
+ def no_map_rejects_resolution_knobs(self: Self) -> Self:
355
+ """Reject ``no_map`` combined with resolution-only knobs (dead config).
356
+
357
+ ``prioritize``, ``avoid``, ``exclude_prefixes``, and ``exclude_regex`` only
358
+ steer fullmap ranking and filtering; with ``no_map: true`` no lookup happens,
359
+ so these fields mislead the reader. Fail loudly at config time, like the
360
+ ``nullable``-on-literal check on :class:`Qualifier`.
361
+ """
362
+ if not self.no_map:
363
+ return self
364
+ dead: list[str] = [
365
+ name
366
+ for name, value in (
367
+ ("prioritize", self.prioritize),
368
+ ("avoid", self.avoid),
369
+ ("exclude_prefixes", self.exclude_prefixes),
370
+ ("exclude_regex", self.exclude_regex),
371
+ )
372
+ if value
373
+ ]
374
+ if dead:
375
+ raise TablassertValidationError(
376
+ f"`no_map: true` skips entity resolution, so {', '.join(f'`{name}`' for name in dead)} can never take effect; remove them.",
377
+ code="no-map-dead-config",
378
+ )
379
+ return self
380
+
381
+ @model_validator(mode="after")
382
+ def no_map_literal_must_be_curie(self: Self) -> Self:
383
+ """Validate that a ``no_map`` literal encodes a CURIE at config time.
384
+
385
+ A ``method: value`` node with ``no_map: true`` becomes the node id verbatim,
386
+ so the literal (after ``prefix``/``suffix`` are applied, matching the build's
387
+ ``encoding()``) must already be CURIE-shaped; otherwise every row would be
388
+ dropped at build time. ``method: column`` cells are audited per row at build
389
+ time instead.
390
+ """
391
+ if not self.no_map or self.method != EncodingMethods.VALUE:
392
+ return self
393
+ # Enum-ranged qualifier literals are closed-vocabulary tokens (audited against
394
+ # the vocabulary, and `no_map` on them is rejected as dead config), not CURIEs.
395
+ if getattr(self, "vocabulary", None) is not None:
396
+ return self
397
+ literal: str = f"{self.prefix or ''}{self.encoding}{self.suffix or ''}".strip()
398
+ if not is_curie(literal):
399
+ raise TablassertValidationError(
400
+ f"`no_map: true` uses the encoded value verbatim as the node CURIE, but the literal {literal!r} is not CURIE-shaped "
401
+ "(expected `PREFIX:local-id`). Fix the literal, or add `prefix`/`suffix` to form a CURIE.",
402
+ code="no-map-bad-curie",
403
+ )
404
+ return self
405
+
315
406
  @field_validator("exclude_regex", mode="after")
316
407
  @classmethod
317
408
  def polars_compatible_exclude_regex(cls, exclude_regex: list[str] | None) -> list[str] | None:
@@ -359,10 +450,26 @@ class Qualifier(NodeEncoding):
359
450
  def resolved(self: Self) -> bool:
360
451
  """Whether this qualifier's values go through fullmap entity resolution.
361
452
 
362
- Only CURIE-ranged qualifiers are resolved. Enum-ranged ones carry a literal
363
- token from a closed Biolink vocabulary and must be passed through verbatim.
453
+ Enum-ranged qualifiers carry a literal token from a closed Biolink vocabulary
454
+ and must be passed through verbatim. CURIE-ranged qualifiers are resolved
455
+ unless they opt out with ``no_map: true`` (their values are already CURIEs).
456
+ """
457
+ return self.vocabulary is None and not self.no_map
458
+
459
+ @model_validator(mode="after")
460
+ def reject_no_map_on_enum_ranged(self: Self) -> Self:
461
+ """Reject ``no_map: true`` on enum-ranged qualifiers as dead config.
462
+
463
+ Enum-ranged qualifiers already pass their literal tokens through verbatim, so
464
+ ``no_map`` would be a no-op that misleads the reader; fail loudly at config
465
+ time, like the ``nullable``-on-literal check below.
364
466
  """
365
- return self.vocabulary is None
467
+ if self.no_map and self.vocabulary is not None:
468
+ raise TablassertValidationError(
469
+ f"{self.qualifier} has a closed Biolink vocabulary and already passes values through verbatim, so `no_map` is redundant; remove it.",
470
+ code="no-map-dead-config",
471
+ )
472
+ return self
366
473
 
367
474
  @model_validator(mode="after")
368
475
  def reject_disabled_qualifiers(self: Self) -> Self:
@@ -272,6 +272,9 @@ OP_COST: dict[Callable, int] = {
272
272
  lib.split_list: 1,
273
273
  # Vocabulary audit: one eager distinct-value pass over a single enum column.
274
274
  lib.qualifier_vocabulary_audit: 1,
275
+ # CURIE audit: one eager pass over a single no_map column (distinct-value check,
276
+ # blank-row filter, category column synthesis).
277
+ lib.curie_audit: 1,
275
278
  # Clean: coerce_columns applies five coercion rules (schema resolve + renames + value
276
279
  # coercion) in one pass, so it outprices a single-expression clean op.
277
280
  lib.coerce_columns: 3,
File without changes
File without changes
File without changes
File without changes
File without changes