tablassert 19.5.1__tar.gz → 19.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {tablassert-19.5.1 → tablassert-19.7.0}/PKG-INFO +2 -2
  2. {tablassert-19.5.1 → tablassert-19.7.0}/pyproject.toml +3 -3
  3. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/biolink.py +205 -42
  4. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/cli.py +146 -20
  5. tablassert-19.7.0/src/tablassert/data/infores_catalog.yaml +6166 -0
  6. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/errors.py +1 -0
  7. tablassert-19.7.0/src/tablassert/infores_registry.py +318 -0
  8. {tablassert-19.5.1 → tablassert-19.7.0}/LICENSE +0 -0
  9. {tablassert-19.5.1 → tablassert-19.7.0}/README.md +0 -0
  10. {tablassert-19.5.1 → tablassert-19.7.0}/rust/Cargo.lock +0 -0
  11. {tablassert-19.5.1 → tablassert-19.7.0}/rust/Cargo.toml +0 -0
  12. {tablassert-19.5.1 → tablassert-19.7.0}/rust/examples/count_tables.rs +0 -0
  13. {tablassert-19.5.1 → tablassert-19.7.0}/rust/src/fullmap.rs +0 -0
  14. {tablassert-19.5.1 → tablassert-19.7.0}/rust/src/json.rs +0 -0
  15. {tablassert-19.5.1 → tablassert-19.7.0}/rust/src/lib.rs +0 -0
  16. {tablassert-19.5.1 → tablassert-19.7.0}/rust/src/ndjson.rs +0 -0
  17. {tablassert-19.5.1 → tablassert-19.7.0}/rust/src/nlp.rs +0 -0
  18. {tablassert-19.5.1 → tablassert-19.7.0}/rust/src/uuid.rs +0 -0
  19. {tablassert-19.5.1 → tablassert-19.7.0}/rust/tests/build_golden.rs +0 -0
  20. {tablassert-19.5.1 → tablassert-19.7.0}/rust/tests/common/mod.rs +0 -0
  21. {tablassert-19.5.1 → tablassert-19.7.0}/rust/tests/extract_prebuilt.rs +0 -0
  22. {tablassert-19.5.1 → tablassert-19.7.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
  23. {tablassert-19.5.1 → tablassert-19.7.0}/rust/tests/nlp_golden.rs +0 -0
  24. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/__init__.py +0 -0
  25. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/_lazy.py +0 -0
  26. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/agent.py +0 -0
  27. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/cli_entry.py +0 -0
  28. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/coerce.py +0 -0
  29. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
  30. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/distill.py +0 -0
  31. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/distill_reward.py +0 -0
  32. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/enums.py +0 -0
  33. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/extras.py +0 -0
  34. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/fullmap.py +0 -0
  35. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/graph_target.py +0 -0
  36. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/ingests.py +0 -0
  37. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/lib.py +0 -0
  38. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/log.py +0 -0
  39. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/models.py +0 -0
  40. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/net.py +0 -0
  41. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/nlp.py +0 -0
  42. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/progress.py +0 -0
  43. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/qc.py +0 -0
  44. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/rig.py +0 -0
  45. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/rs.pyi +0 -0
  46. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/runcache.py +0 -0
  47. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/study.py +0 -0
  48. {tablassert-19.5.1 → tablassert-19.7.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 19.5.1
3
+ Version: 19.7.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -22,7 +22,7 @@ Classifier: Framework :: Pydantic :: 2
22
22
  Classifier: Operating System :: POSIX :: Linux
23
23
  Classifier: Operating System :: MacOS :: MacOS X
24
24
  Classifier: Environment :: Console
25
- Requires-Dist: biolink-model>=4.4.5
25
+ Requires-Dist: biolink-model>=4.4.6
26
26
  Requires-Dist: polars>=2.0.0rc2
27
27
  Requires-Dist: pyarrow>=21.0.0
28
28
  Requires-Dist: rapidfuzz>=3.14.3
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "19.5.1"
3
+ version = "19.7.0"
4
4
  description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -47,7 +47,7 @@ classifiers = [
47
47
  ]
48
48
  requires-python = ">=3.11"
49
49
  dependencies = [
50
- "biolink-model>=4.4.5",
50
+ "biolink-model>=4.4.6",
51
51
  "polars>=2.0.0rc2",
52
52
  "pyarrow>=21.0.0",
53
53
  "rapidfuzz>=3.14.3",
@@ -71,7 +71,7 @@ module-name = "tablassert.rs"
71
71
  bindings = "pyo3"
72
72
  manifest-path = "rust/Cargo.toml"
73
73
  features = ["extension-module"]
74
- include = ["src/tablassert/data/experimental_taxa.yaml"]
74
+ include = ["src/tablassert/data/experimental_taxa.yaml", "src/tablassert/data/infores_catalog.yaml"]
75
75
 
76
76
  [project.scripts]
77
77
  tablassert = "tablassert.cli_entry:main"
@@ -46,6 +46,7 @@ from __future__ import annotations
46
46
 
47
47
  import inspect
48
48
  import json
49
+ import os
49
50
  import re
50
51
  from collections import Counter
51
52
  from enum import Enum
@@ -86,6 +87,7 @@ __all__ = [
86
87
  "legal_predicates",
87
88
  "node_class",
88
89
  "numeric_slot_kind",
90
+ "prune_kgx_edges",
89
91
  "resolve_association_class",
90
92
  "resolve_node_category",
91
93
  "resolve_node_class",
@@ -591,8 +593,12 @@ release from making ``species_context_qualifier`` silently emittable again.
591
593
 
592
594
 
593
595
  CLASS_FIELD_OVERRIDES: dict[str, frozenset[str]] = {
594
- "EntityToDiseaseAssociation": frozenset({"population_context_qualifier", "sex_qualifier", "temporal_context_qualifier"}),
595
- "EntityToPhenotypicFeatureAssociation": frozenset({"population_context_qualifier", "temporal_context_qualifier"}),
596
+ # Absorbed by biolink-model 4.4.6 (now declared natively on both pinned classes,
597
+ # removed by the tripwire; kept here commented as the pattern for future grants):
598
+ # "EntityToDiseaseAssociation": frozenset({"population_context_qualifier", "sex_qualifier", "temporal_context_qualifier"}),
599
+ # "EntityToPhenotypicFeatureAssociation": frozenset({"population_context_qualifier", "temporal_context_qualifier"}),
600
+ "EntityToDiseaseAssociation": frozenset({"sex_qualifier", "temporal_context_qualifier"}),
601
+ "EntityToPhenotypicFeatureAssociation": frozenset({"temporal_context_qualifier"}),
596
602
  }
597
603
  """Per-class grants of edge fields the resolved association class does not declare.
598
604
 
@@ -610,11 +616,12 @@ mixin consolidation attached ``disease_context_qualifier``,
610
616
  ``anatomical_context_qualifier``, and ``frequency_qualifier`` to the whole
611
617
  disease/phenotype family, renamed ``FDA_regulatory_approvals`` to the canonical
612
618
  ``regulatory_approvals`` on the pinned classes, and added ``sex_qualifier`` to
613
- ``EntityToPhenotypicFeatureAssociation``); the tripwire removed those grants the day
614
- the release landed, and the remaining ones stay deliberately ahead of the model
615
- (pending an upstream Biolink widening). ``_validation_record`` strips granted fields
616
- before record validation so the deliberate gap is not reported as
617
- ``extra_forbidden``.
619
+ ``EntityToPhenotypicFeatureAssociation``; biolink-model 4.4.6 then attached
620
+ ``population_context_qualifier`` to both pinned classes, so the tripwire removed
621
+ those grants the day each release landed). The remaining ones stay deliberately
622
+ ahead of the model (pending an upstream Biolink widening).
623
+ ``_validation_record`` strips granted fields before record validation so the
624
+ deliberate gap is not reported as ``extra_forbidden``.
618
625
 
619
626
  A tripwire test asserts every granted field is still absent from its class: the moment
620
627
  a biolink-model release attaches the slot, the suite fails and the stale grant is
@@ -944,45 +951,201 @@ def validate_kgx(nodes_path: Path, edges_path: Path, limit: int = 20) -> dict[st
944
951
  """
945
952
  report: dict[str, Any] = {"biolink_version": BIOLINK_VERSION, "ok": True, "ok_excluding_pending": True}
946
953
  for label, path, edge in (("nodes", nodes_path, False), ("edges", edges_path, True)):
947
- total: int = 0
948
- valid: int = 0
949
- valid_excluding_pending: int = 0
950
- problems: Counter[str] = Counter()
951
- examples: list[dict[str, Any]] = []
952
- # A missing path must never read as a clean bill of health: counting zero records
953
- # out of zero would otherwise exit 0 on a typo'd filename and hide a broken build.
954
- missing: bool = not path.is_file()
955
- if not missing:
956
- with path.open(encoding="utf-8") as handle:
957
- for line in handle:
954
+ section: dict[str, Any] = _validate_file(path, edge=edge, limit=limit)
955
+ report[label] = section
956
+ if section["missing"] or section["total"] != section["valid"]:
957
+ report["ok"] = False
958
+ if section["missing"] or section["total"] != section["valid_excluding_pending"]:
959
+ report["ok_excluding_pending"] = False
960
+ return report
961
+
962
+
963
+ def _validate_file(path: Path, *, edge: bool, limit: int) -> dict[str, Any]:
964
+ """Validate one KGX NDJSON file record by record (the per-file half of ``validate_kgx``).
965
+
966
+ Args:
967
+ path: Path to the ``.ndjson`` file; a missing path reports ``missing`` instead of
968
+ counting zero records out of zero as a pass.
969
+ edge: ``True`` to dispatch on the association family, ``False`` for nodes.
970
+ limit: Maximum number of example failures to retain.
971
+
972
+ Returns:
973
+ The same per-file section ``validate_kgx`` nests under ``"nodes"`` / ``"edges"``.
974
+ """
975
+ total: int = 0
976
+ valid: int = 0
977
+ valid_excluding_pending: int = 0
978
+ problems: Counter[str] = Counter()
979
+ examples: list[dict[str, Any]] = []
980
+ # A missing path must never read as a clean bill of health: counting zero records
981
+ # out of zero would otherwise exit 0 on a typo'd filename and hide a broken build.
982
+ missing: bool = not path.is_file()
983
+ if not missing:
984
+ with path.open(encoding="utf-8") as handle:
985
+ for line in handle:
986
+ if not line.strip():
987
+ continue
988
+ total += 1
989
+ record: dict[str, Any] = json.loads(line)
990
+ errors: list[str] = validate_record(record, edge=edge)
991
+ if not errors:
992
+ valid += 1
993
+ valid_excluding_pending += 1
994
+ continue
995
+ if _record_has_only_pending_problems(record, errors):
996
+ valid_excluding_pending += 1
997
+ problems.update(errors)
998
+ if len(examples) < limit:
999
+ examples.append({"id": record.get("id"), "errors": errors})
1000
+ return {
1001
+ "total": total,
1002
+ "valid": valid,
1003
+ "valid_excluding_pending": valid_excluding_pending,
1004
+ "failures": total - valid,
1005
+ "missing": missing,
1006
+ "problems": dict(problems.most_common()),
1007
+ "examples": examples,
1008
+ }
1009
+
1010
+
1011
+ def prune_kgx_edges(edges_path: Path, limit: int = 20) -> dict[str, Any]:
1012
+ """Drop every edge record that is not strictly KGX/Biolink-compliant, in place.
1013
+
1014
+ A record survives only when it validates against the Biolink Pydantic class named
1015
+ by its own ``category`` with zero errors -- the same strict bar a plain
1016
+ ``validate_kgx`` run uses for its ``ok`` verdict, so a prune can never disagree
1017
+ with what validation reports:
1018
+
1019
+ * a strictly valid record is kept;
1020
+ * a record whose every failure is a known, intentional model gap (the KGX
1021
+ denormalized carryovers in :data:`KNOWN_PENDING_EDGE_FIELDS`, or a
1022
+ :data:`PREDICATE_OVERRIDES` predicate the pinned classes reject) is dropped --
1023
+ these carryovers are most of a real graph, so the caller asked for a
1024
+ strictly-compliant graph over maximal data retention when invoking this;
1025
+ * every other failing record is a real defect and is dropped;
1026
+ * a line that is not valid JSON at all is dropped and counted separately in
1027
+ ``malformed``.
1028
+
1029
+ Nodes are never touched: dropping one would strand references, and cascading that
1030
+ into the edges is a modelling decision, not a validation repair.
1031
+
1032
+ The rewrite is ATOMIC and lossless for kept records: kept lines are written verbatim
1033
+ (byte-identical, order preserved, blank lines passed through) into a sibling
1034
+ ``.{name}.tmp`` and swapped in with :func:`os.replace` only once every line was
1035
+ written, so a crash never leaves a torn edges file. When nothing is prunable the
1036
+ destination is not rewritten at all (``rewritten`` is ``False`` and the bytes are
1037
+ untouched), and a missing file is reported (``missing``) without ever being created
1038
+ -- a typo'd path must never read as a prune.
1039
+
1040
+ Args:
1041
+ edges_path: Path to ``<name>_<version>.edges.ndjson``.
1042
+ limit: Maximum number of example failures to retain per report section.
1043
+
1044
+ Returns:
1045
+ Mapping with ``before`` (the same per-file section ``validate_kgx`` builds over
1046
+ the original file), ``after`` (the same shape over the kept records, i.e. the
1047
+ file's post-prune state, equal to what ``validate_kgx`` reports on the rewritten
1048
+ file), ``dropped`` (the ``pruned`` records' problem histogram and examples --
1049
+ the records that were removed), ``pruned`` / ``kept`` / ``malformed`` /
1050
+ ``rewritten`` counts, and ``ok`` / ``ok_excluding_pending`` computed over the
1051
+ post-prune state. For a readable file ``ok`` is therefore always true -- every
1052
+ kept record is strictly valid -- and ``ok_excluding_pending`` agrees, because
1053
+ nothing pending survives the prune.
1054
+
1055
+ Raises:
1056
+ OSError: the destination's directory is unwritable; the error propagates and the
1057
+ original file keeps its previous content.
1058
+ """
1059
+ total: int = 0
1060
+ valid: int = 0
1061
+ pending: int = 0 # pending-only failures: counted for the BEFORE report, never kept
1062
+ pruned: int = 0
1063
+ malformed: int = 0
1064
+ all_problems: Counter[str] = Counter()
1065
+ all_examples: list[dict[str, Any]] = []
1066
+ pruned_problems: Counter[str] = Counter()
1067
+ pruned_examples: list[dict[str, Any]] = []
1068
+ # A missing path must never read as a clean bill of health: reporting zero records
1069
+ # out of zero would otherwise exit 0 on a typo'd filename and hide a broken build.
1070
+ missing: bool = not edges_path.is_file()
1071
+ if not missing:
1072
+ tmp: Path = edges_path.with_name(f".{edges_path.name}.tmp")
1073
+ try:
1074
+ # newline="" on both ends keeps every kept line byte-identical: the default
1075
+ # universal-newlines mode would silently rewrite "\r\n" as "\n".
1076
+ with edges_path.open(encoding="utf-8", newline="") as src, tmp.open("w", encoding="utf-8", newline="") as dst:
1077
+ for line in src:
958
1078
  if not line.strip():
1079
+ dst.write(line) # blank lines are not records; pass through verbatim
959
1080
  continue
960
1081
  total += 1
961
- record: dict[str, Any] = json.loads(line)
962
- errors: list[str] = validate_record(record, edge=edge)
963
- if not errors:
964
- valid += 1
965
- valid_excluding_pending += 1
1082
+ try:
1083
+ record: dict[str, Any] = json.loads(line)
1084
+ except json.JSONDecodeError:
1085
+ malformed += 1
1086
+ pruned += 1
1087
+ pruned_problems["record: json_parse_error"] += 1
1088
+ if len(pruned_examples) < limit:
1089
+ pruned_examples.append({"id": None, "errors": ["record: json_parse_error"]})
966
1090
  continue
967
- if _record_has_only_pending_problems(record, errors):
968
- valid_excluding_pending += 1
969
- problems.update(errors)
970
- if len(examples) < limit:
971
- examples.append({"id": record.get("id"), "errors": errors})
972
- report[label] = {
973
- "total": total,
974
- "valid": valid,
975
- "valid_excluding_pending": valid_excluding_pending,
976
- "failures": total - valid,
977
- "missing": missing,
978
- "problems": dict(problems.most_common()),
979
- "examples": examples,
980
- }
981
- if missing or total != valid:
982
- report["ok"] = False
983
- if missing or total != valid_excluding_pending:
984
- report["ok_excluding_pending"] = False
985
- return report
1091
+ errors: list[str] = validate_record(record, edge=True)
1092
+ all_problems.update(errors)
1093
+ if len(all_examples) < limit:
1094
+ all_examples.append({"id": record.get("id"), "errors": errors})
1095
+ if errors:
1096
+ # The before-section keeps validate_kgx's pending score so the
1097
+ # pre-report matches a plain validation run; the STRICT bar still
1098
+ # drops the record -- real defects AND deliberate pending gaps.
1099
+ if _record_has_only_pending_problems(record, errors):
1100
+ pending += 1
1101
+ pruned += 1
1102
+ pruned_problems.update(errors)
1103
+ if len(pruned_examples) < limit:
1104
+ pruned_examples.append({"id": record.get("id"), "errors": errors})
1105
+ continue
1106
+ valid += 1
1107
+ dst.write(line)
1108
+ # ATOMIC: the swap happens only after every kept line was written, so a crash
1109
+ # mid-write leaves the original file untouched and the temp never read back.
1110
+ rewritten: bool = pruned > 0
1111
+ if rewritten:
1112
+ os.replace(tmp, edges_path)
1113
+ finally:
1114
+ tmp.unlink(missing_ok=True) # no-op after a successful replace; cleanup after a raise
1115
+ else:
1116
+ rewritten = False
1117
+ kept: int = total - pruned
1118
+ before: dict[str, Any] = {
1119
+ "total": total,
1120
+ "valid": valid,
1121
+ "valid_excluding_pending": valid + pending,
1122
+ "failures": total - valid,
1123
+ "missing": missing,
1124
+ "problems": dict(all_problems.most_common()),
1125
+ "examples": all_examples,
1126
+ }
1127
+ dropped: dict[str, Any] = {"total": pruned, "problems": dict(pruned_problems.most_common()), "examples": pruned_examples}
1128
+ after: dict[str, Any] = {
1129
+ "total": kept,
1130
+ "valid": valid,
1131
+ "valid_excluding_pending": valid, # nothing pending survives, so the scores agree
1132
+ "failures": kept - valid,
1133
+ "missing": missing,
1134
+ "problems": {},
1135
+ "examples": [],
1136
+ }
1137
+ return {
1138
+ "biolink_version": BIOLINK_VERSION,
1139
+ "before": before,
1140
+ "after": after,
1141
+ "dropped": dropped,
1142
+ "pruned": pruned,
1143
+ "kept": kept,
1144
+ "malformed": malformed,
1145
+ "rewritten": rewritten,
1146
+ "ok": not missing and kept == valid,
1147
+ "ok_excluding_pending": not missing and kept == valid,
1148
+ }
986
1149
 
987
1150
 
988
1151
  # Node slots Tablassert can always populate from a fullmap hit. A class requiring
@@ -955,11 +955,29 @@ def validate(
955
955
  run(3, validate_pipeline, configuration_file)
956
956
 
957
957
 
958
+ def _print_kgx_section(label: str, section: dict[str, Any], path: Path) -> None:
959
+ """Print one ``validate-kgx`` per-file section (shared by the plain and ``--prune`` paths)."""
960
+ if section["missing"]:
961
+ # Never let a typo'd path read as a pass: 0/0 valid would otherwise exit 0.
962
+ print(f"{label}: file not found ({path})", file=sys.stderr)
963
+ logger.info(f"validate-kgx {label}: file not found")
964
+ return
965
+ pending: int = section["valid_excluding_pending"] - section["valid"]
966
+ suffix: str = f"; {pending} pending biolink-model support" if pending else ""
967
+ print(f"{label}: {section['valid']}/{section['total']} valid ({section['failures']} failures{suffix})", file=sys.stderr)
968
+ for problem, count in section["problems"].items():
969
+ print(f" {count:>9} {problem}", file=sys.stderr)
970
+ for example in section["examples"][:3]:
971
+ print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
972
+ logger.info(f"validate-kgx {label}: {section['valid']}/{section['total']} valid")
973
+
974
+
958
975
  @APP.command(name="validate-kgx")
959
976
  def validate_kgx_command(
960
977
  nodes: Annotated[Path, cyclopts.Parameter(name=["--nodes", "-n"])],
961
978
  edges: Annotated[Path, cyclopts.Parameter(name=["--edges", "-e"])],
962
979
  limit: Annotated[int, cyclopts.Parameter(name=["--limit"])] = 20,
980
+ prune: Annotated[bool, cyclopts.Parameter(name=["--prune", "-p"])] = False,
963
981
  ) -> None:
964
982
  """Validate built KGX NDJSON against the Biolink Model.
965
983
 
@@ -968,38 +986,146 @@ def validate_kgx_command(
968
986
  and reports failures grouped by field and error type. Exits non-zero when any
969
987
  record fails, so a build can be gated in CI.
970
988
 
989
+ ``--prune`` rewrites the EDGES file in place, keeping only edges that validate
990
+ strictly. Real defects (non-coercible values, extras the model will never declare,
991
+ rejected predicates), the deliberate pending carryovers (``synonym``/``xref``/
992
+ ``relation``/``provided_by``, ...), and lines that are not JSON at all are all
993
+ removed. On a real graph the carryovers are most of the edges, so ``--prune``
994
+ trades them for a strictly-compliant graph. The rewrite is atomic and kept lines
995
+ stay byte-identical; a clean file is not rewritten at all, and a missing edges file
996
+ is reported without ever being pruned. Nodes are validated and reported, never
997
+ pruned. The exit code stays strict over the post-prune state: after a prune, every
998
+ kept edge is strictly valid, so the run is compliant exactly when the nodes are.
999
+
971
1000
  Args:
972
1001
  nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
973
1002
  edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
974
1003
  limit: Maximum example failures retained per file for the ``e.g.`` report lines;
975
1004
  every record is still validated regardless of this cap. A missing file is
976
1005
  reported and fails the run rather than reading as a pass.
1006
+ prune: Rewrite the edges file in place, removing every edge record that is not
1007
+ strictly KGX/Biolink-compliant, including the deliberate pending carryovers.
1008
+ The nodes file is never modified; a clean edges file is left untouched.
977
1009
  """
978
- from tablassert.biolink import validate_kgx
979
-
980
- report: dict[str, Any] = validate_kgx(nodes, edges, limit=limit)
981
- print(f"biolink-model {report['biolink_version']}", file=sys.stderr)
982
- for label in ("nodes", "edges"):
983
- section: dict[str, Any] = report[label]
984
- if section["missing"]:
985
- # Never let a typo'd path read as a pass: 0/0 valid would otherwise exit 0.
986
- print(f"{label}: file not found ({nodes if label == 'nodes' else edges})", file=sys.stderr)
987
- logger.info(f"validate-kgx {label}: file not found")
988
- continue
989
- pending: int = section["valid_excluding_pending"] - section["valid"]
990
- suffix: str = f"; {pending} pending biolink-model support" if pending else ""
991
- print(f"{label}: {section['valid']}/{section['total']} valid ({section['failures']} failures{suffix})", file=sys.stderr)
992
- for problem, count in section["problems"].items():
993
- print(f" {count:>9} {problem}", file=sys.stderr)
994
- for example in section["examples"][:3]:
995
- print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
996
- logger.info(f"validate-kgx {label}: {section['valid']}/{section['total']} valid")
997
- if not report["ok"]:
1010
+ from tablassert.biolink import _validate_file, prune_kgx_edges, validate_kgx
1011
+
1012
+ if not prune:
1013
+ report: dict[str, Any] = validate_kgx(nodes, edges, limit=limit)
1014
+ print(f"biolink-model {report['biolink_version']}", file=sys.stderr)
1015
+ _print_kgx_section("nodes", report["nodes"], nodes)
1016
+ _print_kgx_section("edges", report["edges"], edges)
1017
+ if not report["ok"]:
1018
+ print("KGX output is not Biolink-compliant.", file=sys.stderr)
1019
+ raise SystemExit(1)
1020
+ print("KGX output is Biolink-compliant.", file=sys.stderr)
1021
+ return
1022
+
1023
+ # The prune path validates nodes-only itself and drives prune_kgx_edges' own single
1024
+ # pass for both edges sections, so the edges file is never read and written twice.
1025
+ nodes_section: dict[str, Any] = _validate_file(nodes, edge=False, limit=limit)
1026
+ pruned_report: dict[str, Any] = prune_kgx_edges(edges, limit=limit)
1027
+ print(f"biolink-model {pruned_report['biolink_version']}", file=sys.stderr)
1028
+ _print_kgx_section("nodes", nodes_section, nodes)
1029
+ _print_kgx_section("edges", pruned_report["before"], edges)
1030
+ if pruned_report["before"]["missing"]:
1031
+ print("KGX output is not Biolink-compliant.", file=sys.stderr)
1032
+ raise SystemExit(1)
1033
+ malformed_note: str = f" ({pruned_report['malformed']} malformed lines)" if pruned_report["malformed"] else ""
1034
+ print(f"edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} non-compliant records{malformed_note}", file=sys.stderr)
1035
+ logger.info(f"validate-kgx edges: pruned {pruned_report['pruned']}/{pruned_report['before']['total']} (rewritten={pruned_report['rewritten']})")
1036
+ for problem, count in pruned_report["dropped"]["problems"].items():
1037
+ print(f" {count:>9} {problem}", file=sys.stderr)
1038
+ for example in pruned_report["dropped"]["examples"][:3]:
1039
+ print(f" e.g. {example['id']}: {', '.join(example['errors'])}", file=sys.stderr)
1040
+ _print_kgx_section("edges", pruned_report["after"], edges)
1041
+ nodes_ok: bool = not nodes_section["missing"] and nodes_section["total"] == nodes_section["valid"]
1042
+ if not (nodes_ok and pruned_report["ok"]):
998
1043
  print("KGX output is not Biolink-compliant.", file=sys.stderr)
999
1044
  raise SystemExit(1)
1000
1045
  print("KGX output is Biolink-compliant.", file=sys.stderr)
1001
1046
 
1002
1047
 
1048
+ @APP.command(name="validate-infores")
1049
+ def validate_infores_command(
1050
+ nodes: Annotated[Path, cyclopts.Parameter(name=["--nodes", "-n"])],
1051
+ edges: Annotated[Path, cyclopts.Parameter(name=["--edges", "-e"])],
1052
+ rig: Annotated[Path | None, cyclopts.Parameter(name=["--rig"])] = None,
1053
+ allow_infores: Annotated[list[str] | None, cyclopts.Parameter(name=["--allow-infores"], negative="", allow_leading_hyphen=False)] = None,
1054
+ registry: Annotated[Literal["snapshot", "refresh", "off"], cyclopts.Parameter(name=["--registry"])] = "snapshot",
1055
+ strict: Annotated[bool, cyclopts.Parameter(name=["--strict"])] = False,
1056
+ limit: Annotated[int, cyclopts.Parameter(name=["--limit"])] = 20,
1057
+ ) -> None:
1058
+ """Validate emitted infores CURIEs against the NCATS Translator registry.
1059
+
1060
+ Every knowledge-source identifier the build emits -- edge ``primary_knowledge_source``,
1061
+ ``sources[].resource_id`` and ``sources[].upstream_resource_ids``, node ``provided_by``,
1062
+ and, with ``--rig``, the RIG document's infores values -- is classified against
1063
+ ``biolink/information-resource-registry``'s ``infores_catalog.yaml`` (vendored as the
1064
+ bundled snapshot, so the default path is offline). Values without the ``infores:``
1065
+ prefix are malformed, not unregistered.
1066
+
1067
+ The default posture is advisory: unregistered CURIEs are reported and the command
1068
+ exits 0, because Tablassert legitimately mints graph-local CURIEs that will never sit
1069
+ in the registry. ``--strict`` (CI gates) exits 1 when any unregistered CURIE remains;
1070
+ allowlist legitimate local ids with repeatable ``--allow-infores``. Missing input
1071
+ files fail in every mode: a typo'd path must never read as a clean bill of health.
1072
+
1073
+ Args:
1074
+ nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
1075
+ edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
1076
+ rig: Optional RIG yaml (``<name>_<version>.RIG.yaml``); its infores values are
1077
+ classified too.
1078
+ allow-infores: Registry-exempt CURIE, repeatable; use for the locally-minted
1079
+ graph identifier (``rig.infores()``) and other deliberate locals.
1080
+ registry: Registry source. ``snapshot`` (default) validates offline against the
1081
+ bundled copy; ``refresh`` fetches the live upstream catalog (network) and
1082
+ caches it under the user cache dir; ``off`` skips membership classification
1083
+ and reports structure only.
1084
+ strict: Exit non-zero when any unregistered CURIE remains. Without it the run
1085
+ is advisory and always exits 0 once inputs exist.
1086
+ limit: Maximum number of example unregistered/malformed CURIEs retained for the
1087
+ ``e.g.`` report lines; every observation is still classified regardless.
1088
+ """
1089
+ from tablassert.infores_registry import REGISTRY_RAW_URL, refresh_registry, validate_infores
1090
+
1091
+ ids: frozenset[str] | None = None
1092
+ if registry == "refresh":
1093
+ ids = refresh_registry()
1094
+ report: dict[str, Any] = validate_infores(
1095
+ nodes, edges, rig_path=rig, allow=tuple(allow_infores or ()), registry=ids, check_registry=registry != "off", limit=limit
1096
+ )
1097
+ if ids is not None:
1098
+ report["registry"]["source"] = f"{REGISTRY_RAW_URL} (refreshed)"
1099
+ for label, absent in report["missing"].items():
1100
+ if absent:
1101
+ path_label = {"nodes": nodes, "edges": edges, "rig": rig}.get(label)
1102
+ print(f"{label}: file not found ({path_label})", file=sys.stderr)
1103
+ if any(report["missing"].values()):
1104
+ print("infores validation failed: missing input files.", file=sys.stderr)
1105
+ raise SystemExit(1)
1106
+ print(f"registry: {report['registry']['source']} ({report['registry']['entries']} entries)", file=sys.stderr)
1107
+ counts: str = (
1108
+ f"curies: {report['registered']} registered, {report['allowed']} allowed, "
1109
+ f"{report['unregistered']} unregistered, {report['malformed']} malformed"
1110
+ )
1111
+ print(counts, file=sys.stderr)
1112
+ logger.info(f"validate-infores: {counts}")
1113
+ for example in report["examples"][:3]:
1114
+ print(f" e.g. {example['curie']} ({example['where']}.{example['field']}): {example['problem']}", file=sys.stderr)
1115
+ clean: bool = not report["unregistered"] and not report["malformed"]
1116
+ passed: str = "infores membership check skipped (--registry off)." if registry == "off" else "All emitted infores CURIEs are registered."
1117
+ if strict:
1118
+ if report["ok_strict"]:
1119
+ print(passed if clean else "infores validation passed (strict).", file=sys.stderr)
1120
+ return
1121
+ print("infores validation failed (strict): unregistered CURIEs remain.", file=sys.stderr)
1122
+ raise SystemExit(1)
1123
+ if not clean:
1124
+ print("infores warnings present; pass --strict to fail on them.", file=sys.stderr)
1125
+ return
1126
+ print(passed, file=sys.stderr)
1127
+
1128
+
1003
1129
  @APP.command(name="quick-map")
1004
1130
  def quick_map_command(
1005
1131
  # negative="" suppresses cyclopts' auto-generated --empty-terms help row: the reset