tablassert 19.3.0__tar.gz → 19.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {tablassert-19.3.0 → tablassert-19.5.0}/PKG-INFO +1 -1
  2. {tablassert-19.3.0 → tablassert-19.5.0}/pyproject.toml +1 -1
  3. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/agent.py +15 -7
  4. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/cli.py +81 -12
  5. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/enums.py +1 -0
  6. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/errors.py +1 -0
  7. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/lib.py +73 -2
  8. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/models.py +8 -2
  9. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/runcache.py +3 -0
  10. {tablassert-19.3.0 → tablassert-19.5.0}/LICENSE +0 -0
  11. {tablassert-19.3.0 → tablassert-19.5.0}/README.md +0 -0
  12. {tablassert-19.3.0 → tablassert-19.5.0}/rust/Cargo.lock +0 -0
  13. {tablassert-19.3.0 → tablassert-19.5.0}/rust/Cargo.toml +0 -0
  14. {tablassert-19.3.0 → tablassert-19.5.0}/rust/examples/count_tables.rs +0 -0
  15. {tablassert-19.3.0 → tablassert-19.5.0}/rust/src/fullmap.rs +0 -0
  16. {tablassert-19.3.0 → tablassert-19.5.0}/rust/src/json.rs +0 -0
  17. {tablassert-19.3.0 → tablassert-19.5.0}/rust/src/lib.rs +0 -0
  18. {tablassert-19.3.0 → tablassert-19.5.0}/rust/src/ndjson.rs +0 -0
  19. {tablassert-19.3.0 → tablassert-19.5.0}/rust/src/nlp.rs +0 -0
  20. {tablassert-19.3.0 → tablassert-19.5.0}/rust/src/uuid.rs +0 -0
  21. {tablassert-19.3.0 → tablassert-19.5.0}/rust/tests/build_golden.rs +0 -0
  22. {tablassert-19.3.0 → tablassert-19.5.0}/rust/tests/common/mod.rs +0 -0
  23. {tablassert-19.3.0 → tablassert-19.5.0}/rust/tests/extract_prebuilt.rs +0 -0
  24. {tablassert-19.3.0 → tablassert-19.5.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
  25. {tablassert-19.3.0 → tablassert-19.5.0}/rust/tests/nlp_golden.rs +0 -0
  26. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/__init__.py +0 -0
  27. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/_lazy.py +0 -0
  28. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/biolink.py +0 -0
  29. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/cli_entry.py +0 -0
  30. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/coerce.py +0 -0
  31. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
  32. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/distill.py +0 -0
  33. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/distill_reward.py +0 -0
  34. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/extras.py +0 -0
  35. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/fullmap.py +0 -0
  36. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/graph_target.py +0 -0
  37. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/ingests.py +0 -0
  38. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/log.py +0 -0
  39. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/net.py +0 -0
  40. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/nlp.py +0 -0
  41. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/progress.py +0 -0
  42. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/qc.py +0 -0
  43. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/rig.py +0 -0
  44. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/rs.pyi +0 -0
  45. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/study.py +0 -0
  46. {tablassert-19.3.0 → tablassert-19.5.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 19.3.0
3
+ Version: 19.5.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "19.3.0"
3
+ version = "19.5.0"
4
4
  description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -115,7 +115,7 @@ def is_lazy() -> bool:
115
115
  PMC_BUCKET: str = "pmc-oa-opendata"
116
116
  PMC_HTTPS_BASE: str = "https://pmc-oa-opendata.s3.amazonaws.com"
117
117
  PMC_S3API_BASE: str = "https://pmc-oa-opendata.s3.us-east-1.amazonaws.com"
118
- TABLE_EXTENSIONS: frozenset[str] = frozenset({".xlsx", ".xls", ".csv", ".tsv"})
118
+ TABLE_EXTENSIONS: frozenset[str] = frozenset({".xlsx", ".xls", ".csv", ".tsv", ".parquet"})
119
119
  #: Default minimum number of non-empty data rows for agent table candidates.
120
120
  #:
121
121
  #: Counts use the same header inference as the production readers, so the header is not included.
@@ -662,8 +662,9 @@ def _load_table(path: Path, sheet: str | None = None) -> pl.DataFrame:
662
662
  """Dispatch a local table file to the right polars reader by suffix.
663
663
 
664
664
  ``.csv`` -> ``read_csv``; ``.tsv``/``.txt`` -> ``read_csv(separator="\\t")``;
665
- ``.xlsx``/``.xls`` -> :func:`_read_excel` (``sheet`` selects a worksheet by name;
666
- ignored for csv/tsv). Any polars parse failure becomes a clear ``ValueError``; an
665
+ ``.parquet`` -> ``read_parquet`` (typed columns read as-is); ``.xlsx``/``.xls`` ->
666
+ :func:`_read_excel` (``sheet`` selects a worksheet by name; ignored for
667
+ csv/tsv/parquet). Any polars parse failure becomes a clear ``ValueError``; an
667
668
  unknown suffix is a ``ValueError`` too (never a silent mis-read).
668
669
  """
669
670
  suffix: str = path.suffix.lower()
@@ -674,6 +675,8 @@ def _load_table(path: Path, sheet: str | None = None) -> pl.DataFrame:
674
675
  return pl.read_csv(path)
675
676
  if suffix in {".tsv", ".txt"}:
676
677
  return pl.read_csv(path, separator="\t")
678
+ if suffix == ".parquet":
679
+ return pl.read_parquet(path)
677
680
  except Exception as e:
678
681
  raise ValueError(f"Could not read table {path}: {e}") from e
679
682
  raise ValueError(f"Could not read table {path}: unsupported extension {suffix!r}")
@@ -685,8 +688,9 @@ def _effective_rows_cached(path: str, mtime_ns: int, size: int, sheet: str | Non
685
688
 
686
689
  ``path``, ``mtime_ns``, and ``size`` form the cache key so a re-fetched or rewritten file
687
690
  cannot reuse a stale count. Delimited files are counted by the parser rather than physical
688
- lines, which handles quoted embedded newlines correctly. Excel uses the same reader as
689
- :func:`_load_table` and removes rows that are null in every column (formatted blank rows).
691
+ lines, which handles quoted embedded newlines correctly. Excel and parquet use the same
692
+ reader as :func:`_load_table` and remove rows that are null in every column (formatted
693
+ blank rows).
690
694
  """
691
695
  table_path: Path = Path(path)
692
696
  suffix: str = table_path.suffix.lower()
@@ -694,6 +698,10 @@ def _effective_rows_cached(path: str, mtime_ns: int, size: int, sheet: str | Non
694
698
  frame: pl.DataFrame = _read_excel(table_path, sheet)
695
699
  return frame.filter(~pl.all_horizontal(pl.all().is_null())).height
696
700
 
701
+ if suffix == ".parquet":
702
+ pq: pl.DataFrame = pl.read_parquet(table_path)
703
+ return pq.filter(~pl.all_horizontal(pl.all().is_null())).height
704
+
697
705
  if suffix == ".csv":
698
706
  separator: str = ","
699
707
  elif suffix in {".tsv", ".txt"}:
@@ -1233,7 +1241,7 @@ def normalize_agent_table_config(config_yaml: str, *, base_dirs: Sequence[Path]
1233
1241
  # hand-maintained guess table), so a model default change automatically changes
1234
1242
  # what counts as removable. Semantic guards: the ``provenance`` subtree is never
1235
1243
  # touched (legal attribution), ``kind`` is never dropped (it discriminates the
1236
- # ``Excel | Text`` source union), a non-default null such as ``taxon: null``
1244
+ # ``Excel | Text | Parquet`` source union), a non-default null such as ``taxon: null``
1237
1245
  # (default 9606) is preserved, and in ``{template, sections}`` configs a section
1238
1246
  # entry equal to a model default is only removed when the template cannot change
1239
1247
  # the merged result (fastmerge lets section scalars override template values, so
@@ -1245,7 +1253,7 @@ _COMPACT_ABSENT: object = object()
1245
1253
 
1246
1254
  #: Keys compaction never removes and never recurses into: ``provenance`` values are
1247
1255
  #: the edge's legal attribution (untouched even when they equal a model default),
1248
- #: and ``kind`` discriminates the ``Excel | Text`` source union — dropping it could
1256
+ #: and ``kind`` discriminates the ``Excel | Text | Parquet`` source union — dropping it could
1249
1257
  #: flip which model a re-parsed source validates as.
1250
1258
  _COMPACT_UNTOUCHED_KEYS: frozenset[str] = frozenset({"provenance", "kind"})
1251
1259
 
@@ -912,6 +912,20 @@ def build_kg(
912
912
  demoted edges from unpinned sections (a row escaped its pin and shipped without the
913
913
  class-specific slots ``prune_to_class`` nulled into ``has_supporting_studies``) --
914
914
  and fails the build (non-zero exit) when any assertion is violated.
915
+
916
+ Args:
917
+ graph_configuration_file: Path to the Graph YAML that orchestrates one or more table
918
+ configs into a single knowledge-graph build (accepted positionally or via
919
+ ``--configuration-file``/``-f``).
920
+ release: Emit release-mode artifacts: a slim, significant-only graph that drops
921
+ ``not_significant`` edges before resolution (cached under its own ``.release``
922
+ store, so it never reuses a default build).
923
+ qc: Run quality-control audits and the final study stage over the emitted NDJSON;
924
+ requires the ``[qc]`` extra and exits non-zero when any assertion is violated.
925
+ log: Enable verbose per-section logging (requires the ``[log]`` extra; without it
926
+ Tablassert produces no logs).
927
+ head: Fast output-shape preview: a random sample of up to five rows per section,
928
+ cached under its own ``.head`` store, so it never reuses or replaces a full build.
915
929
  """
916
930
  if qc:
917
931
  extras.require("qc", required_by="--qc")
@@ -928,6 +942,12 @@ def validate(
928
942
  ``--schema graph`` validates the Graph model AND every referenced table; ``--schema
929
943
  table`` validates section syntax only. The schema is selected explicitly rather than
930
944
  sniffed from the YAML, so a config is always checked against the schema you expected.
945
+
946
+ Args:
947
+ configuration_file: Path to the YAML configuration file to validate (accepted
948
+ positionally or via ``--configuration-file``/``-f``).
949
+ schema: Which schema to validate against: ``graph`` checks the Graph model and every
950
+ referenced table; ``table`` checks section syntax of a single table YAML only.
931
951
  """
932
952
  if schema == "graph":
933
953
  run(2, validate_graph_pipeline, configuration_file)
@@ -947,6 +967,13 @@ def validate_kgx_command(
947
967
  ``category`` -- the same classes ``NCATSTranslator/translator-ingests`` builds --
948
968
  and reports failures grouped by field and error type. Exits non-zero when any
949
969
  record fails, so a build can be gated in CI.
970
+
971
+ Args:
972
+ nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
973
+ edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
974
+ limit: Maximum example failures retained per file for the ``e.g.`` report lines;
975
+ every record is still validated regardless of this cap. A missing file is
976
+ reported and fails the run rather than reading as a pass.
950
977
  """
951
978
  from tablassert.biolink import validate_kgx
952
979
 
@@ -975,14 +1002,16 @@ def validate_kgx_command(
975
1002
 
976
1003
  @APP.command(name="quick-map")
977
1004
  def quick_map_command(
978
- terms: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False)],
1005
+ # negative="" suppresses cyclopts' auto-generated --empty-terms help row: the reset
1006
+ # negation is unused noise that rendered as a phantom flag on the help page.
1007
+ terms: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False, negative="")],
979
1008
  *,
980
1009
  fullmap: Annotated[Path, cyclopts.Parameter(name=["--fullmap", "-f"])],
981
1010
  taxon: Annotated[int | None, cyclopts.Parameter(name=["--taxon", "-t"])] = 9606,
982
- prioritize: Annotated[list[str] | None, cyclopts.Parameter(name=["--prioritize", "-p"])] = None,
983
- avoid: Annotated[list[str] | None, cyclopts.Parameter(name=["--avoid", "-a"])] = None,
984
- exclude_prefixes: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-prefixes", "-ep"])] = None,
985
- exclude_regex: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-regex", "-er"])] = None,
1011
+ prioritize: Annotated[list[str] | None, cyclopts.Parameter(name=["--prioritize", "-p"], negative="")] = None,
1012
+ avoid: Annotated[list[str] | None, cyclopts.Parameter(name=["--avoid", "-a"], negative="")] = None,
1013
+ exclude_prefixes: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-prefixes", "-ep"], negative="")] = None,
1014
+ exclude_regex: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-regex", "-er"], negative="")] = None,
986
1015
  ) -> None:
987
1016
  """Show what fullmap entity resolution does with one or more terms, as a build would see it.
988
1017
 
@@ -1096,7 +1125,9 @@ def quick_map_command(
1096
1125
 
1097
1126
  @APP.command(name="agent")
1098
1127
  def agent(
1099
- pmc_ids: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False)],
1128
+ # negative="" suppresses cyclopts' auto-generated --empty-pmc-ids help row, which
1129
+ # rendered as a phantom REQUIRED flag on the help page.
1130
+ pmc_ids: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False, negative="")],
1100
1131
  *,
1101
1132
  graph_configuration_file: Annotated[Path, cyclopts.Parameter(name=["--configuration-file", "-f"])],
1102
1133
  model_id: Annotated[str | None, cyclopts.Parameter(name=["--model-id", "-m"])] = None,
@@ -1112,7 +1143,7 @@ def agent(
1112
1143
  judge_model: Annotated[str | None, cyclopts.Parameter(name=["--judge-model"])] = None,
1113
1144
  judge_threshold: Annotated[float | None, cyclopts.Parameter(name=["--judge-threshold"])] = None,
1114
1145
  biolink_threshold: Annotated[float, cyclopts.Parameter(name=["--biolink-threshold"])] = 0.0,
1115
- local: Annotated[list[str] | None, cyclopts.Parameter(name=["--local", "-l"])] = None,
1146
+ local: Annotated[list[str] | None, cyclopts.Parameter(name=["--local", "-l"], negative="")] = None,
1116
1147
  optimize: Annotated[bool, cyclopts.Parameter(name=["--optimize", "-o"], negative="")] = False,
1117
1148
  distill: Annotated[bool, cyclopts.Parameter(name=["--distill", "-d", "-dt"], negative="")] = False,
1118
1149
  instructions_file: Annotated[Path | None, cyclopts.Parameter(name=["--instructions-file"])] = None,
@@ -1375,7 +1406,9 @@ def distill_weigh(
1375
1406
  *,
1376
1407
  distill_dir: Annotated[Path, cyclopts.Parameter(name=["--distill-dir", "-dd"])],
1377
1408
  out: Annotated[Path, cyclopts.Parameter(name=["--out", "-o"])],
1378
- policy: Annotated[str, cyclopts.Parameter(name=["--policy", "-p"])] = "threshold",
1409
+ # Literal mirrors tablassert.distill_reward.POLICIES (kept inline so the CLI stays
1410
+ # import-light); tests/test_cli_help_friendly.py enforces the two stay in lockstep.
1411
+ policy: Annotated[Literal["threshold", "best-of-n", "replication"], cyclopts.Parameter(name=["--policy", "-p"])] = "threshold",
1379
1412
  threshold: Annotated[float, cyclopts.Parameter(name=["--threshold", "-t"])] = 0.75,
1380
1413
  top_n: Annotated[int, cyclopts.Parameter(name=["--top-n", "-tn"])] = 2,
1381
1414
  replication_k: Annotated[int, cyclopts.Parameter(name=["--replication-k", "-rk"])] = 2,
@@ -1385,7 +1418,39 @@ def distill_weigh(
1385
1418
  final_call_only: Annotated[bool, cyclopts.Parameter(name="--final-call-only", negative="")] = False,
1386
1419
  manifest: Annotated[Path | None, cyclopts.Parameter(name="--manifest")] = None,
1387
1420
  ) -> None:
1388
- """Join distillation records to outcomes and write deterministic training rows."""
1421
+ """Join distillation records to outcomes and write deterministic training rows.
1422
+
1423
+ Data selection for LoRA/QLoRA supervised fine-tuning, not RLHF: one flat training row per
1424
+ input record, weighted by the deterministic reward, plus a reproducibility manifest.
1425
+ Malformed input, invalid knobs, unmatched records, or an ``--out`` inside
1426
+ ``--distill-dir`` fail with exit 2 and an actionable message.
1427
+
1428
+ Args:
1429
+ distill_dir: Input directory holding the raw ``records.ndjson``/``outcomes.ndjson``
1430
+ written by ``tablassert agent --distill``.
1431
+ out: Training NDJSON destination. Must be a file OUTSIDE ``--distill-dir``:
1432
+ ``distill-export`` loads every ``*.ndjson`` there, so an output inside it would
1433
+ duplicate raw records and mix schemas.
1434
+ policy: Row-selection policy. ``threshold`` keeps rows with ``weight >= --threshold``;
1435
+ ``best-of-n`` keeps the top ``--top-n`` rows per ``pmc_id`` group; ``replication``
1436
+ weights repeat occurrences by ``--replication-k``.
1437
+ threshold: Hard weight floor for the ``threshold`` policy, in ``[0, 1]``.
1438
+ top_n: Rows retained per ``pmc_id`` group for the ``best-of-n`` policy (at least 1).
1439
+ replication_k: Replication slope for the ``replication`` policy, bounded to ``[0, 3]``.
1440
+ reward_config: Optional YAML/JSON overriding the default deterministic
1441
+ :class:`tablassert.distill_reward.RewardConfig`.
1442
+ edge_ref: Breadth reference override (finite positive number). Defaults to the
1443
+ corpus median edge count; when the corpus has no comparable build the source is
1444
+ ``null`` and breadth contributes ``0.0`` (warned on stderr).
1445
+ purpose: Keep only records with this ``purpose`` (default ``agent``, the supervisor's
1446
+ own calls), or ``all`` to disable filtering. Any other value must match a purpose
1447
+ present in the corpus; an unknown value exits 2 naming the live purposes.
1448
+ final_call_only: Keep only the highest ``call_index`` record per ``run_id`` (the
1449
+ run's final LLM call). Records with no rankable ``run_id``/``call_index`` pair
1450
+ are retained and counted in a warning, never silently dropped.
1451
+ manifest: Reproducibility-manifest destination (default ``<out>.manifest.json``).
1452
+ Must differ from ``--out`` and stay outside ``--distill-dir``.
1453
+ """
1389
1454
  from tablassert.distill_reward import (
1390
1455
  OUTCOME_COLUMN_PREFIX,
1391
1456
  POLICIES,
@@ -1740,7 +1805,7 @@ def _prebuilt_fullmap_urls(babel_version: str) -> tuple[str, str]:
1740
1805
  installed-package metadata, never hardcoded, so a new release looks itself up.
1741
1806
 
1742
1807
  Args:
1743
- babel_version: BABEL snapshot label (the ``--version`` value), e.g. ``2026jul22``.
1808
+ babel_version: BABEL snapshot label (the ``--babel-version`` value), e.g. ``2026jul22``.
1744
1809
 
1745
1810
  Returns:
1746
1811
  ``(archive_url, checksum_url)`` for ``fullmap.tar.zst`` and ``sha256sum.txt``.
@@ -2017,7 +2082,10 @@ def fullmap_matches_allowlist(output: Path, taxon_allowlist: list[int]) -> bool:
2017
2082
  def build_fullmap(
2018
2083
  output: Annotated[Path, cyclopts.Parameter(name=["--output", "-o"])] = Path("./fullmap/data/fullmap.redb"),
2019
2084
  cache: Annotated[Path, cyclopts.Parameter(name=["--cache", "-c"])] = Path("./fullmap/downloads"),
2020
- version: Annotated[str, cyclopts.Parameter(name=["--version", "-v"])] = BABEL_VERSION,
2085
+ # Named --babel-version because the app-level --version flag shadowed the old --version
2086
+ # long form: `build-fullmap --version <snapshot>` printed the app version and exited 0
2087
+ # without ever running the command. -v keeps the short form.
2088
+ version: Annotated[str, cyclopts.Parameter(name=["--babel-version", "-v"])] = BABEL_VERSION,
2021
2089
  force: Annotated[bool, cyclopts.Parameter(name=["--force", "-f"], negative="")] = False,
2022
2090
  ) -> None:
2023
2091
  """Build an embedded fullmap redb database, or download a prebuilt one from RENCI.
@@ -2044,7 +2112,8 @@ def build_fullmap(
2044
2112
  Args:
2045
2113
  output: Path to write the redb file (prebuilt extraction or build output).
2046
2114
  cache: Directory for downloaded BABEL files when building from scratch.
2047
- version: BABEL snapshot date to fetch (a RENCI stamp, NOT Tablassert's version).
2115
+ version: BABEL snapshot date to fetch via ``--babel-version`` (a RENCI stamp, NOT
2116
+ Tablassert's version).
2048
2117
  force: Skip the prebuilt download and always rebuild from BABEL outputs.
2049
2118
  """
2050
2119
  allowlist_ids: list[int] = load_taxon_allowlist()
@@ -52,6 +52,7 @@ class Functions(str, Enum):
52
52
  class Files(str, Enum):
53
53
  TEXT = "text"
54
54
  EXCEL = "excel"
55
+ PARQUET = "parquet"
55
56
 
56
57
 
57
58
  class EncodingMethods(str, Enum):
@@ -70,6 +70,7 @@ TablassertErrorCodes = Literal[
70
70
  "qualifier-unsatisfiable",
71
71
  "qualifier-nullable-literal",
72
72
  "qualifier-duplicated",
73
+ "qualifier-vocabulary-violation",
73
74
  "rig-bad-infores",
74
75
  "rig-bad-artifact-url",
75
76
  "rig-bad-access-location",
@@ -45,6 +45,7 @@ from tablassert.coerce import (
45
45
  study_size_target,
46
46
  )
47
47
  from tablassert.enums import EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
48
+ from tablassert.errors import TablassertError
48
49
  from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
49
50
  from tablassert.log import cat
50
51
  from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
@@ -952,6 +953,26 @@ def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
952
953
  return df.lazy()
953
954
 
954
955
 
956
+ def parquet(p: Path) -> pl.LazyFrame:
957
+ """Read a parquet source as a LazyFrame with positional ``column_<n>`` names.
958
+
959
+ Args:
960
+ p: Path to the parquet file.
961
+
962
+ Returns:
963
+ LazyFrame whose columns are renamed to ``column_1..`` in schema order.
964
+
965
+ Notes:
966
+ Parquet carries its own column names, but the config contract addresses columns
967
+ positionally (Excel-style ``A-ZZ`` encoding letters resolve to ``column_<n>``), so
968
+ the reader renames whatever names the file carries to ``column_1..`` -- the same
969
+ convention the headerless ``csv``/``excel`` readers produce.
970
+ """
971
+ lf: pl.LazyFrame = pl.scan_parquet(source=p)
972
+ names: list[str] = list(lf.collect_schema().names())
973
+ return lf.rename(dict(zip(names, (f"column_{i + 1}" for i in range(len(names))), strict=True)))
974
+
975
+
955
976
  def crop(lf: pl.LazyFrame, row_slice: list[NonNegativeInt | Literal[Tokens.AUTO]]) -> pl.LazyFrame:
956
977
  """Take a contiguous slice from a LazyFrame.
957
978
 
@@ -1142,6 +1163,51 @@ def trim(lf: pl.LazyFrame, regex: str = r"^column_\d+$") -> pl.LazyFrame:
1142
1163
  return lf.select(pl.exclude(regex))
1143
1164
 
1144
1165
 
1166
+ def qualifier_vocabulary_audit(lf: pl.LazyFrame, col: str, vocabulary: Iterable[str]) -> pl.LazyFrame:
1167
+ """Fail the build when an enum-ranged qualifier column holds off-vocabulary tokens.
1168
+
1169
+ Enum-ranged qualifiers (``ENUM_RANGED_QUALIFIERS``) bypass entity resolution, so nothing
1170
+ else checks that ``method: column`` cells -- or a ``method: value`` literal mutated by
1171
+ fill/regex/prefix/suffix -- carry tokens from the closed Biolink vocabulary. This op runs
1172
+ after the column's full encoding chain and raises a coded error naming the offending
1173
+ values instead of emitting KGX that fails validation much later, far from the cause.
1174
+
1175
+ Args:
1176
+ lf: Source LazyFrame.
1177
+ col: The enum-ranged qualifier column (named after the qualifier key).
1178
+ vocabulary: The closed set of permitted tokens (any iterable; sorted for the check).
1179
+
1180
+ Returns:
1181
+ The LazyFrame unchanged; the op exists for its check.
1182
+
1183
+ Notes:
1184
+ Nulls and null-like text (``""``, ``"NA"``, ...) are exempt: the Rust
1185
+ ``strip_nulls`` drops them at emission, so they never reach the output. Matching is
1186
+ exact -- the vocabularies are closed, and the config-time literal validator
1187
+ (``qualifier-bad-value``) applies the same rule.
1188
+ """
1189
+ values: pl.Series = lf.select(pl.col(col)).collect().get_column(col)
1190
+ if values.dtype.base_type() in (pl.List, pl.Array):
1191
+ values = values.explode()
1192
+ values = values.drop_nulls()
1193
+ if values.len() == 0:
1194
+ return lf
1195
+ allowed: list[str] = sorted(vocabulary)
1196
+ null_like: pl.Series = values.str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT))
1197
+ offending: pl.Series = values.filter(~null_like & ~values.is_in(allowed)).unique().sort()
1198
+ if offending.len() == 0:
1199
+ return lf
1200
+ shown: list[str] = offending.head(10).to_list()
1201
+ more: str = "..." if offending.len() > len(shown) else ""
1202
+ raise TablassertError(
1203
+ f"{col} has a closed Biolink vocabulary; {offending.len()} offending value(s) found in the built column: "
1204
+ f"{', '.join(repr(x) for x in shown)}{more}. Permitted values include: "
1205
+ f"{', '.join(repr(x) for x in allowed[:8])}... Null or blank cells are exempt "
1206
+ "(they are stripped from the output); fix the source table or the qualifier's encoding.",
1207
+ code="qualifier-vocabulary-violation",
1208
+ )
1209
+
1210
+
1145
1211
  def to_store(lf: pl.LazyFrame, p: Path, config_name: str) -> Path:
1146
1212
  """Collect and write a section LazyFrame to a parquet file.
1147
1213
 
@@ -1250,6 +1316,7 @@ class Tcode(Section):
1250
1316
  return [
1251
1317
  (csv, (self.source.local, self.source.delimiter)) if self.source.kind == Files.TEXT else None, # pyright: ignore
1252
1318
  (excel, (self.source.local, self.source.sheet)) if self.source.kind == Files.EXCEL else None, # pyright: ignore
1319
+ (parquet, (self.source.local,)) if self.source.kind == Files.PARQUET else None, # pyright: ignore
1253
1320
  (idx, ()),
1254
1321
  (crop, (self.source.row_slice,)) if self.source.row_slice else None,
1255
1322
  (pick, (self.source.rows,)) if self.source.rows else None,
@@ -1322,8 +1389,10 @@ class Tcode(Section):
1322
1389
  return [
1323
1390
  [self.node_prep(x, col) for x, col in node_columns],
1324
1391
  # Encode only: no pre-resolution snapshot and no NLP normalization, both of
1325
- # which exist to feed entity resolution these columns never undergo.
1326
- [self.encoding(x, x.qualifier) for x in literals],
1392
+ # which exist to feed entity resolution these columns never undergo. Each enum-
1393
+ # ranged column ends with a vocabulary audit: nothing else validates its cell
1394
+ # values against the closed Biolink range before KGX emission.
1395
+ [[*self.encoding(x, x.qualifier), (qualifier_vocabulary_audit, (str(x.qualifier), sorted(x.vocabulary or ())))] for x in literals],
1327
1396
  # Raw ``column_<n>`` columns are dead weight once the encodings above have
1328
1397
  # copied them into named slots; trim before resolution so the frame that
1329
1398
  # resolve_batch materializes and joins stays narrow.
@@ -1459,6 +1528,7 @@ PHASE_OF: dict[Callable, str] = {
1459
1528
  # Maps each Tcode op callable to a short lowercase phase label for progress UX.
1460
1529
  csv: "load",
1461
1530
  excel: "load",
1531
+ parquet: "load",
1462
1532
  idx: "load",
1463
1533
  crop: "filter",
1464
1534
  pick: "filter",
@@ -1472,6 +1542,7 @@ PHASE_OF: dict[Callable, str] = {
1472
1542
  resolve_batch: "resolve",
1473
1543
  fullmap_audit: "qc",
1474
1544
  column: "encode",
1545
+ qualifier_vocabulary_audit: "encode",
1475
1546
  edge_category: "edge",
1476
1547
  publications: "provenance",
1477
1548
  retrieval_sources: "provenance",
@@ -67,7 +67,7 @@ def _shown(name: str, target: str) -> str:
67
67
  return f"`{name}` (coerced to `{target}`)" if target != name else f"`{name}`"
68
68
 
69
69
 
70
- def _section_source_label(source: Excel | Text) -> str:
70
+ def _section_source_label(source: Excel | Text | Parquet) -> str:
71
71
  """Render a section's source for a diagnostic message: the local path, plus the sheet for Excel.
72
72
 
73
73
  Args:
@@ -174,6 +174,12 @@ class Text(BaseSource):
174
174
  delimiter: str | None = Field(",", description="Field delimiter for headerless text/CSV scanning.", examples=[",", "\t", "|"])
175
175
 
176
176
 
177
+ # Parquet is self-describing (typed columns, real headers, no sheets or delimiters), so unlike
178
+ # `Text`/`Excel` it adds no format-specific fields beyond `BaseSource`.
179
+ class Parquet(BaseSource):
180
+ kind: Literal[Files.PARQUET] = Field(Files.PARQUET, description="Source kind; must be 'parquet'.")
181
+
182
+
177
183
  class Regex(TablaBase):
178
184
  pattern: int | float | str = Field(..., description="Regex pattern passed to string replacement.", examples=["\\s+", "\\.$"])
179
185
 
@@ -791,7 +797,7 @@ class Annotation(Encoding):
791
797
  class Section(TablaBase):
792
798
  """Pydantic section model and coercion target for a single table configuration."""
793
799
 
794
- source: Excel | Text = Field(..., description="Input source definition for reading tabular rows.")
800
+ source: Excel | Text | Parquet = Field(..., description="Input source definition for reading tabular rows.")
795
801
  statement: Statement = Field(..., description="Subject-object statement mapping for this section.")
796
802
  provenance: Provenance = Field(..., description="Provenance metadata applied to all produced edges.")
797
803
  annotations: list[Annotation] | None = Field(None, description="Optional extra encoded columns added to each row.")
@@ -252,6 +252,7 @@ OP_COST: dict[Callable, int] = {
252
252
  # to resolution.
253
253
  lib.csv: 2,
254
254
  lib.excel: 2,
255
+ lib.parquet: 2,
255
256
  # Filter: single lazy row predicates/slices.
256
257
  lib.idx: 1,
257
258
  lib.crop: 1,
@@ -269,6 +270,8 @@ OP_COST: dict[Callable, int] = {
269
270
  lib.suffix: 1,
270
271
  lib.math_op: 1,
271
272
  lib.split_list: 1,
273
+ # Vocabulary audit: one eager distinct-value pass over a single enum column.
274
+ lib.qualifier_vocabulary_audit: 1,
272
275
  # Clean: coerce_columns applies five coercion rules (schema resolve + renames + value
273
276
  # coercion) in one pass, so it outprices a single-expression clean op.
274
277
  lib.coerce_columns: 3,
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes