tablassert 19.3.0__tar.gz → 19.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-19.3.0 → tablassert-19.4.0}/PKG-INFO +1 -1
- {tablassert-19.3.0 → tablassert-19.4.0}/pyproject.toml +1 -1
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/cli.py +81 -12
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/errors.py +1 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/lib.py +51 -2
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/runcache.py +2 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/LICENSE +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/README.md +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/Cargo.lock +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/Cargo.toml +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/src/fullmap.rs +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/src/json.rs +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/src/lib.rs +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/src/ndjson.rs +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/src/nlp.rs +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/src/uuid.rs +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/__init__.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/agent.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/biolink.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/cli_entry.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/coerce.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/distill.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/distill_reward.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/enums.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/extras.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/fullmap.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/ingests.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/log.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/models.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/net.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/nlp.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/progress.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/qc.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/rig.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/study.py +0 -0
- {tablassert-19.3.0 → tablassert-19.4.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "19.
|
|
3
|
+
version = "19.4.0"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -912,6 +912,20 @@ def build_kg(
|
|
|
912
912
|
demoted edges from unpinned sections (a row escaped its pin and shipped without the
|
|
913
913
|
class-specific slots ``prune_to_class`` nulled into ``has_supporting_studies``) --
|
|
914
914
|
and fails the build (non-zero exit) when any assertion is violated.
|
|
915
|
+
|
|
916
|
+
Args:
|
|
917
|
+
graph_configuration_file: Path to the Graph YAML that orchestrates one or more table
|
|
918
|
+
configs into a single knowledge-graph build (accepted positionally or via
|
|
919
|
+
``--configuration-file``/``-f``).
|
|
920
|
+
release: Emit release-mode artifacts: a slim, significant-only graph that drops
|
|
921
|
+
``not_significant`` edges before resolution (cached under its own ``.release``
|
|
922
|
+
store, so it never reuses a default build).
|
|
923
|
+
qc: Run quality-control audits and the final study stage over the emitted NDJSON;
|
|
924
|
+
requires the ``[qc]`` extra and exits non-zero when any assertion is violated.
|
|
925
|
+
log: Enable verbose per-section logging (requires the ``[log]`` extra; without it
|
|
926
|
+
Tablassert produces no logs).
|
|
927
|
+
head: Fast output-shape preview: a random sample of up to five rows per section,
|
|
928
|
+
cached under its own ``.head`` store, so it never reuses or replaces a full build.
|
|
915
929
|
"""
|
|
916
930
|
if qc:
|
|
917
931
|
extras.require("qc", required_by="--qc")
|
|
@@ -928,6 +942,12 @@ def validate(
|
|
|
928
942
|
``--schema graph`` validates the Graph model AND every referenced table; ``--schema
|
|
929
943
|
table`` validates section syntax only. The schema is selected explicitly rather than
|
|
930
944
|
sniffed from the YAML, so a config is always checked against the schema you expected.
|
|
945
|
+
|
|
946
|
+
Args:
|
|
947
|
+
configuration_file: Path to the YAML configuration file to validate (accepted
|
|
948
|
+
positionally or via ``--configuration-file``/``-f``).
|
|
949
|
+
schema: Which schema to validate against: ``graph`` checks the Graph model and every
|
|
950
|
+
referenced table; ``table`` checks section syntax of a single table YAML only.
|
|
931
951
|
"""
|
|
932
952
|
if schema == "graph":
|
|
933
953
|
run(2, validate_graph_pipeline, configuration_file)
|
|
@@ -947,6 +967,13 @@ def validate_kgx_command(
|
|
|
947
967
|
``category`` -- the same classes ``NCATSTranslator/translator-ingests`` builds --
|
|
948
968
|
and reports failures grouped by field and error type. Exits non-zero when any
|
|
949
969
|
record fails, so a build can be gated in CI.
|
|
970
|
+
|
|
971
|
+
Args:
|
|
972
|
+
nodes: Path to the built nodes NDJSON file (``<name>_<version>.nodes.ndjson``).
|
|
973
|
+
edges: Path to the built edges NDJSON file (``<name>_<version>.edges.ndjson``).
|
|
974
|
+
limit: Maximum example failures retained per file for the ``e.g.`` report lines;
|
|
975
|
+
every record is still validated regardless of this cap. A missing file is
|
|
976
|
+
reported and fails the run rather than reading as a pass.
|
|
950
977
|
"""
|
|
951
978
|
from tablassert.biolink import validate_kgx
|
|
952
979
|
|
|
@@ -975,14 +1002,16 @@ def validate_kgx_command(
|
|
|
975
1002
|
|
|
976
1003
|
@APP.command(name="quick-map")
|
|
977
1004
|
def quick_map_command(
|
|
978
|
-
terms:
|
|
1005
|
+
# negative="" suppresses cyclopts' auto-generated --empty-terms help row: the reset
|
|
1006
|
+
# negation is unused noise that rendered as a phantom flag on the help page.
|
|
1007
|
+
terms: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False, negative="")],
|
|
979
1008
|
*,
|
|
980
1009
|
fullmap: Annotated[Path, cyclopts.Parameter(name=["--fullmap", "-f"])],
|
|
981
1010
|
taxon: Annotated[int | None, cyclopts.Parameter(name=["--taxon", "-t"])] = 9606,
|
|
982
|
-
prioritize: Annotated[list[str] | None, cyclopts.Parameter(name=["--prioritize", "-p"])] = None,
|
|
983
|
-
avoid: Annotated[list[str] | None, cyclopts.Parameter(name=["--avoid", "-a"])] = None,
|
|
984
|
-
exclude_prefixes: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-prefixes", "-ep"])] = None,
|
|
985
|
-
exclude_regex: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-regex", "-er"])] = None,
|
|
1011
|
+
prioritize: Annotated[list[str] | None, cyclopts.Parameter(name=["--prioritize", "-p"], negative="")] = None,
|
|
1012
|
+
avoid: Annotated[list[str] | None, cyclopts.Parameter(name=["--avoid", "-a"], negative="")] = None,
|
|
1013
|
+
exclude_prefixes: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-prefixes", "-ep"], negative="")] = None,
|
|
1014
|
+
exclude_regex: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-regex", "-er"], negative="")] = None,
|
|
986
1015
|
) -> None:
|
|
987
1016
|
"""Show what fullmap entity resolution does with one or more terms, as a build would see it.
|
|
988
1017
|
|
|
@@ -1096,7 +1125,9 @@ def quick_map_command(
|
|
|
1096
1125
|
|
|
1097
1126
|
@APP.command(name="agent")
|
|
1098
1127
|
def agent(
|
|
1099
|
-
|
|
1128
|
+
# negative="" suppresses cyclopts' auto-generated --empty-pmc-ids help row, which
|
|
1129
|
+
# rendered as a phantom REQUIRED flag on the help page.
|
|
1130
|
+
pmc_ids: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False, negative="")],
|
|
1100
1131
|
*,
|
|
1101
1132
|
graph_configuration_file: Annotated[Path, cyclopts.Parameter(name=["--configuration-file", "-f"])],
|
|
1102
1133
|
model_id: Annotated[str | None, cyclopts.Parameter(name=["--model-id", "-m"])] = None,
|
|
@@ -1112,7 +1143,7 @@ def agent(
|
|
|
1112
1143
|
judge_model: Annotated[str | None, cyclopts.Parameter(name=["--judge-model"])] = None,
|
|
1113
1144
|
judge_threshold: Annotated[float | None, cyclopts.Parameter(name=["--judge-threshold"])] = None,
|
|
1114
1145
|
biolink_threshold: Annotated[float, cyclopts.Parameter(name=["--biolink-threshold"])] = 0.0,
|
|
1115
|
-
local: Annotated[list[str] | None, cyclopts.Parameter(name=["--local", "-l"])] = None,
|
|
1146
|
+
local: Annotated[list[str] | None, cyclopts.Parameter(name=["--local", "-l"], negative="")] = None,
|
|
1116
1147
|
optimize: Annotated[bool, cyclopts.Parameter(name=["--optimize", "-o"], negative="")] = False,
|
|
1117
1148
|
distill: Annotated[bool, cyclopts.Parameter(name=["--distill", "-d", "-dt"], negative="")] = False,
|
|
1118
1149
|
instructions_file: Annotated[Path | None, cyclopts.Parameter(name=["--instructions-file"])] = None,
|
|
@@ -1375,7 +1406,9 @@ def distill_weigh(
|
|
|
1375
1406
|
*,
|
|
1376
1407
|
distill_dir: Annotated[Path, cyclopts.Parameter(name=["--distill-dir", "-dd"])],
|
|
1377
1408
|
out: Annotated[Path, cyclopts.Parameter(name=["--out", "-o"])],
|
|
1378
|
-
|
|
1409
|
+
# Literal mirrors tablassert.distill_reward.POLICIES (kept inline so the CLI stays
|
|
1410
|
+
# import-light); tests/test_cli_help_friendly.py enforces the two stay in lockstep.
|
|
1411
|
+
policy: Annotated[Literal["threshold", "best-of-n", "replication"], cyclopts.Parameter(name=["--policy", "-p"])] = "threshold",
|
|
1379
1412
|
threshold: Annotated[float, cyclopts.Parameter(name=["--threshold", "-t"])] = 0.75,
|
|
1380
1413
|
top_n: Annotated[int, cyclopts.Parameter(name=["--top-n", "-tn"])] = 2,
|
|
1381
1414
|
replication_k: Annotated[int, cyclopts.Parameter(name=["--replication-k", "-rk"])] = 2,
|
|
@@ -1385,7 +1418,39 @@ def distill_weigh(
|
|
|
1385
1418
|
final_call_only: Annotated[bool, cyclopts.Parameter(name="--final-call-only", negative="")] = False,
|
|
1386
1419
|
manifest: Annotated[Path | None, cyclopts.Parameter(name="--manifest")] = None,
|
|
1387
1420
|
) -> None:
|
|
1388
|
-
"""Join distillation records to outcomes and write deterministic training rows.
|
|
1421
|
+
"""Join distillation records to outcomes and write deterministic training rows.
|
|
1422
|
+
|
|
1423
|
+
Data selection for LoRA/QLoRA supervised fine-tuning, not RLHF: one flat training row per
|
|
1424
|
+
input record, weighted by the deterministic reward, plus a reproducibility manifest.
|
|
1425
|
+
Malformed input, invalid knobs, unmatched records, or an ``--out`` inside
|
|
1426
|
+
``--distill-dir`` fail with exit 2 and an actionable message.
|
|
1427
|
+
|
|
1428
|
+
Args:
|
|
1429
|
+
distill_dir: Input directory holding the raw ``records.ndjson``/``outcomes.ndjson``
|
|
1430
|
+
written by ``tablassert agent --distill``.
|
|
1431
|
+
out: Training NDJSON destination. Must be a file OUTSIDE ``--distill-dir``:
|
|
1432
|
+
``distill-export`` loads every ``*.ndjson`` there, so an output inside it would
|
|
1433
|
+
duplicate raw records and mix schemas.
|
|
1434
|
+
policy: Row-selection policy. ``threshold`` keeps rows with ``weight >= --threshold``;
|
|
1435
|
+
``best-of-n`` keeps the top ``--top-n`` rows per ``pmc_id`` group; ``replication``
|
|
1436
|
+
weights repeat occurrences by ``--replication-k``.
|
|
1437
|
+
threshold: Hard weight floor for the ``threshold`` policy, in ``[0, 1]``.
|
|
1438
|
+
top_n: Rows retained per ``pmc_id`` group for the ``best-of-n`` policy (at least 1).
|
|
1439
|
+
replication_k: Replication slope for the ``replication`` policy, bounded to ``[0, 3]``.
|
|
1440
|
+
reward_config: Optional YAML/JSON overriding the default deterministic
|
|
1441
|
+
:class:`tablassert.distill_reward.RewardConfig`.
|
|
1442
|
+
edge_ref: Breadth reference override (finite positive number). Defaults to the
|
|
1443
|
+
corpus median edge count; when the corpus has no comparable build the source is
|
|
1444
|
+
``null`` and breadth contributes ``0.0`` (warned on stderr).
|
|
1445
|
+
purpose: Keep only records with this ``purpose`` (default ``agent``, the supervisor's
|
|
1446
|
+
own calls), or ``all`` to disable filtering. Any other value must match a purpose
|
|
1447
|
+
present in the corpus; an unknown value exits 2 naming the live purposes.
|
|
1448
|
+
final_call_only: Keep only the highest ``call_index`` record per ``run_id`` (the
|
|
1449
|
+
run's final LLM call). Records with no rankable ``run_id``/``call_index`` pair
|
|
1450
|
+
are retained and counted in a warning, never silently dropped.
|
|
1451
|
+
manifest: Reproducibility-manifest destination (default ``<out>.manifest.json``).
|
|
1452
|
+
Must differ from ``--out`` and stay outside ``--distill-dir``.
|
|
1453
|
+
"""
|
|
1389
1454
|
from tablassert.distill_reward import (
|
|
1390
1455
|
OUTCOME_COLUMN_PREFIX,
|
|
1391
1456
|
POLICIES,
|
|
@@ -1740,7 +1805,7 @@ def _prebuilt_fullmap_urls(babel_version: str) -> tuple[str, str]:
|
|
|
1740
1805
|
installed-package metadata, never hardcoded, so a new release looks itself up.
|
|
1741
1806
|
|
|
1742
1807
|
Args:
|
|
1743
|
-
babel_version: BABEL snapshot label (the ``--version`` value), e.g. ``2026jul22``.
|
|
1808
|
+
babel_version: BABEL snapshot label (the ``--babel-version`` value), e.g. ``2026jul22``.
|
|
1744
1809
|
|
|
1745
1810
|
Returns:
|
|
1746
1811
|
``(archive_url, checksum_url)`` for ``fullmap.tar.zst`` and ``sha256sum.txt``.
|
|
@@ -2017,7 +2082,10 @@ def fullmap_matches_allowlist(output: Path, taxon_allowlist: list[int]) -> bool:
|
|
|
2017
2082
|
def build_fullmap(
|
|
2018
2083
|
output: Annotated[Path, cyclopts.Parameter(name=["--output", "-o"])] = Path("./fullmap/data/fullmap.redb"),
|
|
2019
2084
|
cache: Annotated[Path, cyclopts.Parameter(name=["--cache", "-c"])] = Path("./fullmap/downloads"),
|
|
2020
|
-
|
|
2085
|
+
# Named --babel-version because the app-level --version flag shadowed the old --version
|
|
2086
|
+
# long form: `build-fullmap --version <snapshot>` printed the app version and exited 0
|
|
2087
|
+
# without ever running the command. -v keeps the short form.
|
|
2088
|
+
version: Annotated[str, cyclopts.Parameter(name=["--babel-version", "-v"])] = BABEL_VERSION,
|
|
2021
2089
|
force: Annotated[bool, cyclopts.Parameter(name=["--force", "-f"], negative="")] = False,
|
|
2022
2090
|
) -> None:
|
|
2023
2091
|
"""Build an embedded fullmap redb database, or download a prebuilt one from RENCI.
|
|
@@ -2044,7 +2112,8 @@ def build_fullmap(
|
|
|
2044
2112
|
Args:
|
|
2045
2113
|
output: Path to write the redb file (prebuilt extraction or build output).
|
|
2046
2114
|
cache: Directory for downloaded BABEL files when building from scratch.
|
|
2047
|
-
version: BABEL snapshot date to fetch (a RENCI stamp, NOT
|
|
2115
|
+
version: BABEL snapshot date to fetch via ``--babel-version`` (a RENCI stamp, NOT
|
|
2116
|
+
Tablassert's version).
|
|
2048
2117
|
force: Skip the prebuilt download and always rebuild from BABEL outputs.
|
|
2049
2118
|
"""
|
|
2050
2119
|
allowlist_ids: list[int] = load_taxon_allowlist()
|
|
@@ -45,6 +45,7 @@ from tablassert.coerce import (
|
|
|
45
45
|
study_size_target,
|
|
46
46
|
)
|
|
47
47
|
from tablassert.enums import EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
|
|
48
|
+
from tablassert.errors import TablassertError
|
|
48
49
|
from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
|
|
49
50
|
from tablassert.log import cat
|
|
50
51
|
from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
|
|
@@ -1142,6 +1143,51 @@ def trim(lf: pl.LazyFrame, regex: str = r"^column_\d+$") -> pl.LazyFrame:
|
|
|
1142
1143
|
return lf.select(pl.exclude(regex))
|
|
1143
1144
|
|
|
1144
1145
|
|
|
1146
|
+
def qualifier_vocabulary_audit(lf: pl.LazyFrame, col: str, vocabulary: Iterable[str]) -> pl.LazyFrame:
|
|
1147
|
+
"""Fail the build when an enum-ranged qualifier column holds off-vocabulary tokens.
|
|
1148
|
+
|
|
1149
|
+
Enum-ranged qualifiers (``ENUM_RANGED_QUALIFIERS``) bypass entity resolution, so nothing
|
|
1150
|
+
else checks that ``method: column`` cells -- or a ``method: value`` literal mutated by
|
|
1151
|
+
fill/regex/prefix/suffix -- carry tokens from the closed Biolink vocabulary. This op runs
|
|
1152
|
+
after the column's full encoding chain and raises a coded error naming the offending
|
|
1153
|
+
values instead of emitting KGX that fails validation much later, far from the cause.
|
|
1154
|
+
|
|
1155
|
+
Args:
|
|
1156
|
+
lf: Source LazyFrame.
|
|
1157
|
+
col: The enum-ranged qualifier column (named after the qualifier key).
|
|
1158
|
+
vocabulary: The closed set of permitted tokens (any iterable; sorted for the check).
|
|
1159
|
+
|
|
1160
|
+
Returns:
|
|
1161
|
+
The LazyFrame unchanged; the op exists for its check.
|
|
1162
|
+
|
|
1163
|
+
Notes:
|
|
1164
|
+
Nulls and null-like text (``""``, ``"NA"``, ...) are exempt: the Rust
|
|
1165
|
+
``strip_nulls`` drops them at emission, so they never reach the output. Matching is
|
|
1166
|
+
exact -- the vocabularies are closed, and the config-time literal validator
|
|
1167
|
+
(``qualifier-bad-value``) applies the same rule.
|
|
1168
|
+
"""
|
|
1169
|
+
values: pl.Series = lf.select(pl.col(col)).collect().get_column(col)
|
|
1170
|
+
if values.dtype.base_type() in (pl.List, pl.Array):
|
|
1171
|
+
values = values.explode()
|
|
1172
|
+
values = values.drop_nulls()
|
|
1173
|
+
if values.len() == 0:
|
|
1174
|
+
return lf
|
|
1175
|
+
allowed: list[str] = sorted(vocabulary)
|
|
1176
|
+
null_like: pl.Series = values.str.strip_chars().str.to_lowercase().is_in(list(_NULL_LIKE_TEXT))
|
|
1177
|
+
offending: pl.Series = values.filter(~null_like & ~values.is_in(allowed)).unique().sort()
|
|
1178
|
+
if offending.len() == 0:
|
|
1179
|
+
return lf
|
|
1180
|
+
shown: list[str] = offending.head(10).to_list()
|
|
1181
|
+
more: str = "..." if offending.len() > len(shown) else ""
|
|
1182
|
+
raise TablassertError(
|
|
1183
|
+
f"{col} has a closed Biolink vocabulary; {offending.len()} offending value(s) found in the built column: "
|
|
1184
|
+
f"{', '.join(repr(x) for x in shown)}{more}. Permitted values include: "
|
|
1185
|
+
f"{', '.join(repr(x) for x in allowed[:8])}... Null or blank cells are exempt "
|
|
1186
|
+
"(they are stripped from the output); fix the source table or the qualifier's encoding.",
|
|
1187
|
+
code="qualifier-vocabulary-violation",
|
|
1188
|
+
)
|
|
1189
|
+
|
|
1190
|
+
|
|
1145
1191
|
def to_store(lf: pl.LazyFrame, p: Path, config_name: str) -> Path:
|
|
1146
1192
|
"""Collect and write a section LazyFrame to a parquet file.
|
|
1147
1193
|
|
|
@@ -1322,8 +1368,10 @@ class Tcode(Section):
|
|
|
1322
1368
|
return [
|
|
1323
1369
|
[self.node_prep(x, col) for x, col in node_columns],
|
|
1324
1370
|
# Encode only: no pre-resolution snapshot and no NLP normalization, both of
|
|
1325
|
-
# which exist to feed entity resolution these columns never undergo.
|
|
1326
|
-
|
|
1371
|
+
# which exist to feed entity resolution these columns never undergo. Each enum-
|
|
1372
|
+
# ranged column ends with a vocabulary audit: nothing else validates its cell
|
|
1373
|
+
# values against the closed Biolink range before KGX emission.
|
|
1374
|
+
[[*self.encoding(x, x.qualifier), (qualifier_vocabulary_audit, (str(x.qualifier), sorted(x.vocabulary or ())))] for x in literals],
|
|
1327
1375
|
# Raw ``column_<n>`` columns are dead weight once the encodings above have
|
|
1328
1376
|
# copied them into named slots; trim before resolution so the frame that
|
|
1329
1377
|
# resolve_batch materializes and joins stays narrow.
|
|
@@ -1472,6 +1520,7 @@ PHASE_OF: dict[Callable, str] = {
|
|
|
1472
1520
|
resolve_batch: "resolve",
|
|
1473
1521
|
fullmap_audit: "qc",
|
|
1474
1522
|
column: "encode",
|
|
1523
|
+
qualifier_vocabulary_audit: "encode",
|
|
1475
1524
|
edge_category: "edge",
|
|
1476
1525
|
publications: "provenance",
|
|
1477
1526
|
retrieval_sources: "provenance",
|
|
@@ -269,6 +269,8 @@ OP_COST: dict[Callable, int] = {
|
|
|
269
269
|
lib.suffix: 1,
|
|
270
270
|
lib.math_op: 1,
|
|
271
271
|
lib.split_list: 1,
|
|
272
|
+
# Vocabulary audit: one eager distinct-value pass over a single enum column.
|
|
273
|
+
lib.qualifier_vocabulary_audit: 1,
|
|
272
274
|
# Clean: coerce_columns applies five coercion rules (schema resolve + renames + value
|
|
273
275
|
# coercion) in one pass, so it outprices a single-expression clean op.
|
|
274
276
|
lib.coerce_columns: 3,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|