tablassert 22.0.1__tar.gz → 22.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-22.0.1 → tablassert-22.2.0}/PKG-INFO +1 -1
- {tablassert-22.0.1 → tablassert-22.2.0}/pyproject.toml +1 -1
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/biolink.py +8 -4
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/enums.py +4 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/errors.py +14 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/fullmap.py +113 -6
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/lib.py +201 -11
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/models.py +179 -4
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/runcache.py +3 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/study.py +37 -6
- {tablassert-22.0.1 → tablassert-22.2.0}/LICENSE +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/README.md +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/Cargo.lock +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/Cargo.toml +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/fullmap.rs +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/json.rs +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/lib.rs +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/ndjson.rs +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/nlp.rs +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/uuid.rs +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/__init__.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/cli.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/cli_entry.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/coerce.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/data/infores_catalog.yaml +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/extras.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/infores_registry.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/ingests.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/log.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/net.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/nlp.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/progress.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/qc.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/rig.py +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "22.0
|
|
3
|
+
version = "22.2.0"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -582,13 +582,17 @@ description dump.
|
|
|
582
582
|
"""
|
|
583
583
|
|
|
584
584
|
|
|
585
|
-
DISABLED_EDGE_FIELDS: frozenset[str] = frozenset(
|
|
586
|
-
"""Edge fields Tablassert intentionally never emits or accepts.
|
|
585
|
+
DISABLED_EDGE_FIELDS: frozenset[str] = frozenset()
|
|
586
|
+
"""Edge fields Tablassert intentionally never emits or accepts, whatever the model says.
|
|
587
587
|
|
|
588
588
|
This policy is separate from :data:`UNSATISFIABLE_EDGE_FIELDS`: the latter tracks
|
|
589
589
|
Biolink Model attachment and may change with a dependency release, while this set is
|
|
590
|
-
a stable Tablassert product decision.
|
|
591
|
-
|
|
590
|
+
a stable Tablassert product decision. It is currently empty: the one entry it ever
|
|
591
|
+
held, ``species_context_qualifier``, was re-enabled as a declarable, CURIE-ranged
|
|
592
|
+
qualifier (Biolink Model 4.4.6 declares the slot on the root ``association`` class,
|
|
593
|
+
range ``organism taxon``) after its old node-taxonomy auto-derivation was removed; no
|
|
594
|
+
derivation returns, so the field only appears when a config declares it. The empty
|
|
595
|
+
set keeps the policy mechanism alive for future product decisions.
|
|
592
596
|
"""
|
|
593
597
|
|
|
594
598
|
|
|
@@ -42,6 +42,10 @@ class Comparisons(str, Enum):
|
|
|
42
42
|
LE = "le"
|
|
43
43
|
EQ = "eq"
|
|
44
44
|
NE = "ne"
|
|
45
|
+
# Membership comparisons: the comparator is a list of identifiers, matched against the
|
|
46
|
+
# raw (pre-resolution) cell value. JOIN keeps matching rows, ANTIJOIN drops them.
|
|
47
|
+
JOIN = "join"
|
|
48
|
+
ANTIJOIN = "antijoin"
|
|
45
49
|
|
|
46
50
|
|
|
47
51
|
class Functions(str, Enum):
|
|
@@ -47,9 +47,23 @@ TablassertErrorCodes = Literal[
|
|
|
47
47
|
"babel-download-failed",
|
|
48
48
|
"source-file-unreadable",
|
|
49
49
|
"resolve-bad-specs",
|
|
50
|
+
"taxon-from-conflict",
|
|
51
|
+
"taxon-from-unknown-column",
|
|
52
|
+
"taxon-from-self",
|
|
53
|
+
"taxon-from-cycle",
|
|
54
|
+
"taxon-from-unresolved-column",
|
|
55
|
+
"taxon-from-not-ncbitaxon",
|
|
50
56
|
"config-rows-and-row-slice-conflict",
|
|
51
57
|
"comparison-bad-comparator-type",
|
|
52
58
|
"comparison-nonnumeric-comparator",
|
|
59
|
+
# Membership (join/antijoin) family: the first two are config-validation failures on the
|
|
60
|
+
# `comparator` field, the last two are op-level failures on the column being tested.
|
|
61
|
+
"comparison-membership-comparator-required",
|
|
62
|
+
"comparison-membership-empty",
|
|
63
|
+
"comparison-membership-member-not-a-string",
|
|
64
|
+
"reindex-membership-column-missing",
|
|
65
|
+
"reindex-membership-column-dtype-unsupported",
|
|
66
|
+
"comparison-regex-requires-eq-ne",
|
|
53
67
|
"encoding-bad-excel-column",
|
|
54
68
|
"regex-bad-pattern",
|
|
55
69
|
"regex-bad-replacement",
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import re
|
|
3
4
|
import time
|
|
4
5
|
from collections import OrderedDict
|
|
5
6
|
from collections.abc import Callable
|
|
@@ -575,7 +576,9 @@ def join_matches(lf: pl.LazyFrame, col: str, matches: pl.DataFrame, tag: str = "
|
|
|
575
576
|
return _join_matches_eager(lf.collect(), col, matches, tag, drop_unresolved).lazy()
|
|
576
577
|
|
|
577
578
|
|
|
578
|
-
def _join_matches_eager(
|
|
579
|
+
def _join_matches_eager(
|
|
580
|
+
df: pl.DataFrame, col: str, matches: pl.DataFrame, tag: str = "_two", drop_unresolved: bool = True, taxon_key: str | None = None
|
|
581
|
+
) -> pl.DataFrame:
|
|
579
582
|
"""Eager core of ``join_matches``: join ranked matches into an in-memory frame.
|
|
580
583
|
|
|
581
584
|
Args:
|
|
@@ -585,6 +588,10 @@ def _join_matches_eager(df: pl.DataFrame, col: str, matches: pl.DataFrame, tag:
|
|
|
585
588
|
tag: Suffix used to derive the level-two column name.
|
|
586
589
|
drop_unresolved: When True rows whose ``col`` did not match are dropped;
|
|
587
590
|
when False the row is kept and the resolved columns stay null.
|
|
591
|
+
taxon_key: Optional join-key column present on BOTH sides (``df`` and
|
|
592
|
+
``matches``) that must match in addition to the term, used by per-row
|
|
593
|
+
taxon scoping (``ResolveSpec.taxon_from``) so one term's matches from
|
|
594
|
+
different taxon scopes never leak across rows. Dropped from the output.
|
|
588
595
|
|
|
589
596
|
Returns:
|
|
590
597
|
DataFrame with resolved columns. Tied best-ranked CURIEs are returned as
|
|
@@ -594,14 +601,25 @@ def _join_matches_eager(df: pl.DataFrame, col: str, matches: pl.DataFrame, tag:
|
|
|
594
601
|
l1: str = col
|
|
595
602
|
l2: str = l1 + tag
|
|
596
603
|
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
604
|
+
if taxon_key is not None:
|
|
605
|
+
result: pl.DataFrame = df.join(
|
|
606
|
+
matches.filter(pl.col("NLP_LEVEL").eq(1)), left_on=[l1, taxon_key], right_on=["term", taxon_key], how="left", suffix="_l1"
|
|
607
|
+
)
|
|
608
|
+
l2_matches: pl.DataFrame = matches.filter(pl.col("NLP_LEVEL").eq(2))
|
|
609
|
+
result = result.join(l2_matches, left_on=[l2, taxon_key], right_on=["term", taxon_key], how="left", suffix="_l2")
|
|
610
|
+
else:
|
|
611
|
+
result = df.join(matches.filter(pl.col("NLP_LEVEL").eq(1)), left_on=l1, right_on="term", how="left", suffix="_l1")
|
|
612
|
+
l2_matches = matches.filter(pl.col("NLP_LEVEL").eq(2))
|
|
613
|
+
result = result.join(l2_matches, left_on=l2, right_on="term", how="left", suffix="_l2")
|
|
601
614
|
|
|
602
615
|
result = result.with_columns([_coalesce_expr(col, suffix, base, prefix, cast_str) for suffix, base, prefix, cast_str in _JOIN_COALESCE_SPECS])
|
|
603
616
|
|
|
604
|
-
|
|
617
|
+
exclude: str = r"^(CURIE|PREFERRED_NAME|CATEGORY_NAME|TAXON_ID|SOURCE_NAME|SOURCE_VERSION|NLP_LEVEL|PR|FREQUENCY)(_l2)?$"
|
|
618
|
+
if taxon_key is not None:
|
|
619
|
+
exclude = (
|
|
620
|
+
r"^(CURIE|PREFERRED_NAME|CATEGORY_NAME|TAXON_ID|SOURCE_NAME|SOURCE_VERSION|NLP_LEVEL|PR|FREQUENCY|" + re.escape(taxon_key) + r")(_l2)?$"
|
|
621
|
+
)
|
|
622
|
+
result = result.select(pl.exclude(exclude))
|
|
605
623
|
result = result.select(pl.exclude(col + tag))
|
|
606
624
|
result = result.with_columns(pl.col(f"{col}_taxon").replace("NCBITaxon:0", None))
|
|
607
625
|
if drop_unresolved:
|
|
@@ -627,6 +645,89 @@ class ResolveSpec(NamedTuple):
|
|
|
627
645
|
Set only for ``nullable`` qualifiers; subject/object always resolve strict so an
|
|
628
646
|
edge with a missing node is dropped, never emitted.
|
|
629
647
|
"""
|
|
648
|
+
taxon_from: str | None = None
|
|
649
|
+
"""When set, per-row taxon scoping from another resolved column of the same frame.
|
|
650
|
+
|
|
651
|
+
Names a node column that an earlier ``resolve_batch`` pass already resolved: its
|
|
652
|
+
``<col>_taxon`` metadata (an ``NCBITaxon:<id>`` string) scopes this column's
|
|
653
|
+
``filter_and_rank`` taxon filter row by row instead of the static ``taxon``.
|
|
654
|
+
Rows whose taxon cell is blank fall back to the 9606 default; a non-null cell
|
|
655
|
+
whose resolved taxon is not an NCBITaxon id fails loudly rather than silently
|
|
656
|
+
resolving everything under one species.
|
|
657
|
+
"""
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
_TAXON_SCOPE: str = "__taxon_scope"
|
|
661
|
+
"""Helper column carrying each row's per-row taxon scope during ``resolve_batch``.
|
|
662
|
+
|
|
663
|
+
Filled by :func:`_per_row_taxon_matches` from the ``taxon_from`` column's resolved
|
|
664
|
+
metadata (an NCBITaxon id string, or the ``9606`` fallback for blank taxon cells),
|
|
665
|
+
used as the second join key so one term's matches from different taxon scopes never
|
|
666
|
+
leak across rows, and dropped before the frame leaves the op.
|
|
667
|
+
"""
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def _per_row_taxon_matches(
|
|
671
|
+
result: pl.DataFrame, spec: ResolveSpec, raw: pl.DataFrame, tag: str, column_context: bool
|
|
672
|
+
) -> tuple[pl.DataFrame, pl.DataFrame]:
|
|
673
|
+
"""Scope this spec's matches per row by the ``taxon_from`` column's resolved taxon.
|
|
674
|
+
|
|
675
|
+
The ``taxon_from`` column was resolved by an earlier ``resolve_batch`` pass, so
|
|
676
|
+
``result`` already carries its resolved CURIE (``<col>``) and its taxon metadata
|
|
677
|
+
(``<col>_taxon``, ``NCBITaxon:<id>`` or null for a non-taxon-bearing concept).
|
|
678
|
+
Rows are bucketed by that scope; each bucket's terms go through the same
|
|
679
|
+
``filter_and_rank`` a static-taxon spec uses, keyed by the scope so the join back
|
|
680
|
+
picks each row's own bucket. Blank taxon cells fall back to the 9606 default; a
|
|
681
|
+
non-null cell whose taxon is not an NCBITaxon id fails the build instead of
|
|
682
|
+
silently resolving everything under one species.
|
|
683
|
+
|
|
684
|
+
Returns:
|
|
685
|
+
The frame with ``_TAXON_SCOPE`` appended (join key; the caller drops it after
|
|
686
|
+
the join) and the concatenated per-scope matches carrying the same key.
|
|
687
|
+
|
|
688
|
+
Raises:
|
|
689
|
+
TablassertError: ``taxon-from-unresolved-column`` when the taxon source's
|
|
690
|
+
resolved or metadata column is absent (the earlier pass never ran);
|
|
691
|
+
``taxon-from-not-ncbitaxon`` when a non-null taxon cell resolves without
|
|
692
|
+
an NCBITaxon taxon id.
|
|
693
|
+
"""
|
|
694
|
+
source: str = spec.taxon_from or ""
|
|
695
|
+
meta_col: str = f"{source}_taxon"
|
|
696
|
+
missing: list[str] = [c for c in (source, meta_col) if c not in result.columns]
|
|
697
|
+
if missing:
|
|
698
|
+
raise TablassertError(
|
|
699
|
+
f"ResolveSpec for {spec.col!r} sets taxon_from={source!r} but the frame lacks {', '.join(repr(c) for c in missing)}; "
|
|
700
|
+
"the named column must be resolved by an earlier resolve_batch pass in the same pipeline.",
|
|
701
|
+
code="taxon-from-unresolved-column",
|
|
702
|
+
)
|
|
703
|
+
value: pl.Series = result.select(pl.col(source).cast(pl.String).str.strip_chars().alias("v"))["v"]
|
|
704
|
+
meta: pl.Series = result.select(pl.col(meta_col).cast(pl.String).str.strip_chars().alias("m"))["m"]
|
|
705
|
+
has_value: pl.Series = value.is_not_null() & (value != "")
|
|
706
|
+
is_ncbitaxon: pl.Series = meta.str.contains(r"^NCBITaxon:\d+$").fill_null(False)
|
|
707
|
+
bad: pl.Series = has_value & ~is_ncbitaxon
|
|
708
|
+
if bad.any():
|
|
709
|
+
offenders: list[str] = sorted({f"{v!r} -> {m!r}" for v, m in zip(value.filter(bad).to_list(), meta.filter(bad).to_list(), strict=True)})[:5]
|
|
710
|
+
raise TablassertError(
|
|
711
|
+
f"taxon_from column {source!r} carries value(s) that did not resolve to an NCBITaxon taxon: "
|
|
712
|
+
f"{', '.join(offenders)}. Fix the source table so every non-blank taxon cell names a species, "
|
|
713
|
+
"or the row's entity resolution cannot be scoped.",
|
|
714
|
+
code="taxon-from-not-ncbitaxon",
|
|
715
|
+
)
|
|
716
|
+
scope: pl.Expr = (
|
|
717
|
+
pl.when(has_value & is_ncbitaxon)
|
|
718
|
+
.then(pl.col(meta_col).cast(pl.String).str.strip_chars().str.replace("NCBITaxon:", ""))
|
|
719
|
+
.otherwise(pl.lit("9606"))
|
|
720
|
+
)
|
|
721
|
+
result = result.with_columns(scope.alias(_TAXON_SCOPE))
|
|
722
|
+
parts: list[pl.DataFrame] = []
|
|
723
|
+
for taxon in sorted(set(result[_TAXON_SCOPE].to_list())):
|
|
724
|
+
bucket: pl.DataFrame = result.filter(pl.col(_TAXON_SCOPE) == taxon)
|
|
725
|
+
bucket_terms: pl.DataFrame = distinct(bucket.lazy(), spec.col, spec.col + tag).collect()
|
|
726
|
+
bucket_matches: pl.DataFrame = filter_and_rank(
|
|
727
|
+
raw, bucket_terms, taxon, spec.prioritize, spec.avoid, column_context, spec.exclude_prefixes, spec.exclude_regex
|
|
728
|
+
)
|
|
729
|
+
parts.append(bucket_matches.with_columns(pl.lit(taxon).alias(_TAXON_SCOPE)))
|
|
730
|
+
return result, (pl.concat(parts) if parts else empty_matches(column_context))
|
|
630
731
|
|
|
631
732
|
|
|
632
733
|
def resolve_batch(
|
|
@@ -721,6 +822,12 @@ def resolve_batch(
|
|
|
721
822
|
for spec in specs:
|
|
722
823
|
if on_phase is not None:
|
|
723
824
|
on_phase(f"resolve:{spec.col}")
|
|
825
|
+
if spec.taxon_from is not None:
|
|
826
|
+
scoped_result, matches = _per_row_taxon_matches(result, spec, raw, tag, column_context)
|
|
827
|
+
if log:
|
|
828
|
+
log_unmatched(spec.col, terms_by_col[spec.col].lazy(), matches, section_hash, config_file)
|
|
829
|
+
result = _join_matches_eager(scoped_result, spec.col, matches, tag, drop_unresolved=not spec.nullable, taxon_key=_TAXON_SCOPE)
|
|
830
|
+
continue
|
|
724
831
|
terms_df: pl.DataFrame = terms_by_col[spec.col]
|
|
725
832
|
matches: pl.DataFrame = filter_and_rank(
|
|
726
833
|
raw, terms_df, spec.taxon, spec.prioritize, spec.avoid, column_context, spec.exclude_prefixes, spec.exclude_regex
|
|
@@ -45,11 +45,11 @@ from tablassert.coerce import (
|
|
|
45
45
|
sig,
|
|
46
46
|
study_size_target,
|
|
47
47
|
)
|
|
48
|
-
from tablassert.enums import EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
|
|
48
|
+
from tablassert.enums import Comparisons, EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
|
|
49
49
|
from tablassert.errors import TablassertError
|
|
50
50
|
from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
|
|
51
51
|
from tablassert.log import cat
|
|
52
|
-
from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
|
|
52
|
+
from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, Reindex, RIGConfig, Section
|
|
53
53
|
from tablassert.nlp import level_one, level_two
|
|
54
54
|
from tablassert.qc import fullmap_audit
|
|
55
55
|
from tablassert.rig import (
|
|
@@ -1070,6 +1070,178 @@ def reindex(df: pl.LazyFrame, col: str, op: Callable, comp: str | int | float, c
|
|
|
1070
1070
|
return df.filter(op(expr, comp))
|
|
1071
1071
|
|
|
1072
1072
|
|
|
1073
|
+
MEMBERSHIP_KEY_COLUMN: str = "_reindex_membership_key"
|
|
1074
|
+
"""Base name for the throwaway join key column on the cast branch; extended with underscores until
|
|
1075
|
+
it is unused if a source frame already carries it, and dropped again before the op returns."""
|
|
1076
|
+
|
|
1077
|
+
MEMBERSHIP_UNCASTABLE_DTYPES: frozenset[str] = frozenset({"List", "Array", "Struct", "Object", "Binary"})
|
|
1078
|
+
"""polars dtype class NAMES whose cells have no single-string identity, so membership is refused.
|
|
1079
|
+
|
|
1080
|
+
Matched by name because a module-level ``pl.List`` reference would resolve the lazy proxy and import
|
|
1081
|
+
polars at ``import tablassert.lib``, which is exactly what :mod:`tablassert._lazy` exists to avoid.
|
|
1082
|
+
Every parametrized variant reports its base name (``List(String)`` is named ``List``).
|
|
1083
|
+
|
|
1084
|
+
A ``List``/``Array`` cell cannot be cast to ``String`` at all (``InvalidOperationError``), an
|
|
1085
|
+
``Object`` cell casts through python and never compares equal to an identifier, a ``Struct`` casts to
|
|
1086
|
+
a brace literal no identifier is spelled to match, and a ``Binary`` cell casts only when it happens to
|
|
1087
|
+
be valid UTF-8, so one list would match some rows and raise on the rest. Joining list cells on a
|
|
1088
|
+
separator instead would make a multi-value cell never match a single identifier, a silent wrong
|
|
1089
|
+
answer, so these dtypes raise a coded error.
|
|
1090
|
+
"""
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
def _membership_key_column(schema: pl.Schema) -> str:
|
|
1094
|
+
"""Return a join key column name the frame does not already use.
|
|
1095
|
+
|
|
1096
|
+
Args:
|
|
1097
|
+
schema: Collected schema of the frame being filtered.
|
|
1098
|
+
|
|
1099
|
+
Returns:
|
|
1100
|
+
:data:`MEMBERSHIP_KEY_COLUMN`, or that name extended with underscores until it is free.
|
|
1101
|
+
"""
|
|
1102
|
+
key_col: str = MEMBERSHIP_KEY_COLUMN
|
|
1103
|
+
while key_col in schema:
|
|
1104
|
+
key_col = f"{key_col}_"
|
|
1105
|
+
return key_col
|
|
1106
|
+
|
|
1107
|
+
|
|
1108
|
+
def reindex_membership(df: pl.LazyFrame, col: str, keys: list[str], anti: bool = False) -> pl.LazyFrame:
|
|
1109
|
+
"""Reindex a LazyFrame by exact membership in a list of identifiers (join / antijoin).
|
|
1110
|
+
|
|
1111
|
+
Args:
|
|
1112
|
+
df: Source LazyFrame.
|
|
1113
|
+
col: Column whose raw (pre-resolution) cell values are tested.
|
|
1114
|
+
keys: Identifier list. Duplicates are ignored: a semi/anti join cannot multiply left
|
|
1115
|
+
rows, and the join build side is cheaper once deduplicated.
|
|
1116
|
+
anti: ``False`` (join) keeps rows whose value is in ``keys``; ``True`` (antijoin) keeps
|
|
1117
|
+
the rows that are not.
|
|
1118
|
+
|
|
1119
|
+
Returns:
|
|
1120
|
+
Filtered LazyFrame keeping the surviving rows in left-frame order, with every original
|
|
1121
|
+
column and its original dtype.
|
|
1122
|
+
|
|
1123
|
+
Raises:
|
|
1124
|
+
TablassertError: ``comparison-membership-empty`` for an empty list,
|
|
1125
|
+
``reindex-membership-column-missing`` when the source lacks ``col``, and
|
|
1126
|
+
``reindex-membership-column-dtype-unsupported`` for cells with no single string form
|
|
1127
|
+
(``List``, ``Array``, ``Struct``, ``Object``, ``Binary``).
|
|
1128
|
+
|
|
1129
|
+
Notes:
|
|
1130
|
+
Why a join and not ``is_in``: on the pinned polars (``>=2.0.0rc2``) a semi/anti hash join
|
|
1131
|
+
against a once-built key frame is the faster measured path once the list is non-trivial (1M
|
|
1132
|
+
rows, median of three collects: 100k keys 0.018s vs 0.029s filtered; 1k keys 0.003s vs
|
|
1133
|
+
0.004s; a tie at ten keys), and it is the membership idiom already used in
|
|
1134
|
+
:mod:`tablassert.qc`. No ``is_in`` fast path exists for tiny lists: it propagates nulls
|
|
1135
|
+
differently from antijoin, and the gap is milliseconds per million rows.
|
|
1136
|
+
|
|
1137
|
+
``maintain_order="left"`` is explicit, not incidental: polars documents the default order as
|
|
1138
|
+
unspecified ("might differ across Polars versions or even between different runs"), and the
|
|
1139
|
+
dependency is an open range, so a future 2.x must not be able to reshuffle a section's rows.
|
|
1140
|
+
Measured free here (1M rows / 100k keys, five runs, median): semi 5.1ms -> 4.6ms, anti 5.4ms
|
|
1141
|
+
-> 5.4ms.
|
|
1142
|
+
|
|
1143
|
+
The string path joins straight on ``col``, because semi/anti joins emit only the left frame's
|
|
1144
|
+
columns. Only the cast path adds a throwaway helper column, and it is dropped again, so
|
|
1145
|
+
entity resolution downstream sees the cell exactly as the source produced it.
|
|
1146
|
+
|
|
1147
|
+
Null semantics are join semantics and deliberately differ from ``ne``: a null cell never
|
|
1148
|
+
matches, so join drops it while antijoin KEEPS it.
|
|
1149
|
+
|
|
1150
|
+
Membership is textual, so an identifier must be spelled the way polars' cast renders it.
|
|
1151
|
+
Measured forms: ``Int64`` 3 -> ``"3"``; ``Float64`` 3.0 -> ``"3.0"`` (1e+20 keeps scientific
|
|
1152
|
+
form); ``Bool`` -> ``"true"``/``"false"``; ``Date`` -> ``"2024-01-15"``; ``Datetime`` ->
|
|
1153
|
+
``"2024-01-15 06:24:30.000000"`` (space, not ``T``, with microsecond precision); ``Time`` ->
|
|
1154
|
+
``"01:02:03"``; ``Decimal(10,2)`` 1.5 -> ``"1.50"`` (scale kept). A misspelled list is not an
|
|
1155
|
+
error: join emits an empty section and antijoin emits every row. No zero-match warning is
|
|
1156
|
+
emitted, because detecting it would cost an extra pass over the frame on every build.
|
|
1157
|
+
"""
|
|
1158
|
+
if not keys:
|
|
1159
|
+
raise TablassertError("`join`/`antijoin` reindex requires at least one identifier.", code="comparison-membership-empty")
|
|
1160
|
+
|
|
1161
|
+
schema: pl.Schema = df.collect_schema()
|
|
1162
|
+
if col not in schema:
|
|
1163
|
+
raise TablassertError(f"`join`/`antijoin` reindex targets `{col}`, which the source does not have.", code="reindex-membership-column-missing")
|
|
1164
|
+
if type(schema[col]).__name__ in MEMBERSHIP_UNCASTABLE_DTYPES:
|
|
1165
|
+
raise TablassertError(
|
|
1166
|
+
f"`join`/`antijoin` reindex cannot test `{col}`: {schema[col]} cells have no single string form to match identifiers against.",
|
|
1167
|
+
code="reindex-membership-column-dtype-unsupported",
|
|
1168
|
+
)
|
|
1169
|
+
|
|
1170
|
+
unique: list[str] = list(dict.fromkeys(keys))
|
|
1171
|
+
how: Literal["semi", "anti"] = "anti" if anti else "semi"
|
|
1172
|
+
if isinstance(schema[col], pl.String):
|
|
1173
|
+
# No helper column: the join returns the left frame's columns unchanged.
|
|
1174
|
+
return df.join(pl.DataFrame({col: unique}).lazy(), on=col, how=how, maintain_order="left")
|
|
1175
|
+
key_col: str = _membership_key_column(schema)
|
|
1176
|
+
left: pl.LazyFrame = df.with_columns(pl.col(col).cast(pl.String, strict=False).alias(key_col))
|
|
1177
|
+
return left.join(pl.DataFrame({key_col: unique}).lazy(), on=key_col, how=how, maintain_order="left").drop(key_col)
|
|
1178
|
+
|
|
1179
|
+
|
|
1180
|
+
def reindex_regex(df: pl.LazyFrame, col: str, pattern: str, negate: bool = False) -> pl.LazyFrame:
|
|
1181
|
+
"""Reindex a LazyFrame by filtering rows on a polars regex match against a column.
|
|
1182
|
+
|
|
1183
|
+
Args:
|
|
1184
|
+
df: Source LazyFrame.
|
|
1185
|
+
col: Column to test.
|
|
1186
|
+
pattern: Polars-compatible regex, matched unanchored and case-sensitively (write `^`/`$`
|
|
1187
|
+
to pin a whole cell).
|
|
1188
|
+
negate: When True keep the rows that do NOT match (the `ne` filter); when False keep the
|
|
1189
|
+
matches (the `eq` filter).
|
|
1190
|
+
|
|
1191
|
+
Returns:
|
|
1192
|
+
LazyFrame containing only rows whose predicate is true.
|
|
1193
|
+
|
|
1194
|
+
Notes:
|
|
1195
|
+
The column is cast to String first, so a typed source (parquet, or a numeric column in a
|
|
1196
|
+
text table) filters on its text form instead of raising polars' ``InvalidOperationError:
|
|
1197
|
+
expected String type``. A null cell matches nothing and ``not_()`` leaves it null, so null
|
|
1198
|
+
rows drop under both directions -- the same null rule literal `eq`/`ne` already have, since
|
|
1199
|
+
a null comparison is never true.
|
|
1200
|
+
"""
|
|
1201
|
+
match: pl.Expr = pl.col(col).cast(pl.String).str.contains(pattern)
|
|
1202
|
+
return df.filter(match.not_() if negate else match)
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
def reindex_op(entry: Reindex) -> tuple[Callable, tuple[Any, ...]]:
|
|
1206
|
+
"""Compile one ``Reindex`` config entry into its Tcode op tuple.
|
|
1207
|
+
|
|
1208
|
+
Args:
|
|
1209
|
+
entry: Validated ``Reindex`` model. The validator guarantees the pairing: a membership entry
|
|
1210
|
+
carries a non-empty ``list[str]`` comparator, ``regex`` only ever sits on ``eq``/``ne``,
|
|
1211
|
+
and a scalar entry carries a scalar comparator.
|
|
1212
|
+
|
|
1213
|
+
Returns:
|
|
1214
|
+
``(callable, args)`` for the pipeline: :func:`reindex_membership` for ``join``/``antijoin``,
|
|
1215
|
+
:func:`reindex_regex` for a ``regex`` entry, otherwise :func:`reindex` with the ``operator``
|
|
1216
|
+
module callable and the same ``cast`` choice it has always used (numeric comparisons cast to
|
|
1217
|
+
Float64, ``eq``/``ne`` do not).
|
|
1218
|
+
|
|
1219
|
+
Notes:
|
|
1220
|
+
Legacy entries keep emitting exactly the tuples they always did, so their run-cache op digests
|
|
1221
|
+
do not move; membership is the only new shape.
|
|
1222
|
+
|
|
1223
|
+
``Reindex.comparison`` carries the plain enum *value* at runtime because ``TablaBase`` sets
|
|
1224
|
+
``use_enum_values=True``, so this branches on the strings, exactly as the inline comprehension
|
|
1225
|
+
it replaces did.
|
|
1226
|
+
"""
|
|
1227
|
+
col: str = idxname(entry.column)
|
|
1228
|
+
comparison: str = entry.comparison
|
|
1229
|
+
if comparison in (Comparisons.JOIN.value, Comparisons.ANTIJOIN.value):
|
|
1230
|
+
if not isinstance(entry.comparator, list):
|
|
1231
|
+
raise TablassertError(
|
|
1232
|
+
f"`{comparison}` reindex requires a list of identifiers, got {type(entry.comparator).__name__}.",
|
|
1233
|
+
code="comparison-membership-comparator-required",
|
|
1234
|
+
)
|
|
1235
|
+
return (reindex_membership, (col, entry.comparator, comparison == Comparisons.ANTIJOIN.value))
|
|
1236
|
+
if entry.regex:
|
|
1237
|
+
# models.Reindex rejects `regex` off eq/ne, so this arm never needs the numeric cast or an
|
|
1238
|
+
# operator lookup.
|
|
1239
|
+
return (reindex_regex, (col, entry.comparator, comparison == Comparisons.NE.value))
|
|
1240
|
+
if comparison in (Comparisons.NE.value, Comparisons.EQ.value):
|
|
1241
|
+
return (reindex, (col, getattr(operator, comparison), entry.comparator, False))
|
|
1242
|
+
return (reindex, (col, getattr(operator, comparison), entry.comparator))
|
|
1243
|
+
|
|
1244
|
+
|
|
1073
1245
|
def drop_not_significant(lf: pl.LazyFrame, col: str = "statistical_significance_qualifier") -> pl.LazyFrame:
|
|
1074
1246
|
"""Drop release-mode edges whose significance qualifier is ``not_significant``.
|
|
1075
1247
|
|
|
@@ -1338,14 +1510,7 @@ class Tcode(Section):
|
|
|
1338
1510
|
(idx, ()),
|
|
1339
1511
|
(crop, (self.source.row_slice,)) if self.source.row_slice else None,
|
|
1340
1512
|
(pick, (self.source.rows,)) if self.source.rows else None,
|
|
1341
|
-
[
|
|
1342
|
-
(reindex, (idxname(x.column), getattr(operator, x.comparison), x.comparator))
|
|
1343
|
-
if x.comparison not in ["ne", "eq"]
|
|
1344
|
-
else (reindex, (idxname(x.column), getattr(operator, x.comparison), x.comparator, False))
|
|
1345
|
-
for x in self.source.reindex
|
|
1346
|
-
]
|
|
1347
|
-
if self.source.reindex
|
|
1348
|
-
else None,
|
|
1513
|
+
[reindex_op(x) for x in self.source.reindex] if self.source.reindex else None,
|
|
1349
1514
|
# --head preview: randomly sample min(HEAD_ROWS, height) rows before any encoding/resolve.
|
|
1350
1515
|
(head, (HEAD_ROWS,)) if self.head else None,
|
|
1351
1516
|
[
|
|
@@ -1401,9 +1566,32 @@ class Tcode(Section):
|
|
|
1401
1566
|
x.exclude_prefixes,
|
|
1402
1567
|
x.exclude_regex,
|
|
1403
1568
|
x.nullable if isinstance(x, Qualifier) else False,
|
|
1569
|
+
x.taxon_from,
|
|
1404
1570
|
)
|
|
1405
1571
|
for x, col in node_columns
|
|
1406
1572
|
]
|
|
1573
|
+
# Per-row taxon scoping (ResolveSpec.taxon_from): a scoped column reads the
|
|
1574
|
+
# taxon metadata its source column's resolution emits, so sources must resolve
|
|
1575
|
+
# first. Layer the specs by dependency (config validation has already rejected
|
|
1576
|
+
# cycles and unknown targets); with no taxon_from anywhere this is one batch
|
|
1577
|
+
# and the op list is exactly what it always was.
|
|
1578
|
+
spec_cols: set[str] = {spec.col for spec in specs}
|
|
1579
|
+
batches: list[list[ResolveSpec]] = []
|
|
1580
|
+
placed: set[str] = set()
|
|
1581
|
+
pending: list[ResolveSpec] = list(specs)
|
|
1582
|
+
while pending:
|
|
1583
|
+
ready: list[ResolveSpec] = [s for s in pending if s.taxon_from is None or s.taxon_from in placed or s.taxon_from not in spec_cols]
|
|
1584
|
+
if not ready:
|
|
1585
|
+
raise TablassertError(
|
|
1586
|
+
f"taxon_from references form a cycle among {', '.join(sorted(s.col for s in pending))}; "
|
|
1587
|
+
"no column can resolve first. This is rejected at config time, so a build reaching here means the "
|
|
1588
|
+
"section was assembled programmatically -- fix the spec layering.",
|
|
1589
|
+
code="taxon-from-cycle",
|
|
1590
|
+
)
|
|
1591
|
+
batches.append(ready)
|
|
1592
|
+
placed.update(s.col for s in ready)
|
|
1593
|
+
pending = [s for s in pending if s.col not in placed]
|
|
1594
|
+
resolve_ops: list[Any] = [(resolve_batch, (batch, db, self.log, self.store.stem, self.config.name, True, "_two")) for batch in batches]
|
|
1407
1595
|
return [
|
|
1408
1596
|
[self.node_prep(x, col) for x, col in node_columns],
|
|
1409
1597
|
# Encode only: no pre-resolution snapshot and no NLP normalization, both of
|
|
@@ -1415,7 +1603,7 @@ class Tcode(Section):
|
|
|
1415
1603
|
# copied them into named slots; trim before resolution so the frame that
|
|
1416
1604
|
# resolve_batch materializes and joins stays narrow.
|
|
1417
1605
|
(trim, ()),
|
|
1418
|
-
|
|
1606
|
+
*resolve_ops,
|
|
1419
1607
|
# QC audits only the strict columns: a nullable qualifier's nulls are expected
|
|
1420
1608
|
# (blank cell / no match), not resolution errors for the audit to delete.
|
|
1421
1609
|
[
|
|
@@ -1565,6 +1753,8 @@ PHASE_OF: dict[Callable, str] = {
|
|
|
1565
1753
|
crop: "filter",
|
|
1566
1754
|
pick: "filter",
|
|
1567
1755
|
reindex: "filter",
|
|
1756
|
+
reindex_membership: "filter",
|
|
1757
|
+
reindex_regex: "filter",
|
|
1568
1758
|
head: "filter",
|
|
1569
1759
|
coerce_columns: "clean",
|
|
1570
1760
|
clean_numeric: "clean",
|
|
@@ -119,21 +119,119 @@ class TablaBase(BaseModel):
|
|
|
119
119
|
class Reindex(TablaBase):
|
|
120
120
|
column: str = Field(..., pattern=r"^[A-Z]{1,3}$", description="Source column letters used for row filtering.", examples=["A", "AA"])
|
|
121
121
|
comparison: Comparisons = Field(
|
|
122
|
-
Comparisons.NE,
|
|
122
|
+
Comparisons.NE,
|
|
123
|
+
description="Comparison operator used in reindex filtering.",
|
|
124
|
+
examples=[Comparisons.NE, Comparisons.EQ, Comparisons.GT, Comparisons.JOIN, Comparisons.ANTIJOIN],
|
|
123
125
|
)
|
|
124
|
-
comparator: str | int | float = Field(
|
|
126
|
+
comparator: str | int | float | list[str] = Field(
|
|
127
|
+
...,
|
|
128
|
+
description=(
|
|
129
|
+
"Right-side value compared against the selected column. For `join`/`antijoin` this is the list of "
|
|
130
|
+
"identifiers matched exactly against the raw (pre-resolution) cell value."
|
|
131
|
+
),
|
|
132
|
+
examples=["N/A", 0, 1.5, ["CHEBI:41774", "CHEBI:48217"]],
|
|
133
|
+
)
|
|
134
|
+
regex: bool = Field(
|
|
135
|
+
False,
|
|
136
|
+
description=(
|
|
137
|
+
"When true, `comparator` is a polars-compatible regex matched against the cell instead of a literal "
|
|
138
|
+
"value: `eq` keeps rows whose cell matches, `ne` keeps rows that do not. Matching is case-sensitive and "
|
|
139
|
+
"unanchored (add `^`/`$` to pin the whole cell), the column is compared as text, and a null cell never "
|
|
140
|
+
"matches, so null rows drop under both operators. Only valid with `eq`/`ne`; an invalid or empty pattern "
|
|
141
|
+
"is rejected at config time (`regex-bad-pattern`)."
|
|
142
|
+
),
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
@model_validator(mode="before")
|
|
146
|
+
@classmethod
|
|
147
|
+
def membership_members_are_strings(cls, data: Any) -> Any:
|
|
148
|
+
"""Reject an identifier list whose members are not strings, with a code rather than a field type error.
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
data: Raw config value for one ``Reindex`` entry.
|
|
152
|
+
|
|
153
|
+
Returns:
|
|
154
|
+
``data`` unchanged when the entry is not a membership list, or when every member is a
|
|
155
|
+
string.
|
|
156
|
+
|
|
157
|
+
Raises:
|
|
158
|
+
TablassertValidationError: ``comparison-membership-member-not-a-string`` when a
|
|
159
|
+
``join``/``antijoin`` list holds a non-string member.
|
|
160
|
+
|
|
161
|
+
Notes:
|
|
162
|
+
Without this guard the pydantic union rejects `comparator: [3, 4]` (the natural spelling
|
|
163
|
+
against an `Int64` column, whose string form is `3`) as a bare `string_type` field error
|
|
164
|
+
with no stable code and no docs URL, while every other failure in this family is coded.
|
|
165
|
+
The fix stays "quote the identifiers": membership is textual, so an unquoted number is a
|
|
166
|
+
config error rather than something to coerce silently.
|
|
167
|
+
"""
|
|
168
|
+
if not isinstance(data, dict):
|
|
169
|
+
return data
|
|
170
|
+
comparison = data.get("comparison")
|
|
171
|
+
comparator = data.get("comparator")
|
|
172
|
+
if comparison not in (Comparisons.JOIN.value, Comparisons.ANTIJOIN.value) or not isinstance(comparator, list):
|
|
173
|
+
return data
|
|
174
|
+
for index, member in enumerate(comparator):
|
|
175
|
+
if not isinstance(member, str):
|
|
176
|
+
raise TablassertValidationError(
|
|
177
|
+
f"`join`/`antijoin` identifier lists must hold quoted strings: member {index} is {type(member).__name__} ({member!r}). "
|
|
178
|
+
f'Quote it (for example ["{member}"]) to match the cell as text.',
|
|
179
|
+
code="comparison-membership-member-not-a-string",
|
|
180
|
+
)
|
|
181
|
+
return data
|
|
125
182
|
|
|
126
183
|
@model_validator(mode="after")
|
|
127
184
|
def comparison_datatypes(self: Self) -> Self:
|
|
128
185
|
x: Comparisons = self.comparison
|
|
129
|
-
y: str | int | float = self.comparator
|
|
186
|
+
y: str | int | float | list[str] = self.comparator
|
|
130
187
|
|
|
131
|
-
if x == Comparisons.
|
|
188
|
+
if x == Comparisons.JOIN or x == Comparisons.ANTIJOIN:
|
|
189
|
+
if self.regex:
|
|
190
|
+
raise TablassertValidationError(
|
|
191
|
+
"`regex` is only meaningful with the `eq`/`ne` comparisons; `join`/`antijoin` match identifiers by exact string identity against a list.",
|
|
192
|
+
code="comparison-regex-requires-eq-ne",
|
|
193
|
+
)
|
|
194
|
+
if not isinstance(y, list):
|
|
195
|
+
raise TablassertValidationError(
|
|
196
|
+
f"`join`/`antijoin` comparisons require a list of identifier strings, got {type(y).__name__}.",
|
|
197
|
+
code="comparison-membership-comparator-required",
|
|
198
|
+
)
|
|
199
|
+
if not y:
|
|
200
|
+
raise TablassertValidationError(
|
|
201
|
+
"`join`/`antijoin` comparators must hold at least one identifier.", code="comparison-membership-empty"
|
|
202
|
+
)
|
|
203
|
+
elif x == Comparisons.NE or x == Comparisons.EQ:
|
|
132
204
|
if not isinstance(y, str):
|
|
133
205
|
raise TablassertValidationError(
|
|
134
206
|
f"`eq`/`ne` comparisons require a str comparator, got {type(y).__name__}.", code="comparison-bad-comparator-type"
|
|
135
207
|
)
|
|
208
|
+
if self.regex:
|
|
209
|
+
# An empty-or-everything pattern compiles, but it matches EVERY cell, so a `ne` filter
|
|
210
|
+
# would silently drop the whole table. Reject whitespace-only spellings and any pattern
|
|
211
|
+
# that can match the empty string (`^`, `()`, `.*`, `x*`) with the same code the
|
|
212
|
+
# `exclude_regex` probe uses.
|
|
213
|
+
if not y.strip():
|
|
214
|
+
raise TablassertValidationError(
|
|
215
|
+
f"`comparator` must be a non-empty pattern when `regex` is true (an empty pattern matches every cell), got {y!r}.",
|
|
216
|
+
code="regex-bad-pattern",
|
|
217
|
+
)
|
|
218
|
+
try:
|
|
219
|
+
pl.Series([""]).str.contains(y)
|
|
220
|
+
except Exception as e:
|
|
221
|
+
raise TablassertValidationError(
|
|
222
|
+
f"`comparator` must be a polars-compatible regex when `regex` is true, got {y!r}: {e}", code="regex-bad-pattern"
|
|
223
|
+
) from e
|
|
224
|
+
if pl.Series([""]).str.contains(y).item():
|
|
225
|
+
raise TablassertValidationError(
|
|
226
|
+
f"`comparator` must not match an empty cell when `regex` is true (it would match every cell, so an `ne` filter would drop the whole table), got {y!r}.",
|
|
227
|
+
code="regex-bad-pattern",
|
|
228
|
+
)
|
|
136
229
|
else:
|
|
230
|
+
if self.regex:
|
|
231
|
+
raise TablassertValidationError(
|
|
232
|
+
"`regex` is only meaningful with the `eq`/`ne` comparisons; ordering comparisons cast the column to Float64 and compare numerically.",
|
|
233
|
+
code="comparison-regex-requires-eq-ne",
|
|
234
|
+
)
|
|
137
235
|
if not isinstance(y, (int, float)):
|
|
138
236
|
raise TablassertValidationError(
|
|
139
237
|
f"Comparisons other than `eq`/`ne` require a float or int comparator, got {type(y).__name__}.",
|
|
@@ -307,6 +405,36 @@ class NodeEncoding(Encoding):
|
|
|
307
405
|
),
|
|
308
406
|
examples=[9606, 10090],
|
|
309
407
|
)
|
|
408
|
+
taxon_from: str | None = Field(
|
|
409
|
+
None,
|
|
410
|
+
description=(
|
|
411
|
+
"Per-row taxon scoping: name another resolved node column of the same statement (a CURIE-ranged qualifier key, "
|
|
412
|
+
"or `subject`/`object`) whose resolved NCBITaxon CURIE scopes this column's entity resolution row by row, instead of "
|
|
413
|
+
"the static `taxon`. The named column is resolved first; rows whose taxon cell is blank fall back to the 9606 "
|
|
414
|
+
"default, and a non-null cell that does not resolve to an NCBITaxon CURIE fails the build (`taxon-from-not-ncbitaxon`). "
|
|
415
|
+
"Mutually exclusive with an explicit `taxon`."
|
|
416
|
+
),
|
|
417
|
+
examples=["species_context_qualifier"],
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
@model_validator(mode="before")
|
|
421
|
+
@classmethod
|
|
422
|
+
def taxon_from_excludes_explicit_taxon(cls, data: Any) -> Any:
|
|
423
|
+
"""Reject `taxon_from` set together with an explicit `taxon`.
|
|
424
|
+
|
|
425
|
+
Two taxon knobs on one encoding would compete for the same filter, and the
|
|
426
|
+
winner would depend on evaluation order. `taxon` defaults to 9606, so the
|
|
427
|
+
check must run on the raw input: only an author-written `taxon` conflicts,
|
|
428
|
+
and `taxon_from` supplies its own default (9606 for blank taxon cells).
|
|
429
|
+
"""
|
|
430
|
+
if isinstance(data, dict) and data.get("taxon_from") is not None and "taxon" in data:
|
|
431
|
+
raise TablassertValidationError(
|
|
432
|
+
"`taxon_from` and an explicit `taxon` are mutually exclusive; remove one. `taxon_from` scopes resolution "
|
|
433
|
+
"per row from the named column and falls back to the 9606 default for blank taxon cells.",
|
|
434
|
+
code="taxon-from-conflict",
|
|
435
|
+
)
|
|
436
|
+
return data
|
|
437
|
+
|
|
310
438
|
prioritize: list[Categories] | None = Field(
|
|
311
439
|
None, description="Biolink categories ranked higher during entity resolution.", examples=[[Categories.GENE, Categories.PROTEIN]]
|
|
312
440
|
)
|
|
@@ -484,6 +612,53 @@ class Statement(TablaBase):
|
|
|
484
612
|
seen.add(key)
|
|
485
613
|
return self
|
|
486
614
|
|
|
615
|
+
@model_validator(mode="after")
|
|
616
|
+
def validate_taxon_from_targets(self: Self) -> Self:
|
|
617
|
+
"""Validate every `taxon_from` reference before a build can run.
|
|
618
|
+
|
|
619
|
+
`taxon_from` scopes one node column's resolution by another resolved node
|
|
620
|
+
column of the same statement, so the target must exist, must itself resolve
|
|
621
|
+
(enum-ranged qualifiers never resolve), must not be the column carrying the
|
|
622
|
+
reference, and must not participate in a reference cycle -- a cyclic pair
|
|
623
|
+
would have no valid resolution order. All four failures are config-time so a
|
|
624
|
+
bad reference dies at load, not hours into a build.
|
|
625
|
+
"""
|
|
626
|
+
resolved: dict[str, NodeEncoding | Qualifier] = {
|
|
627
|
+
"subject": self.subject,
|
|
628
|
+
"object": self.object,
|
|
629
|
+
**{str(q.qualifier): q for q in self.qualifiers or [] if q.resolved},
|
|
630
|
+
}
|
|
631
|
+
for name, encoding in resolved.items():
|
|
632
|
+
target: str | None = getattr(encoding, "taxon_from", None)
|
|
633
|
+
if target is None:
|
|
634
|
+
continue
|
|
635
|
+
if target not in resolved:
|
|
636
|
+
raise TablassertValidationError(
|
|
637
|
+
f"`taxon_from: {target}` on `{name}` does not name a resolved node column of this statement; "
|
|
638
|
+
"expected `subject`, `object`, or a CURIE-ranged qualifier key.",
|
|
639
|
+
code="taxon-from-unknown-column",
|
|
640
|
+
)
|
|
641
|
+
if target == name:
|
|
642
|
+
raise TablassertValidationError(
|
|
643
|
+
f"`taxon_from: {target}` on `{name}` references itself; a column cannot be scoped by its own resolution.", code="taxon-from-self"
|
|
644
|
+
)
|
|
645
|
+
# Chase the reference chain from the target: any path back to a column that
|
|
646
|
+
# already carries a reference is a cycle with no valid resolution order.
|
|
647
|
+
seen: set[str] = {name}
|
|
648
|
+
cursor: str = target
|
|
649
|
+
while cursor in resolved:
|
|
650
|
+
if cursor in seen:
|
|
651
|
+
raise TablassertValidationError(
|
|
652
|
+
f"`taxon_from` references form a cycle through `{cursor}`; there is no column that can resolve first.",
|
|
653
|
+
code="taxon-from-cycle",
|
|
654
|
+
)
|
|
655
|
+
seen.add(cursor)
|
|
656
|
+
nxt: str | None = getattr(resolved[cursor], "taxon_from", None)
|
|
657
|
+
if nxt is None:
|
|
658
|
+
break
|
|
659
|
+
cursor = nxt
|
|
660
|
+
return self
|
|
661
|
+
|
|
487
662
|
@model_validator(mode="after")
|
|
488
663
|
def warn_when_an_override_is_demoted_by_the_predicate(self: Self) -> Self:
|
|
489
664
|
"""Warn when a pinned association class cannot carry the section predicate.
|
|
@@ -258,6 +258,9 @@ OP_COST: dict[Callable, int] = {
|
|
|
258
258
|
lib.crop: 1,
|
|
259
259
|
lib.pick: 1,
|
|
260
260
|
lib.reindex: 1,
|
|
261
|
+
# Membership filter: one hash join against a small key frame, same single-predicate price.
|
|
262
|
+
lib.reindex_membership: 1,
|
|
263
|
+
lib.reindex_regex: 1,
|
|
261
264
|
# head samples min(HEAD_ROWS, height) rows, so it needs a height pass before slicing.
|
|
262
265
|
lib.head: 2,
|
|
263
266
|
# Encode: each is one lazy expression over a single column.
|
|
@@ -20,6 +20,7 @@ _MESSAGES: dict[str, str] = {
|
|
|
20
20
|
"unnamed-nodes": "nodes with no name or an empty name",
|
|
21
21
|
"unidentified-nodes": "nodes with no id or an empty id",
|
|
22
22
|
"incomplete-edges": "edges missing subject, predicate, or object",
|
|
23
|
+
"self-loop-edges": "edges whose subject and object are the same CURIE",
|
|
23
24
|
"demoted-edges": "edges demoted to bare biolink:Association despite a declared category_override",
|
|
24
25
|
"undeclared-nodes": "nodes referenced by edges but not declared in the nodes file",
|
|
25
26
|
"isolated-nodes": "declared nodes participating in no edge",
|
|
@@ -61,6 +62,9 @@ class _FileScan:
|
|
|
61
62
|
#: predicate. A demotion is only a violation when the build declared a
|
|
62
63
|
#: ``category_override`` (see :func:`study_kgx`); the count is collected either way.
|
|
63
64
|
demoted: Counter[str]
|
|
65
|
+
#: Edges that assert a concept relates to itself, keyed by the shared CURIE. See
|
|
66
|
+
#: :func:`study_kgx` for why a self-referential edge is a violation.
|
|
67
|
+
self_loops: Counter[str]
|
|
64
68
|
malformed: int
|
|
65
69
|
missing: bool
|
|
66
70
|
path: Path
|
|
@@ -94,17 +98,18 @@ def _scan_ndjson(path: Path, *, edge: bool) -> _FileScan:
|
|
|
94
98
|
|
|
95
99
|
Args:
|
|
96
100
|
path: Path to a ``.nodes.ndjson`` or ``.edges.ndjson`` file.
|
|
97
|
-
edge: ``True`` to collect referenced ids from ``subject``/``object
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
+
edge: ``True`` to collect referenced ids from ``subject``/``object``, assert the
|
|
102
|
+
three core edge slots are present, and tally self-referential
|
|
103
|
+
(``subject == object``) edges; ``False`` to collect declared node ``id``s and
|
|
104
|
+
track duplicates, nodes with no name or an empty name, and nodes with no id
|
|
105
|
+
or an empty id.
|
|
101
106
|
|
|
102
107
|
Returns:
|
|
103
108
|
A :class:`_FileScan`; ``missing`` is set (and nothing else) when the
|
|
104
109
|
file does not exist, so a typo'd path can never read as a clean pass.
|
|
105
110
|
"""
|
|
106
111
|
scan: _FileScan = _FileScan(
|
|
107
|
-
set(), set(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), 0, not path.is_file(), path
|
|
112
|
+
set(), set(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), 0, not path.is_file(), path
|
|
108
113
|
)
|
|
109
114
|
if scan.missing:
|
|
110
115
|
return scan
|
|
@@ -162,6 +167,19 @@ def _scan_ndjson(path: Path, *, edge: bool) -> _FileScan:
|
|
|
162
167
|
slot_value: object = record.get(slot)
|
|
163
168
|
if slot_value is None or (isinstance(slot_value, str) and not slot_value.strip()):
|
|
164
169
|
scan.incomplete[slot] += 1
|
|
170
|
+
# A self-referential edge states that a concept relates to itself. In a
|
|
171
|
+
# table-derived graph that is almost never the claim the author made: it is
|
|
172
|
+
# the signature of a resolution collapse, where two differently worded cells
|
|
173
|
+
# (or one cell duplicated into both slots) land on the same CURIE -- the same
|
|
174
|
+
# failure mode that makes two rows share an edge id. Such an edge carries no
|
|
175
|
+
# relational information, so downstream consumers either drop it silently or
|
|
176
|
+
# count it as evidence. Only a non-empty string pair is flagged; absent or
|
|
177
|
+
# blank slots are already reported by ``incomplete-edges``, and a blank-vs-blank
|
|
178
|
+
# pair must not read as "the same CURIE".
|
|
179
|
+
subject: object = record.get("subject")
|
|
180
|
+
object_: object = record.get("object")
|
|
181
|
+
if isinstance(subject, str) and isinstance(object_, str) and subject == object_ and subject.strip():
|
|
182
|
+
scan.self_loops[subject] += 1
|
|
165
183
|
# Bare ``biolink:Association`` means the row lost every specific class:
|
|
166
184
|
# ``lib.edge_category`` derives a class from the (subject role, object
|
|
167
185
|
# role) pair -- optionally pinned per object category by the section's
|
|
@@ -215,7 +233,8 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
|
|
|
215
233
|
Streams both files once each and checks: duplicate node ids, duplicate edge
|
|
216
234
|
ids, nodes with no
|
|
217
235
|
name or an empty name, nodes with no id or an empty id, edges missing any
|
|
218
|
-
of ``subject``/``predicate``/``object``,
|
|
236
|
+
of ``subject``/``predicate``/``object``, edges whose ``subject`` and ``object`` are
|
|
237
|
+
the same CURIE (self-referential), nodes referenced by edges but
|
|
219
238
|
never declared (``undeclared``), declared nodes participating in no edge
|
|
220
239
|
(``isolated``), empty/malformed lines, string values carrying leading/trailing
|
|
221
240
|
whitespace, and null or empty values in any field (a stronger contract than
|
|
@@ -229,6 +248,15 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
|
|
|
229
248
|
``has_supporting_studies``. Every check is an assertion -- the caller decides
|
|
230
249
|
whether violations fail the build.
|
|
231
250
|
|
|
251
|
+
A self-referential edge (``subject`` and ``object`` resolved to one CURIE) is an
|
|
252
|
+
assertion that a concept relates to itself. Table-derived graphs produce them almost
|
|
253
|
+
always by accident: two differently worded cells collapsing onto the same identifier,
|
|
254
|
+
or one cell filling both statement slots, so the edge carries no relational content and
|
|
255
|
+
ships as noise a consumer has to filter. QC therefore reports it rather than quietly
|
|
256
|
+
deleting it, so the author sees which CURIEs collapsed and fixes the mapping (or drops
|
|
257
|
+
the row) in the config. Intentionally reflexive assertions are a known accepted caveat:
|
|
258
|
+
they fail a ``--qc`` build like every other violation.
|
|
259
|
+
|
|
232
260
|
Args:
|
|
233
261
|
nodes_path: Path to ``<name>_<version>.nodes.ndjson``.
|
|
234
262
|
edges_path: Path to ``<name>_<version>.edges.ndjson``.
|
|
@@ -270,6 +298,9 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
|
|
|
270
298
|
if edges.incomplete:
|
|
271
299
|
examples = [f"{slot} ({n})" for slot, n in edges.incomplete.most_common(example_limit)]
|
|
272
300
|
violations.append(StudyViolation("incomplete-edges", "edges", sum(edges.incomplete.values()), examples))
|
|
301
|
+
if edges.self_loops:
|
|
302
|
+
examples = [f"{curie} ({n})" for curie, n in edges.self_loops.most_common(example_limit)]
|
|
303
|
+
violations.append(StudyViolation("self-loop-edges", "edges", sum(edges.self_loops.values()), examples))
|
|
273
304
|
if category_override_declared and edges.demoted:
|
|
274
305
|
examples = [f"{predicate} ({n})" for predicate, n in edges.demoted.most_common(example_limit)]
|
|
275
306
|
violations.append(StudyViolation("demoted-edges", "edges", sum(edges.demoted.values()), examples))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|