tablassert 22.0.1__tar.gz → 22.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {tablassert-22.0.1 → tablassert-22.2.0}/PKG-INFO +1 -1
  2. {tablassert-22.0.1 → tablassert-22.2.0}/pyproject.toml +1 -1
  3. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/biolink.py +8 -4
  4. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/enums.py +4 -0
  5. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/errors.py +14 -0
  6. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/fullmap.py +113 -6
  7. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/lib.py +201 -11
  8. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/models.py +179 -4
  9. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/runcache.py +3 -0
  10. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/study.py +37 -6
  11. {tablassert-22.0.1 → tablassert-22.2.0}/LICENSE +0 -0
  12. {tablassert-22.0.1 → tablassert-22.2.0}/README.md +0 -0
  13. {tablassert-22.0.1 → tablassert-22.2.0}/rust/Cargo.lock +0 -0
  14. {tablassert-22.0.1 → tablassert-22.2.0}/rust/Cargo.toml +0 -0
  15. {tablassert-22.0.1 → tablassert-22.2.0}/rust/examples/count_tables.rs +0 -0
  16. {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/fullmap.rs +0 -0
  17. {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/json.rs +0 -0
  18. {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/lib.rs +0 -0
  19. {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/ndjson.rs +0 -0
  20. {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/nlp.rs +0 -0
  21. {tablassert-22.0.1 → tablassert-22.2.0}/rust/src/uuid.rs +0 -0
  22. {tablassert-22.0.1 → tablassert-22.2.0}/rust/tests/build_golden.rs +0 -0
  23. {tablassert-22.0.1 → tablassert-22.2.0}/rust/tests/common/mod.rs +0 -0
  24. {tablassert-22.0.1 → tablassert-22.2.0}/rust/tests/extract_prebuilt.rs +0 -0
  25. {tablassert-22.0.1 → tablassert-22.2.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
  26. {tablassert-22.0.1 → tablassert-22.2.0}/rust/tests/nlp_golden.rs +0 -0
  27. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/__init__.py +0 -0
  28. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/_lazy.py +0 -0
  29. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/cli.py +0 -0
  30. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/cli_entry.py +0 -0
  31. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/coerce.py +0 -0
  32. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
  33. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/data/infores_catalog.yaml +0 -0
  34. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/extras.py +0 -0
  35. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/infores_registry.py +0 -0
  36. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/ingests.py +0 -0
  37. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/log.py +0 -0
  38. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/net.py +0 -0
  39. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/nlp.py +0 -0
  40. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/progress.py +0 -0
  41. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/qc.py +0 -0
  42. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/rig.py +0 -0
  43. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/rs.pyi +0 -0
  44. {tablassert-22.0.1 → tablassert-22.2.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 22.0.1
3
+ Version: 22.2.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "22.0.1"
3
+ version = "22.2.0"
4
4
  description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -582,13 +582,17 @@ description dump.
582
582
  """
583
583
 
584
584
 
585
- DISABLED_EDGE_FIELDS: frozenset[str] = frozenset({"species_context_qualifier"})
586
- """Edge fields Tablassert intentionally never emits or accepts.
585
+ DISABLED_EDGE_FIELDS: frozenset[str] = frozenset()
586
+ """Edge fields Tablassert intentionally never emits or accepts, whatever the model says.
587
587
 
588
588
  This policy is separate from :data:`UNSATISFIABLE_EDGE_FIELDS`: the latter tracks
589
589
  Biolink Model attachment and may change with a dependency release, while this set is
590
- a stable Tablassert product decision. Keeping it separate prevents a future Biolink
591
- release from making ``species_context_qualifier`` silently emittable again.
590
+ a stable Tablassert product decision. It is currently empty: the one entry it ever
591
+ held, ``species_context_qualifier``, was re-enabled as a declarable, CURIE-ranged
592
+ qualifier (Biolink Model 4.4.6 declares the slot on the root ``association`` class,
593
+ range ``organism taxon``) after its old node-taxonomy auto-derivation was removed; no
594
+ derivation returns, so the field only appears when a config declares it. The empty
595
+ set keeps the policy mechanism alive for future product decisions.
592
596
  """
593
597
 
594
598
 
@@ -42,6 +42,10 @@ class Comparisons(str, Enum):
42
42
  LE = "le"
43
43
  EQ = "eq"
44
44
  NE = "ne"
45
+ # Membership comparisons: the comparator is a list of identifiers, matched against the
46
+ # raw (pre-resolution) cell value. JOIN keeps matching rows, ANTIJOIN drops them.
47
+ JOIN = "join"
48
+ ANTIJOIN = "antijoin"
45
49
 
46
50
 
47
51
  class Functions(str, Enum):
@@ -47,9 +47,23 @@ TablassertErrorCodes = Literal[
47
47
  "babel-download-failed",
48
48
  "source-file-unreadable",
49
49
  "resolve-bad-specs",
50
+ "taxon-from-conflict",
51
+ "taxon-from-unknown-column",
52
+ "taxon-from-self",
53
+ "taxon-from-cycle",
54
+ "taxon-from-unresolved-column",
55
+ "taxon-from-not-ncbitaxon",
50
56
  "config-rows-and-row-slice-conflict",
51
57
  "comparison-bad-comparator-type",
52
58
  "comparison-nonnumeric-comparator",
59
+ # Membership (join/antijoin) family: the first two are config-validation failures on the
60
+ # `comparator` field, the last two are op-level failures on the column being tested.
61
+ "comparison-membership-comparator-required",
62
+ "comparison-membership-empty",
63
+ "comparison-membership-member-not-a-string",
64
+ "reindex-membership-column-missing",
65
+ "reindex-membership-column-dtype-unsupported",
66
+ "comparison-regex-requires-eq-ne",
53
67
  "encoding-bad-excel-column",
54
68
  "regex-bad-pattern",
55
69
  "regex-bad-replacement",
@@ -1,5 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import re
3
4
  import time
4
5
  from collections import OrderedDict
5
6
  from collections.abc import Callable
@@ -575,7 +576,9 @@ def join_matches(lf: pl.LazyFrame, col: str, matches: pl.DataFrame, tag: str = "
575
576
  return _join_matches_eager(lf.collect(), col, matches, tag, drop_unresolved).lazy()
576
577
 
577
578
 
578
- def _join_matches_eager(df: pl.DataFrame, col: str, matches: pl.DataFrame, tag: str = "_two", drop_unresolved: bool = True) -> pl.DataFrame:
579
+ def _join_matches_eager(
580
+ df: pl.DataFrame, col: str, matches: pl.DataFrame, tag: str = "_two", drop_unresolved: bool = True, taxon_key: str | None = None
581
+ ) -> pl.DataFrame:
579
582
  """Eager core of ``join_matches``: join ranked matches into an in-memory frame.
580
583
 
581
584
  Args:
@@ -585,6 +588,10 @@ def _join_matches_eager(df: pl.DataFrame, col: str, matches: pl.DataFrame, tag:
585
588
  tag: Suffix used to derive the level-two column name.
586
589
  drop_unresolved: When True rows whose ``col`` did not match are dropped;
587
590
  when False the row is kept and the resolved columns stay null.
591
+ taxon_key: Optional join-key column present on BOTH sides (``df`` and
592
+ ``matches``) that must match in addition to the term, used by per-row
593
+ taxon scoping (``ResolveSpec.taxon_from``) so one term's matches from
594
+ different taxon scopes never leak across rows. Dropped from the output.
588
595
 
589
596
  Returns:
590
597
  DataFrame with resolved columns. Tied best-ranked CURIEs are returned as
@@ -594,14 +601,25 @@ def _join_matches_eager(df: pl.DataFrame, col: str, matches: pl.DataFrame, tag:
594
601
  l1: str = col
595
602
  l2: str = l1 + tag
596
603
 
597
- result: pl.DataFrame = df.join(matches.filter(pl.col("NLP_LEVEL").eq(1)), left_on=l1, right_on="term", how="left", suffix="_l1")
598
-
599
- l2_matches: pl.DataFrame = matches.filter(pl.col("NLP_LEVEL").eq(2))
600
- result = result.join(l2_matches, left_on=l2, right_on="term", how="left", suffix="_l2")
604
+ if taxon_key is not None:
605
+ result: pl.DataFrame = df.join(
606
+ matches.filter(pl.col("NLP_LEVEL").eq(1)), left_on=[l1, taxon_key], right_on=["term", taxon_key], how="left", suffix="_l1"
607
+ )
608
+ l2_matches: pl.DataFrame = matches.filter(pl.col("NLP_LEVEL").eq(2))
609
+ result = result.join(l2_matches, left_on=[l2, taxon_key], right_on=["term", taxon_key], how="left", suffix="_l2")
610
+ else:
611
+ result = df.join(matches.filter(pl.col("NLP_LEVEL").eq(1)), left_on=l1, right_on="term", how="left", suffix="_l1")
612
+ l2_matches = matches.filter(pl.col("NLP_LEVEL").eq(2))
613
+ result = result.join(l2_matches, left_on=l2, right_on="term", how="left", suffix="_l2")
601
614
 
602
615
  result = result.with_columns([_coalesce_expr(col, suffix, base, prefix, cast_str) for suffix, base, prefix, cast_str in _JOIN_COALESCE_SPECS])
603
616
 
604
- result = result.select(pl.exclude(r"^(CURIE|PREFERRED_NAME|CATEGORY_NAME|TAXON_ID|SOURCE_NAME|SOURCE_VERSION|NLP_LEVEL|PR|FREQUENCY)(_l2)?$"))
617
+ exclude: str = r"^(CURIE|PREFERRED_NAME|CATEGORY_NAME|TAXON_ID|SOURCE_NAME|SOURCE_VERSION|NLP_LEVEL|PR|FREQUENCY)(_l2)?$"
618
+ if taxon_key is not None:
619
+ exclude = (
620
+ r"^(CURIE|PREFERRED_NAME|CATEGORY_NAME|TAXON_ID|SOURCE_NAME|SOURCE_VERSION|NLP_LEVEL|PR|FREQUENCY|" + re.escape(taxon_key) + r")(_l2)?$"
621
+ )
622
+ result = result.select(pl.exclude(exclude))
605
623
  result = result.select(pl.exclude(col + tag))
606
624
  result = result.with_columns(pl.col(f"{col}_taxon").replace("NCBITaxon:0", None))
607
625
  if drop_unresolved:
@@ -627,6 +645,89 @@ class ResolveSpec(NamedTuple):
627
645
  Set only for ``nullable`` qualifiers; subject/object always resolve strict so an
628
646
  edge with a missing node is dropped, never emitted.
629
647
  """
648
+ taxon_from: str | None = None
649
+ """When set, per-row taxon scoping from another resolved column of the same frame.
650
+
651
+ Names a node column that an earlier ``resolve_batch`` pass already resolved: its
652
+ ``<col>_taxon`` metadata (an ``NCBITaxon:<id>`` string) scopes this column's
653
+ ``filter_and_rank`` taxon filter row by row instead of the static ``taxon``.
654
+ Rows whose taxon cell is blank fall back to the 9606 default; a non-null cell
655
+ whose resolved taxon is not an NCBITaxon id fails loudly rather than silently
656
+ resolving everything under one species.
657
+ """
658
+
659
+
660
+ _TAXON_SCOPE: str = "__taxon_scope"
661
+ """Helper column carrying each row's per-row taxon scope during ``resolve_batch``.
662
+
663
+ Filled by :func:`_per_row_taxon_matches` from the ``taxon_from`` column's resolved
664
+ metadata (an NCBITaxon id string, or the ``9606`` fallback for blank taxon cells),
665
+ used as the second join key so one term's matches from different taxon scopes never
666
+ leak across rows, and dropped before the frame leaves the op.
667
+ """
668
+
669
+
670
+ def _per_row_taxon_matches(
671
+ result: pl.DataFrame, spec: ResolveSpec, raw: pl.DataFrame, tag: str, column_context: bool
672
+ ) -> tuple[pl.DataFrame, pl.DataFrame]:
673
+ """Scope this spec's matches per row by the ``taxon_from`` column's resolved taxon.
674
+
675
+ The ``taxon_from`` column was resolved by an earlier ``resolve_batch`` pass, so
676
+ ``result`` already carries its resolved CURIE (``<col>``) and its taxon metadata
677
+ (``<col>_taxon``, ``NCBITaxon:<id>`` or null for a non-taxon-bearing concept).
678
+ Rows are bucketed by that scope; each bucket's terms go through the same
679
+ ``filter_and_rank`` a static-taxon spec uses, keyed by the scope so the join back
680
+ picks each row's own bucket. Blank taxon cells fall back to the 9606 default; a
681
+ non-null cell whose taxon is not an NCBITaxon id fails the build instead of
682
+ silently resolving everything under one species.
683
+
684
+ Returns:
685
+ The frame with ``_TAXON_SCOPE`` appended (join key; the caller drops it after
686
+ the join) and the concatenated per-scope matches carrying the same key.
687
+
688
+ Raises:
689
+ TablassertError: ``taxon-from-unresolved-column`` when the taxon source's
690
+ resolved or metadata column is absent (the earlier pass never ran);
691
+ ``taxon-from-not-ncbitaxon`` when a non-null taxon cell resolves without
692
+ an NCBITaxon taxon id.
693
+ """
694
+ source: str = spec.taxon_from or ""
695
+ meta_col: str = f"{source}_taxon"
696
+ missing: list[str] = [c for c in (source, meta_col) if c not in result.columns]
697
+ if missing:
698
+ raise TablassertError(
699
+ f"ResolveSpec for {spec.col!r} sets taxon_from={source!r} but the frame lacks {', '.join(repr(c) for c in missing)}; "
700
+ "the named column must be resolved by an earlier resolve_batch pass in the same pipeline.",
701
+ code="taxon-from-unresolved-column",
702
+ )
703
+ value: pl.Series = result.select(pl.col(source).cast(pl.String).str.strip_chars().alias("v"))["v"]
704
+ meta: pl.Series = result.select(pl.col(meta_col).cast(pl.String).str.strip_chars().alias("m"))["m"]
705
+ has_value: pl.Series = value.is_not_null() & (value != "")
706
+ is_ncbitaxon: pl.Series = meta.str.contains(r"^NCBITaxon:\d+$").fill_null(False)
707
+ bad: pl.Series = has_value & ~is_ncbitaxon
708
+ if bad.any():
709
+ offenders: list[str] = sorted({f"{v!r} -> {m!r}" for v, m in zip(value.filter(bad).to_list(), meta.filter(bad).to_list(), strict=True)})[:5]
710
+ raise TablassertError(
711
+ f"taxon_from column {source!r} carries value(s) that did not resolve to an NCBITaxon taxon: "
712
+ f"{', '.join(offenders)}. Fix the source table so every non-blank taxon cell names a species, "
713
+ "or the row's entity resolution cannot be scoped.",
714
+ code="taxon-from-not-ncbitaxon",
715
+ )
716
+ scope: pl.Expr = (
717
+ pl.when(has_value & is_ncbitaxon)
718
+ .then(pl.col(meta_col).cast(pl.String).str.strip_chars().str.replace("NCBITaxon:", ""))
719
+ .otherwise(pl.lit("9606"))
720
+ )
721
+ result = result.with_columns(scope.alias(_TAXON_SCOPE))
722
+ parts: list[pl.DataFrame] = []
723
+ for taxon in sorted(set(result[_TAXON_SCOPE].to_list())):
724
+ bucket: pl.DataFrame = result.filter(pl.col(_TAXON_SCOPE) == taxon)
725
+ bucket_terms: pl.DataFrame = distinct(bucket.lazy(), spec.col, spec.col + tag).collect()
726
+ bucket_matches: pl.DataFrame = filter_and_rank(
727
+ raw, bucket_terms, taxon, spec.prioritize, spec.avoid, column_context, spec.exclude_prefixes, spec.exclude_regex
728
+ )
729
+ parts.append(bucket_matches.with_columns(pl.lit(taxon).alias(_TAXON_SCOPE)))
730
+ return result, (pl.concat(parts) if parts else empty_matches(column_context))
630
731
 
631
732
 
632
733
  def resolve_batch(
@@ -721,6 +822,12 @@ def resolve_batch(
721
822
  for spec in specs:
722
823
  if on_phase is not None:
723
824
  on_phase(f"resolve:{spec.col}")
825
+ if spec.taxon_from is not None:
826
+ scoped_result, matches = _per_row_taxon_matches(result, spec, raw, tag, column_context)
827
+ if log:
828
+ log_unmatched(spec.col, terms_by_col[spec.col].lazy(), matches, section_hash, config_file)
829
+ result = _join_matches_eager(scoped_result, spec.col, matches, tag, drop_unresolved=not spec.nullable, taxon_key=_TAXON_SCOPE)
830
+ continue
724
831
  terms_df: pl.DataFrame = terms_by_col[spec.col]
725
832
  matches: pl.DataFrame = filter_and_rank(
726
833
  raw, terms_df, spec.taxon, spec.prioritize, spec.avoid, column_context, spec.exclude_prefixes, spec.exclude_regex
@@ -45,11 +45,11 @@ from tablassert.coerce import (
45
45
  sig,
46
46
  study_size_target,
47
47
  )
48
- from tablassert.enums import EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
48
+ from tablassert.enums import Comparisons, EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
49
49
  from tablassert.errors import TablassertError
50
50
  from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
51
51
  from tablassert.log import cat
52
- from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
52
+ from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, Reindex, RIGConfig, Section
53
53
  from tablassert.nlp import level_one, level_two
54
54
  from tablassert.qc import fullmap_audit
55
55
  from tablassert.rig import (
@@ -1070,6 +1070,178 @@ def reindex(df: pl.LazyFrame, col: str, op: Callable, comp: str | int | float, c
1070
1070
  return df.filter(op(expr, comp))
1071
1071
 
1072
1072
 
1073
+ MEMBERSHIP_KEY_COLUMN: str = "_reindex_membership_key"
1074
+ """Base name for the throwaway join key column on the cast branch; extended with underscores until
1075
+ it is unused if a source frame already carries it, and dropped again before the op returns."""
1076
+
1077
+ MEMBERSHIP_UNCASTABLE_DTYPES: frozenset[str] = frozenset({"List", "Array", "Struct", "Object", "Binary"})
1078
+ """polars dtype class NAMES whose cells have no single-string identity, so membership is refused.
1079
+
1080
+ Matched by name because a module-level ``pl.List`` reference would resolve the lazy proxy and import
1081
+ polars at ``import tablassert.lib``, which is exactly what :mod:`tablassert._lazy` exists to avoid.
1082
+ Every parametrized variant reports its base name (``List(String)`` is named ``List``).
1083
+
1084
+ A ``List``/``Array`` cell cannot be cast to ``String`` at all (``InvalidOperationError``), an
1085
+ ``Object`` cell casts through python and never compares equal to an identifier, a ``Struct`` casts to
1086
+ a brace literal no identifier is spelled to match, and a ``Binary`` cell casts only when it happens to
1087
+ be valid UTF-8, so one list would match some rows and raise on the rest. Joining list cells on a
1088
+ separator instead would make a multi-value cell never match a single identifier, a silent wrong
1089
+ answer, so these dtypes raise a coded error.
1090
+ """
1091
+
1092
+
1093
+ def _membership_key_column(schema: pl.Schema) -> str:
1094
+ """Return a join key column name the frame does not already use.
1095
+
1096
+ Args:
1097
+ schema: Collected schema of the frame being filtered.
1098
+
1099
+ Returns:
1100
+ :data:`MEMBERSHIP_KEY_COLUMN`, or that name extended with underscores until it is free.
1101
+ """
1102
+ key_col: str = MEMBERSHIP_KEY_COLUMN
1103
+ while key_col in schema:
1104
+ key_col = f"{key_col}_"
1105
+ return key_col
1106
+
1107
+
1108
+ def reindex_membership(df: pl.LazyFrame, col: str, keys: list[str], anti: bool = False) -> pl.LazyFrame:
1109
+ """Reindex a LazyFrame by exact membership in a list of identifiers (join / antijoin).
1110
+
1111
+ Args:
1112
+ df: Source LazyFrame.
1113
+ col: Column whose raw (pre-resolution) cell values are tested.
1114
+ keys: Identifier list. Duplicates are ignored: a semi/anti join cannot multiply left
1115
+ rows, and the join build side is cheaper once deduplicated.
1116
+ anti: ``False`` (join) keeps rows whose value is in ``keys``; ``True`` (antijoin) keeps
1117
+ the rows that are not.
1118
+
1119
+ Returns:
1120
+ Filtered LazyFrame keeping the surviving rows in left-frame order, with every original
1121
+ column and its original dtype.
1122
+
1123
+ Raises:
1124
+ TablassertError: ``comparison-membership-empty`` for an empty list,
1125
+ ``reindex-membership-column-missing`` when the source lacks ``col``, and
1126
+ ``reindex-membership-column-dtype-unsupported`` for cells with no single string form
1127
+ (``List``, ``Array``, ``Struct``, ``Object``, ``Binary``).
1128
+
1129
+ Notes:
1130
+ Why a join and not ``is_in``: on the pinned polars (``>=2.0.0rc2``) a semi/anti hash join
1131
+ against a once-built key frame is the faster measured path once the list is non-trivial (1M
1132
+ rows, median of three collects: 100k keys 0.018s vs 0.029s filtered; 1k keys 0.003s vs
1133
+ 0.004s; a tie at ten keys), and it is the membership idiom already used in
1134
+ :mod:`tablassert.qc`. No ``is_in`` fast path exists for tiny lists: it propagates nulls
1135
+ differently from antijoin, and the gap is milliseconds per million rows.
1136
+
1137
+ ``maintain_order="left"`` is explicit, not incidental: polars documents the default order as
1138
+ unspecified ("might differ across Polars versions or even between different runs"), and the
1139
+ dependency is an open range, so a future 2.x must not be able to reshuffle a section's rows.
1140
+ Measured free here (1M rows / 100k keys, five runs, median): semi 5.1ms -> 4.6ms, anti 5.4ms
1141
+ -> 5.4ms.
1142
+
1143
+ The string path joins straight on ``col``, because semi/anti joins emit only the left frame's
1144
+ columns. Only the cast path adds a throwaway helper column, and it is dropped again, so
1145
+ entity resolution downstream sees the cell exactly as the source produced it.
1146
+
1147
+ Null semantics are join semantics and deliberately differ from ``ne``: a null cell never
1148
+ matches, so join drops it while antijoin KEEPS it.
1149
+
1150
+ Membership is textual, so an identifier must be spelled the way polars' cast renders it.
1151
+ Measured forms: ``Int64`` 3 -> ``"3"``; ``Float64`` 3.0 -> ``"3.0"`` (1e+20 keeps scientific
1152
+ form); ``Bool`` -> ``"true"``/``"false"``; ``Date`` -> ``"2024-01-15"``; ``Datetime`` ->
1153
+ ``"2024-01-15 06:24:30.000000"`` (space, not ``T``, with microsecond precision); ``Time`` ->
1154
+ ``"01:02:03"``; ``Decimal(10,2)`` 1.5 -> ``"1.50"`` (scale kept). A misspelled list is not an
1155
+ error: join emits an empty section and antijoin emits every row. No zero-match warning is
1156
+ emitted, because detecting it would cost an extra pass over the frame on every build.
1157
+ """
1158
+ if not keys:
1159
+ raise TablassertError("`join`/`antijoin` reindex requires at least one identifier.", code="comparison-membership-empty")
1160
+
1161
+ schema: pl.Schema = df.collect_schema()
1162
+ if col not in schema:
1163
+ raise TablassertError(f"`join`/`antijoin` reindex targets `{col}`, which the source does not have.", code="reindex-membership-column-missing")
1164
+ if type(schema[col]).__name__ in MEMBERSHIP_UNCASTABLE_DTYPES:
1165
+ raise TablassertError(
1166
+ f"`join`/`antijoin` reindex cannot test `{col}`: {schema[col]} cells have no single string form to match identifiers against.",
1167
+ code="reindex-membership-column-dtype-unsupported",
1168
+ )
1169
+
1170
+ unique: list[str] = list(dict.fromkeys(keys))
1171
+ how: Literal["semi", "anti"] = "anti" if anti else "semi"
1172
+ if isinstance(schema[col], pl.String):
1173
+ # No helper column: the join returns the left frame's columns unchanged.
1174
+ return df.join(pl.DataFrame({col: unique}).lazy(), on=col, how=how, maintain_order="left")
1175
+ key_col: str = _membership_key_column(schema)
1176
+ left: pl.LazyFrame = df.with_columns(pl.col(col).cast(pl.String, strict=False).alias(key_col))
1177
+ return left.join(pl.DataFrame({key_col: unique}).lazy(), on=key_col, how=how, maintain_order="left").drop(key_col)
1178
+
1179
+
1180
+ def reindex_regex(df: pl.LazyFrame, col: str, pattern: str, negate: bool = False) -> pl.LazyFrame:
1181
+ """Reindex a LazyFrame by filtering rows on a polars regex match against a column.
1182
+
1183
+ Args:
1184
+ df: Source LazyFrame.
1185
+ col: Column to test.
1186
+ pattern: Polars-compatible regex, matched unanchored and case-sensitively (write `^`/`$`
1187
+ to pin a whole cell).
1188
+ negate: When True keep the rows that do NOT match (the `ne` filter); when False keep the
1189
+ matches (the `eq` filter).
1190
+
1191
+ Returns:
1192
+ LazyFrame containing only rows whose predicate is true.
1193
+
1194
+ Notes:
1195
+ The column is cast to String first, so a typed source (parquet, or a numeric column in a
1196
+ text table) filters on its text form instead of raising polars' ``InvalidOperationError:
1197
+ expected String type``. A null cell matches nothing and ``not_()`` leaves it null, so null
1198
+ rows drop under both directions -- the same null rule literal `eq`/`ne` already have, since
1199
+ a null comparison is never true.
1200
+ """
1201
+ match: pl.Expr = pl.col(col).cast(pl.String).str.contains(pattern)
1202
+ return df.filter(match.not_() if negate else match)
1203
+
1204
+
1205
+ def reindex_op(entry: Reindex) -> tuple[Callable, tuple[Any, ...]]:
1206
+ """Compile one ``Reindex`` config entry into its Tcode op tuple.
1207
+
1208
+ Args:
1209
+ entry: Validated ``Reindex`` model. The validator guarantees the pairing: a membership entry
1210
+ carries a non-empty ``list[str]`` comparator, ``regex`` only ever sits on ``eq``/``ne``,
1211
+ and a scalar entry carries a scalar comparator.
1212
+
1213
+ Returns:
1214
+ ``(callable, args)`` for the pipeline: :func:`reindex_membership` for ``join``/``antijoin``,
1215
+ :func:`reindex_regex` for a ``regex`` entry, otherwise :func:`reindex` with the ``operator``
1216
+ module callable and the same ``cast`` choice it has always used (numeric comparisons cast to
1217
+ Float64, ``eq``/``ne`` do not).
1218
+
1219
+ Notes:
1220
+ Legacy entries keep emitting exactly the tuples they always did, so their run-cache op digests
1221
+ do not move; membership is the only new shape.
1222
+
1223
+ ``Reindex.comparison`` carries the plain enum *value* at runtime because ``TablaBase`` sets
1224
+ ``use_enum_values=True``, so this branches on the strings, exactly as the inline comprehension
1225
+ it replaces did.
1226
+ """
1227
+ col: str = idxname(entry.column)
1228
+ comparison: str = entry.comparison
1229
+ if comparison in (Comparisons.JOIN.value, Comparisons.ANTIJOIN.value):
1230
+ if not isinstance(entry.comparator, list):
1231
+ raise TablassertError(
1232
+ f"`{comparison}` reindex requires a list of identifiers, got {type(entry.comparator).__name__}.",
1233
+ code="comparison-membership-comparator-required",
1234
+ )
1235
+ return (reindex_membership, (col, entry.comparator, comparison == Comparisons.ANTIJOIN.value))
1236
+ if entry.regex:
1237
+ # models.Reindex rejects `regex` off eq/ne, so this arm never needs the numeric cast or an
1238
+ # operator lookup.
1239
+ return (reindex_regex, (col, entry.comparator, comparison == Comparisons.NE.value))
1240
+ if comparison in (Comparisons.NE.value, Comparisons.EQ.value):
1241
+ return (reindex, (col, getattr(operator, comparison), entry.comparator, False))
1242
+ return (reindex, (col, getattr(operator, comparison), entry.comparator))
1243
+
1244
+
1073
1245
  def drop_not_significant(lf: pl.LazyFrame, col: str = "statistical_significance_qualifier") -> pl.LazyFrame:
1074
1246
  """Drop release-mode edges whose significance qualifier is ``not_significant``.
1075
1247
 
@@ -1338,14 +1510,7 @@ class Tcode(Section):
1338
1510
  (idx, ()),
1339
1511
  (crop, (self.source.row_slice,)) if self.source.row_slice else None,
1340
1512
  (pick, (self.source.rows,)) if self.source.rows else None,
1341
- [
1342
- (reindex, (idxname(x.column), getattr(operator, x.comparison), x.comparator))
1343
- if x.comparison not in ["ne", "eq"]
1344
- else (reindex, (idxname(x.column), getattr(operator, x.comparison), x.comparator, False))
1345
- for x in self.source.reindex
1346
- ]
1347
- if self.source.reindex
1348
- else None,
1513
+ [reindex_op(x) for x in self.source.reindex] if self.source.reindex else None,
1349
1514
  # --head preview: randomly sample min(HEAD_ROWS, height) rows before any encoding/resolve.
1350
1515
  (head, (HEAD_ROWS,)) if self.head else None,
1351
1516
  [
@@ -1401,9 +1566,32 @@ class Tcode(Section):
1401
1566
  x.exclude_prefixes,
1402
1567
  x.exclude_regex,
1403
1568
  x.nullable if isinstance(x, Qualifier) else False,
1569
+ x.taxon_from,
1404
1570
  )
1405
1571
  for x, col in node_columns
1406
1572
  ]
1573
+ # Per-row taxon scoping (ResolveSpec.taxon_from): a scoped column reads the
1574
+ # taxon metadata its source column's resolution emits, so sources must resolve
1575
+ # first. Layer the specs by dependency (config validation has already rejected
1576
+ # cycles and unknown targets); with no taxon_from anywhere this is one batch
1577
+ # and the op list is exactly what it always was.
1578
+ spec_cols: set[str] = {spec.col for spec in specs}
1579
+ batches: list[list[ResolveSpec]] = []
1580
+ placed: set[str] = set()
1581
+ pending: list[ResolveSpec] = list(specs)
1582
+ while pending:
1583
+ ready: list[ResolveSpec] = [s for s in pending if s.taxon_from is None or s.taxon_from in placed or s.taxon_from not in spec_cols]
1584
+ if not ready:
1585
+ raise TablassertError(
1586
+ f"taxon_from references form a cycle among {', '.join(sorted(s.col for s in pending))}; "
1587
+ "no column can resolve first. This is rejected at config time, so a build reaching here means the "
1588
+ "section was assembled programmatically -- fix the spec layering.",
1589
+ code="taxon-from-cycle",
1590
+ )
1591
+ batches.append(ready)
1592
+ placed.update(s.col for s in ready)
1593
+ pending = [s for s in pending if s.col not in placed]
1594
+ resolve_ops: list[Any] = [(resolve_batch, (batch, db, self.log, self.store.stem, self.config.name, True, "_two")) for batch in batches]
1407
1595
  return [
1408
1596
  [self.node_prep(x, col) for x, col in node_columns],
1409
1597
  # Encode only: no pre-resolution snapshot and no NLP normalization, both of
@@ -1415,7 +1603,7 @@ class Tcode(Section):
1415
1603
  # copied them into named slots; trim before resolution so the frame that
1416
1604
  # resolve_batch materializes and joins stays narrow.
1417
1605
  (trim, ()),
1418
- (resolve_batch, (specs, db, self.log, self.store.stem, self.config.name, True, "_two")),
1606
+ *resolve_ops,
1419
1607
  # QC audits only the strict columns: a nullable qualifier's nulls are expected
1420
1608
  # (blank cell / no match), not resolution errors for the audit to delete.
1421
1609
  [
@@ -1565,6 +1753,8 @@ PHASE_OF: dict[Callable, str] = {
1565
1753
  crop: "filter",
1566
1754
  pick: "filter",
1567
1755
  reindex: "filter",
1756
+ reindex_membership: "filter",
1757
+ reindex_regex: "filter",
1568
1758
  head: "filter",
1569
1759
  coerce_columns: "clean",
1570
1760
  clean_numeric: "clean",
@@ -119,21 +119,119 @@ class TablaBase(BaseModel):
119
119
  class Reindex(TablaBase):
120
120
  column: str = Field(..., pattern=r"^[A-Z]{1,3}$", description="Source column letters used for row filtering.", examples=["A", "AA"])
121
121
  comparison: Comparisons = Field(
122
- Comparisons.NE, description="Comparison operator used in reindex filtering.", examples=[Comparisons.NE, Comparisons.EQ, Comparisons.GT]
122
+ Comparisons.NE,
123
+ description="Comparison operator used in reindex filtering.",
124
+ examples=[Comparisons.NE, Comparisons.EQ, Comparisons.GT, Comparisons.JOIN, Comparisons.ANTIJOIN],
123
125
  )
124
- comparator: str | int | float = Field(..., description="Right-side value compared against the selected column.", examples=["N/A", 0, 1.5])
126
+ comparator: str | int | float | list[str] = Field(
127
+ ...,
128
+ description=(
129
+ "Right-side value compared against the selected column. For `join`/`antijoin` this is the list of "
130
+ "identifiers matched exactly against the raw (pre-resolution) cell value."
131
+ ),
132
+ examples=["N/A", 0, 1.5, ["CHEBI:41774", "CHEBI:48217"]],
133
+ )
134
+ regex: bool = Field(
135
+ False,
136
+ description=(
137
+ "When true, `comparator` is a polars-compatible regex matched against the cell instead of a literal "
138
+ "value: `eq` keeps rows whose cell matches, `ne` keeps rows that do not. Matching is case-sensitive and "
139
+ "unanchored (add `^`/`$` to pin the whole cell), the column is compared as text, and a null cell never "
140
+ "matches, so null rows drop under both operators. Only valid with `eq`/`ne`; an invalid or empty pattern "
141
+ "is rejected at config time (`regex-bad-pattern`)."
142
+ ),
143
+ )
144
+
145
+ @model_validator(mode="before")
146
+ @classmethod
147
+ def membership_members_are_strings(cls, data: Any) -> Any:
148
+ """Reject an identifier list whose members are not strings, with a code rather than a field type error.
149
+
150
+ Args:
151
+ data: Raw config value for one ``Reindex`` entry.
152
+
153
+ Returns:
154
+ ``data`` unchanged when the entry is not a membership list, or when every member is a
155
+ string.
156
+
157
+ Raises:
158
+ TablassertValidationError: ``comparison-membership-member-not-a-string`` when a
159
+ ``join``/``antijoin`` list holds a non-string member.
160
+
161
+ Notes:
162
+ Without this guard the pydantic union rejects `comparator: [3, 4]` (the natural spelling
163
+ against an `Int64` column, whose string form is `3`) as a bare `string_type` field error
164
+ with no stable code and no docs URL, while every other failure in this family is coded.
165
+ The fix stays "quote the identifiers": membership is textual, so an unquoted number is a
166
+ config error rather than something to coerce silently.
167
+ """
168
+ if not isinstance(data, dict):
169
+ return data
170
+ comparison = data.get("comparison")
171
+ comparator = data.get("comparator")
172
+ if comparison not in (Comparisons.JOIN.value, Comparisons.ANTIJOIN.value) or not isinstance(comparator, list):
173
+ return data
174
+ for index, member in enumerate(comparator):
175
+ if not isinstance(member, str):
176
+ raise TablassertValidationError(
177
+ f"`join`/`antijoin` identifier lists must hold quoted strings: member {index} is {type(member).__name__} ({member!r}). "
178
+ f'Quote it (for example ["{member}"]) to match the cell as text.',
179
+ code="comparison-membership-member-not-a-string",
180
+ )
181
+ return data
125
182
 
126
183
  @model_validator(mode="after")
127
184
  def comparison_datatypes(self: Self) -> Self:
128
185
  x: Comparisons = self.comparison
129
- y: str | int | float = self.comparator
186
+ y: str | int | float | list[str] = self.comparator
130
187
 
131
- if x == Comparisons.NE or x == Comparisons.EQ:
188
+ if x == Comparisons.JOIN or x == Comparisons.ANTIJOIN:
189
+ if self.regex:
190
+ raise TablassertValidationError(
191
+ "`regex` is only meaningful with the `eq`/`ne` comparisons; `join`/`antijoin` match identifiers by exact string identity against a list.",
192
+ code="comparison-regex-requires-eq-ne",
193
+ )
194
+ if not isinstance(y, list):
195
+ raise TablassertValidationError(
196
+ f"`join`/`antijoin` comparisons require a list of identifier strings, got {type(y).__name__}.",
197
+ code="comparison-membership-comparator-required",
198
+ )
199
+ if not y:
200
+ raise TablassertValidationError(
201
+ "`join`/`antijoin` comparators must hold at least one identifier.", code="comparison-membership-empty"
202
+ )
203
+ elif x == Comparisons.NE or x == Comparisons.EQ:
132
204
  if not isinstance(y, str):
133
205
  raise TablassertValidationError(
134
206
  f"`eq`/`ne` comparisons require a str comparator, got {type(y).__name__}.", code="comparison-bad-comparator-type"
135
207
  )
208
+ if self.regex:
209
+ # An empty-or-everything pattern compiles, but it matches EVERY cell, so a `ne` filter
210
+ # would silently drop the whole table. Reject whitespace-only spellings and any pattern
211
+ # that can match the empty string (`^`, `()`, `.*`, `x*`) with the same code the
212
+ # `exclude_regex` probe uses.
213
+ if not y.strip():
214
+ raise TablassertValidationError(
215
+ f"`comparator` must be a non-empty pattern when `regex` is true (an empty pattern matches every cell), got {y!r}.",
216
+ code="regex-bad-pattern",
217
+ )
218
+ try:
219
+ pl.Series([""]).str.contains(y)
220
+ except Exception as e:
221
+ raise TablassertValidationError(
222
+ f"`comparator` must be a polars-compatible regex when `regex` is true, got {y!r}: {e}", code="regex-bad-pattern"
223
+ ) from e
224
+ if pl.Series([""]).str.contains(y).item():
225
+ raise TablassertValidationError(
226
+ f"`comparator` must not match an empty cell when `regex` is true (it would match every cell, so an `ne` filter would drop the whole table), got {y!r}.",
227
+ code="regex-bad-pattern",
228
+ )
136
229
  else:
230
+ if self.regex:
231
+ raise TablassertValidationError(
232
+ "`regex` is only meaningful with the `eq`/`ne` comparisons; ordering comparisons cast the column to Float64 and compare numerically.",
233
+ code="comparison-regex-requires-eq-ne",
234
+ )
137
235
  if not isinstance(y, (int, float)):
138
236
  raise TablassertValidationError(
139
237
  f"Comparisons other than `eq`/`ne` require a float or int comparator, got {type(y).__name__}.",
@@ -307,6 +405,36 @@ class NodeEncoding(Encoding):
307
405
  ),
308
406
  examples=[9606, 10090],
309
407
  )
408
+ taxon_from: str | None = Field(
409
+ None,
410
+ description=(
411
+ "Per-row taxon scoping: name another resolved node column of the same statement (a CURIE-ranged qualifier key, "
412
+ "or `subject`/`object`) whose resolved NCBITaxon CURIE scopes this column's entity resolution row by row, instead of "
413
+ "the static `taxon`. The named column is resolved first; rows whose taxon cell is blank fall back to the 9606 "
414
+ "default, and a non-null cell that does not resolve to an NCBITaxon CURIE fails the build (`taxon-from-not-ncbitaxon`). "
415
+ "Mutually exclusive with an explicit `taxon`."
416
+ ),
417
+ examples=["species_context_qualifier"],
418
+ )
419
+
420
+ @model_validator(mode="before")
421
+ @classmethod
422
+ def taxon_from_excludes_explicit_taxon(cls, data: Any) -> Any:
423
+ """Reject `taxon_from` set together with an explicit `taxon`.
424
+
425
+ Two taxon knobs on one encoding would compete for the same filter, and the
426
+ winner would depend on evaluation order. `taxon` defaults to 9606, so the
427
+ check must run on the raw input: only an author-written `taxon` conflicts,
428
+ and `taxon_from` supplies its own default (9606 for blank taxon cells).
429
+ """
430
+ if isinstance(data, dict) and data.get("taxon_from") is not None and "taxon" in data:
431
+ raise TablassertValidationError(
432
+ "`taxon_from` and an explicit `taxon` are mutually exclusive; remove one. `taxon_from` scopes resolution "
433
+ "per row from the named column and falls back to the 9606 default for blank taxon cells.",
434
+ code="taxon-from-conflict",
435
+ )
436
+ return data
437
+
310
438
  prioritize: list[Categories] | None = Field(
311
439
  None, description="Biolink categories ranked higher during entity resolution.", examples=[[Categories.GENE, Categories.PROTEIN]]
312
440
  )
@@ -484,6 +612,53 @@ class Statement(TablaBase):
484
612
  seen.add(key)
485
613
  return self
486
614
 
615
+ @model_validator(mode="after")
616
+ def validate_taxon_from_targets(self: Self) -> Self:
617
+ """Validate every `taxon_from` reference before a build can run.
618
+
619
+ `taxon_from` scopes one node column's resolution by another resolved node
620
+ column of the same statement, so the target must exist, must itself resolve
621
+ (enum-ranged qualifiers never resolve), must not be the column carrying the
622
+ reference, and must not participate in a reference cycle -- a cyclic pair
623
+ would have no valid resolution order. All four failures are config-time so a
624
+ bad reference dies at load, not hours into a build.
625
+ """
626
+ resolved: dict[str, NodeEncoding | Qualifier] = {
627
+ "subject": self.subject,
628
+ "object": self.object,
629
+ **{str(q.qualifier): q for q in self.qualifiers or [] if q.resolved},
630
+ }
631
+ for name, encoding in resolved.items():
632
+ target: str | None = getattr(encoding, "taxon_from", None)
633
+ if target is None:
634
+ continue
635
+ if target not in resolved:
636
+ raise TablassertValidationError(
637
+ f"`taxon_from: {target}` on `{name}` does not name a resolved node column of this statement; "
638
+ "expected `subject`, `object`, or a CURIE-ranged qualifier key.",
639
+ code="taxon-from-unknown-column",
640
+ )
641
+ if target == name:
642
+ raise TablassertValidationError(
643
+ f"`taxon_from: {target}` on `{name}` references itself; a column cannot be scoped by its own resolution.", code="taxon-from-self"
644
+ )
645
+ # Chase the reference chain from the target: any path back to a column that
646
+ # already carries a reference is a cycle with no valid resolution order.
647
+ seen: set[str] = {name}
648
+ cursor: str = target
649
+ while cursor in resolved:
650
+ if cursor in seen:
651
+ raise TablassertValidationError(
652
+ f"`taxon_from` references form a cycle through `{cursor}`; there is no column that can resolve first.",
653
+ code="taxon-from-cycle",
654
+ )
655
+ seen.add(cursor)
656
+ nxt: str | None = getattr(resolved[cursor], "taxon_from", None)
657
+ if nxt is None:
658
+ break
659
+ cursor = nxt
660
+ return self
661
+
487
662
  @model_validator(mode="after")
488
663
  def warn_when_an_override_is_demoted_by_the_predicate(self: Self) -> Self:
489
664
  """Warn when a pinned association class cannot carry the section predicate.
@@ -258,6 +258,9 @@ OP_COST: dict[Callable, int] = {
258
258
  lib.crop: 1,
259
259
  lib.pick: 1,
260
260
  lib.reindex: 1,
261
+ # Membership filter: one hash join against a small key frame, same single-predicate price.
262
+ lib.reindex_membership: 1,
263
+ lib.reindex_regex: 1,
261
264
  # head samples min(HEAD_ROWS, height) rows, so it needs a height pass before slicing.
262
265
  lib.head: 2,
263
266
  # Encode: each is one lazy expression over a single column.
@@ -20,6 +20,7 @@ _MESSAGES: dict[str, str] = {
20
20
  "unnamed-nodes": "nodes with no name or an empty name",
21
21
  "unidentified-nodes": "nodes with no id or an empty id",
22
22
  "incomplete-edges": "edges missing subject, predicate, or object",
23
+ "self-loop-edges": "edges whose subject and object are the same CURIE",
23
24
  "demoted-edges": "edges demoted to bare biolink:Association despite a declared category_override",
24
25
  "undeclared-nodes": "nodes referenced by edges but not declared in the nodes file",
25
26
  "isolated-nodes": "declared nodes participating in no edge",
@@ -61,6 +62,9 @@ class _FileScan:
61
62
  #: predicate. A demotion is only a violation when the build declared a
62
63
  #: ``category_override`` (see :func:`study_kgx`); the count is collected either way.
63
64
  demoted: Counter[str]
65
+ #: Edges that assert a concept relates to itself, keyed by the shared CURIE. See
66
+ #: :func:`study_kgx` for why a self-referential edge is a violation.
67
+ self_loops: Counter[str]
64
68
  malformed: int
65
69
  missing: bool
66
70
  path: Path
@@ -94,17 +98,18 @@ def _scan_ndjson(path: Path, *, edge: bool) -> _FileScan:
94
98
 
95
99
  Args:
96
100
  path: Path to a ``.nodes.ndjson`` or ``.edges.ndjson`` file.
97
- edge: ``True`` to collect referenced ids from ``subject``/``object`` and
98
- assert the three core edge slots are present; ``False`` to collect
99
- declared node ``id``s and track duplicates, nodes with no name or
100
- an empty name, and nodes with no id or an empty id.
101
+ edge: ``True`` to collect referenced ids from ``subject``/``object``, assert the
102
+ three core edge slots are present, and tally self-referential
103
+ (``subject == object``) edges; ``False`` to collect declared node ``id``s and
104
+ track duplicates, nodes with no name or an empty name, and nodes with no id
105
+ or an empty id.
101
106
 
102
107
  Returns:
103
108
  A :class:`_FileScan`; ``missing`` is set (and nothing else) when the
104
109
  file does not exist, so a typo'd path can never read as a clean pass.
105
110
  """
106
111
  scan: _FileScan = _FileScan(
107
- set(), set(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), 0, not path.is_file(), path
112
+ set(), set(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), 0, not path.is_file(), path
108
113
  )
109
114
  if scan.missing:
110
115
  return scan
@@ -162,6 +167,19 @@ def _scan_ndjson(path: Path, *, edge: bool) -> _FileScan:
162
167
  slot_value: object = record.get(slot)
163
168
  if slot_value is None or (isinstance(slot_value, str) and not slot_value.strip()):
164
169
  scan.incomplete[slot] += 1
170
+ # A self-referential edge states that a concept relates to itself. In a
171
+ # table-derived graph that is almost never the claim the author made: it is
172
+ # the signature of a resolution collapse, where two differently worded cells
173
+ # (or one cell duplicated into both slots) land on the same CURIE -- the same
174
+ # failure mode that makes two rows share an edge id. Such an edge carries no
175
+ # relational information, so downstream consumers either drop it silently or
176
+ # count it as evidence. Only a non-empty string pair is flagged; absent or
177
+ # blank slots are already reported by ``incomplete-edges``, and a blank-vs-blank
178
+ # pair must not read as "the same CURIE".
179
+ subject: object = record.get("subject")
180
+ object_: object = record.get("object")
181
+ if isinstance(subject, str) and isinstance(object_, str) and subject == object_ and subject.strip():
182
+ scan.self_loops[subject] += 1
165
183
  # Bare ``biolink:Association`` means the row lost every specific class:
166
184
  # ``lib.edge_category`` derives a class from the (subject role, object
167
185
  # role) pair -- optionally pinned per object category by the section's
@@ -215,7 +233,8 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
215
233
  Streams both files once each and checks: duplicate node ids, duplicate edge
216
234
  ids, nodes with no
217
235
  name or an empty name, nodes with no id or an empty id, edges missing any
218
- of ``subject``/``predicate``/``object``, nodes referenced by edges but
236
+ of ``subject``/``predicate``/``object``, edges whose ``subject`` and ``object`` are
237
+ the same CURIE (self-referential), nodes referenced by edges but
219
238
  never declared (``undeclared``), declared nodes participating in no edge
220
239
  (``isolated``), empty/malformed lines, string values carrying leading/trailing
221
240
  whitespace, and null or empty values in any field (a stronger contract than
@@ -229,6 +248,15 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
229
248
  ``has_supporting_studies``. Every check is an assertion -- the caller decides
230
249
  whether violations fail the build.
231
250
 
251
+ A self-referential edge (``subject`` and ``object`` resolved to one CURIE) is an
252
+ assertion that a concept relates to itself. Table-derived graphs produce them almost
253
+ always by accident: two differently worded cells collapsing onto the same identifier,
254
+ or one cell filling both statement slots, so the edge carries no relational content and
255
+ ships as noise a consumer has to filter. QC therefore reports it rather than quietly
256
+ deleting it, so the author sees which CURIEs collapsed and fixes the mapping (or drops
257
+ the row) in the config. Intentionally reflexive assertions are a known accepted caveat:
258
+ they fail a ``--qc`` build like every other violation.
259
+
232
260
  Args:
233
261
  nodes_path: Path to ``<name>_<version>.nodes.ndjson``.
234
262
  edges_path: Path to ``<name>_<version>.edges.ndjson``.
@@ -270,6 +298,9 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
270
298
  if edges.incomplete:
271
299
  examples = [f"{slot} ({n})" for slot, n in edges.incomplete.most_common(example_limit)]
272
300
  violations.append(StudyViolation("incomplete-edges", "edges", sum(edges.incomplete.values()), examples))
301
+ if edges.self_loops:
302
+ examples = [f"{curie} ({n})" for curie, n in edges.self_loops.most_common(example_limit)]
303
+ violations.append(StudyViolation("self-loop-edges", "edges", sum(edges.self_loops.values()), examples))
273
304
  if category_override_declared and edges.demoted:
274
305
  examples = [f"{predicate} ({n})" for predicate, n in edges.demoted.most_common(example_limit)]
275
306
  violations.append(StudyViolation("demoted-edges", "edges", sum(edges.demoted.values()), examples))
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes