tablassert 22.1.0__tar.gz → 22.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {tablassert-22.1.0 → tablassert-22.2.0}/PKG-INFO +1 -1
  2. {tablassert-22.1.0 → tablassert-22.2.0}/pyproject.toml +1 -1
  3. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/enums.py +4 -0
  4. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/errors.py +8 -0
  5. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/lib.py +177 -10
  6. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/models.py +102 -4
  7. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/runcache.py +3 -0
  8. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/study.py +37 -6
  9. {tablassert-22.1.0 → tablassert-22.2.0}/LICENSE +0 -0
  10. {tablassert-22.1.0 → tablassert-22.2.0}/README.md +0 -0
  11. {tablassert-22.1.0 → tablassert-22.2.0}/rust/Cargo.lock +0 -0
  12. {tablassert-22.1.0 → tablassert-22.2.0}/rust/Cargo.toml +0 -0
  13. {tablassert-22.1.0 → tablassert-22.2.0}/rust/examples/count_tables.rs +0 -0
  14. {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/fullmap.rs +0 -0
  15. {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/json.rs +0 -0
  16. {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/lib.rs +0 -0
  17. {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/ndjson.rs +0 -0
  18. {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/nlp.rs +0 -0
  19. {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/uuid.rs +0 -0
  20. {tablassert-22.1.0 → tablassert-22.2.0}/rust/tests/build_golden.rs +0 -0
  21. {tablassert-22.1.0 → tablassert-22.2.0}/rust/tests/common/mod.rs +0 -0
  22. {tablassert-22.1.0 → tablassert-22.2.0}/rust/tests/extract_prebuilt.rs +0 -0
  23. {tablassert-22.1.0 → tablassert-22.2.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
  24. {tablassert-22.1.0 → tablassert-22.2.0}/rust/tests/nlp_golden.rs +0 -0
  25. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/__init__.py +0 -0
  26. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/_lazy.py +0 -0
  27. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/biolink.py +0 -0
  28. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/cli.py +0 -0
  29. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/cli_entry.py +0 -0
  30. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/coerce.py +0 -0
  31. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
  32. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/data/infores_catalog.yaml +0 -0
  33. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/extras.py +0 -0
  34. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/fullmap.py +0 -0
  35. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/infores_registry.py +0 -0
  36. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/ingests.py +0 -0
  37. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/log.py +0 -0
  38. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/net.py +0 -0
  39. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/nlp.py +0 -0
  40. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/progress.py +0 -0
  41. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/qc.py +0 -0
  42. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/rig.py +0 -0
  43. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/rs.pyi +0 -0
  44. {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 22.1.0
3
+ Version: 22.2.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "22.1.0"
3
+ version = "22.2.0"
4
4
  description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -42,6 +42,10 @@ class Comparisons(str, Enum):
42
42
  LE = "le"
43
43
  EQ = "eq"
44
44
  NE = "ne"
45
+ # Membership comparisons: the comparator is a list of identifiers, matched against the
46
+ # raw (pre-resolution) cell value. JOIN keeps matching rows, ANTIJOIN drops them.
47
+ JOIN = "join"
48
+ ANTIJOIN = "antijoin"
45
49
 
46
50
 
47
51
  class Functions(str, Enum):
@@ -56,6 +56,14 @@ TablassertErrorCodes = Literal[
56
56
  "config-rows-and-row-slice-conflict",
57
57
  "comparison-bad-comparator-type",
58
58
  "comparison-nonnumeric-comparator",
59
+ # Membership (join/antijoin) family: the first two are config-validation failures on the
60
+ # `comparator` field, the last two are op-level failures on the column being tested.
61
+ "comparison-membership-comparator-required",
62
+ "comparison-membership-empty",
63
+ "comparison-membership-member-not-a-string",
64
+ "reindex-membership-column-missing",
65
+ "reindex-membership-column-dtype-unsupported",
66
+ "comparison-regex-requires-eq-ne",
59
67
  "encoding-bad-excel-column",
60
68
  "regex-bad-pattern",
61
69
  "regex-bad-replacement",
@@ -45,11 +45,11 @@ from tablassert.coerce import (
45
45
  sig,
46
46
  study_size_target,
47
47
  )
48
- from tablassert.enums import EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
48
+ from tablassert.enums import Comparisons, EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
49
49
  from tablassert.errors import TablassertError
50
50
  from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
51
51
  from tablassert.log import cat
52
- from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
52
+ from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, Reindex, RIGConfig, Section
53
53
  from tablassert.nlp import level_one, level_two
54
54
  from tablassert.qc import fullmap_audit
55
55
  from tablassert.rig import (
@@ -1070,6 +1070,178 @@ def reindex(df: pl.LazyFrame, col: str, op: Callable, comp: str | int | float, c
1070
1070
  return df.filter(op(expr, comp))
1071
1071
 
1072
1072
 
1073
+ MEMBERSHIP_KEY_COLUMN: str = "_reindex_membership_key"
1074
+ """Base name for the throwaway join key column on the cast branch; extended with underscores until
1075
+ it is unused if a source frame already carries it, and dropped again before the op returns."""
1076
+
1077
+ MEMBERSHIP_UNCASTABLE_DTYPES: frozenset[str] = frozenset({"List", "Array", "Struct", "Object", "Binary"})
1078
+ """polars dtype class NAMES whose cells have no single-string identity, so membership is refused.
1079
+
1080
+ Matched by name because a module-level ``pl.List`` reference would resolve the lazy proxy and import
1081
+ polars at ``import tablassert.lib``, which is exactly what :mod:`tablassert._lazy` exists to avoid.
1082
+ Every parametrized variant reports its base name (``List(String)`` is named ``List``).
1083
+
1084
+ A ``List``/``Array`` cell cannot be cast to ``String`` at all (``InvalidOperationError``), an
1085
+ ``Object`` cell casts through python and never compares equal to an identifier, a ``Struct`` casts to
1086
+ a brace literal no identifier is spelled to match, and a ``Binary`` cell casts only when it happens to
1087
+ be valid UTF-8, so one list would match some rows and raise on the rest. Joining list cells on a
1088
+ separator instead would make a multi-value cell never match a single identifier, a silent wrong
1089
+ answer, so these dtypes raise a coded error.
1090
+ """
1091
+
1092
+
1093
+ def _membership_key_column(schema: pl.Schema) -> str:
1094
+ """Return a join key column name the frame does not already use.
1095
+
1096
+ Args:
1097
+ schema: Collected schema of the frame being filtered.
1098
+
1099
+ Returns:
1100
+ :data:`MEMBERSHIP_KEY_COLUMN`, or that name extended with underscores until it is free.
1101
+ """
1102
+ key_col: str = MEMBERSHIP_KEY_COLUMN
1103
+ while key_col in schema:
1104
+ key_col = f"{key_col}_"
1105
+ return key_col
1106
+
1107
+
1108
+ def reindex_membership(df: pl.LazyFrame, col: str, keys: list[str], anti: bool = False) -> pl.LazyFrame:
1109
+ """Reindex a LazyFrame by exact membership in a list of identifiers (join / antijoin).
1110
+
1111
+ Args:
1112
+ df: Source LazyFrame.
1113
+ col: Column whose raw (pre-resolution) cell values are tested.
1114
+ keys: Identifier list. Duplicates are ignored: a semi/anti join cannot multiply left
1115
+ rows, and the join build side is cheaper once deduplicated.
1116
+ anti: ``False`` (join) keeps rows whose value is in ``keys``; ``True`` (antijoin) keeps
1117
+ the rows that are not.
1118
+
1119
+ Returns:
1120
+ Filtered LazyFrame keeping the surviving rows in left-frame order, with every original
1121
+ column and its original dtype.
1122
+
1123
+ Raises:
1124
+ TablassertError: ``comparison-membership-empty`` for an empty list,
1125
+ ``reindex-membership-column-missing`` when the source lacks ``col``, and
1126
+ ``reindex-membership-column-dtype-unsupported`` for cells with no single string form
1127
+ (``List``, ``Array``, ``Struct``, ``Object``, ``Binary``).
1128
+
1129
+ Notes:
1130
+ Why a join and not ``is_in``: on the pinned polars (``>=2.0.0rc2``) a semi/anti hash join
1131
+ against a once-built key frame is the faster measured path once the list is non-trivial (1M
1132
+ rows, median of three collects: 100k keys 0.018s vs 0.029s filtered; 1k keys 0.003s vs
1133
+ 0.004s; a tie at ten keys), and it is the membership idiom already used in
1134
+ :mod:`tablassert.qc`. No ``is_in`` fast path exists for tiny lists: it propagates nulls
1135
+ differently from antijoin, and the gap is milliseconds per million rows.
1136
+
1137
+ ``maintain_order="left"`` is explicit, not incidental: polars documents the default order as
1138
+ unspecified ("might differ across Polars versions or even between different runs"), and the
1139
+ dependency is an open range, so a future 2.x must not be able to reshuffle a section's rows.
1140
+ Measured free here (1M rows / 100k keys, five runs, median): semi 5.1ms -> 4.6ms, anti 5.4ms
1141
+ -> 5.4ms.
1142
+
1143
+ The string path joins straight on ``col``, because semi/anti joins emit only the left frame's
1144
+ columns. Only the cast path adds a throwaway helper column, and it is dropped again, so
1145
+ entity resolution downstream sees the cell exactly as the source produced it.
1146
+
1147
+ Null semantics are join semantics and deliberately differ from ``ne``: a null cell never
1148
+ matches, so join drops it while antijoin KEEPS it.
1149
+
1150
+ Membership is textual, so an identifier must be spelled the way polars' cast renders it.
1151
+ Measured forms: ``Int64`` 3 -> ``"3"``; ``Float64`` 3.0 -> ``"3.0"`` (1e+20 keeps scientific
1152
+ form); ``Bool`` -> ``"true"``/``"false"``; ``Date`` -> ``"2024-01-15"``; ``Datetime`` ->
1153
+ ``"2024-01-15 06:24:30.000000"`` (space, not ``T``, with microsecond precision); ``Time`` ->
1154
+ ``"01:02:03"``; ``Decimal(10,2)`` 1.5 -> ``"1.50"`` (scale kept). A misspelled list is not an
1155
+ error: join emits an empty section and antijoin emits every row. No zero-match warning is
1156
+ emitted, because detecting it would cost an extra pass over the frame on every build.
1157
+ """
1158
+ if not keys:
1159
+ raise TablassertError("`join`/`antijoin` reindex requires at least one identifier.", code="comparison-membership-empty")
1160
+
1161
+ schema: pl.Schema = df.collect_schema()
1162
+ if col not in schema:
1163
+ raise TablassertError(f"`join`/`antijoin` reindex targets `{col}`, which the source does not have.", code="reindex-membership-column-missing")
1164
+ if type(schema[col]).__name__ in MEMBERSHIP_UNCASTABLE_DTYPES:
1165
+ raise TablassertError(
1166
+ f"`join`/`antijoin` reindex cannot test `{col}`: {schema[col]} cells have no single string form to match identifiers against.",
1167
+ code="reindex-membership-column-dtype-unsupported",
1168
+ )
1169
+
1170
+ unique: list[str] = list(dict.fromkeys(keys))
1171
+ how: Literal["semi", "anti"] = "anti" if anti else "semi"
1172
+ if isinstance(schema[col], pl.String):
1173
+ # No helper column: the join returns the left frame's columns unchanged.
1174
+ return df.join(pl.DataFrame({col: unique}).lazy(), on=col, how=how, maintain_order="left")
1175
+ key_col: str = _membership_key_column(schema)
1176
+ left: pl.LazyFrame = df.with_columns(pl.col(col).cast(pl.String, strict=False).alias(key_col))
1177
+ return left.join(pl.DataFrame({key_col: unique}).lazy(), on=key_col, how=how, maintain_order="left").drop(key_col)
1178
+
1179
+
1180
+ def reindex_regex(df: pl.LazyFrame, col: str, pattern: str, negate: bool = False) -> pl.LazyFrame:
1181
+ """Reindex a LazyFrame by filtering rows on a polars regex match against a column.
1182
+
1183
+ Args:
1184
+ df: Source LazyFrame.
1185
+ col: Column to test.
1186
+ pattern: Polars-compatible regex, matched unanchored and case-sensitively (write `^`/`$`
1187
+ to pin a whole cell).
1188
+ negate: When True keep the rows that do NOT match (the `ne` filter); when False keep the
1189
+ matches (the `eq` filter).
1190
+
1191
+ Returns:
1192
+ LazyFrame containing only rows whose predicate is true.
1193
+
1194
+ Notes:
1195
+ The column is cast to String first, so a typed source (parquet, or a numeric column in a
1196
+ text table) filters on its text form instead of raising polars' ``InvalidOperationError:
1197
+ expected String type``. A null cell matches nothing and ``not_()`` leaves it null, so null
1198
+ rows drop under both directions -- the same null rule literal `eq`/`ne` already have, since
1199
+ a null comparison is never true.
1200
+ """
1201
+ match: pl.Expr = pl.col(col).cast(pl.String).str.contains(pattern)
1202
+ return df.filter(match.not_() if negate else match)
1203
+
1204
+
1205
+ def reindex_op(entry: Reindex) -> tuple[Callable, tuple[Any, ...]]:
1206
+ """Compile one ``Reindex`` config entry into its Tcode op tuple.
1207
+
1208
+ Args:
1209
+ entry: Validated ``Reindex`` model. The validator guarantees the pairing: a membership entry
1210
+ carries a non-empty ``list[str]`` comparator, ``regex`` only ever sits on ``eq``/``ne``,
1211
+ and a scalar entry carries a scalar comparator.
1212
+
1213
+ Returns:
1214
+ ``(callable, args)`` for the pipeline: :func:`reindex_membership` for ``join``/``antijoin``,
1215
+ :func:`reindex_regex` for a ``regex`` entry, otherwise :func:`reindex` with the ``operator``
1216
+ module callable and the same ``cast`` choice it has always used (numeric comparisons cast to
1217
+ Float64, ``eq``/``ne`` do not).
1218
+
1219
+ Notes:
1220
+ Legacy entries keep emitting exactly the tuples they always did, so their run-cache op digests
1221
+ do not move; membership is the only new shape.
1222
+
1223
+ ``Reindex.comparison`` carries the plain enum *value* at runtime because ``TablaBase`` sets
1224
+ ``use_enum_values=True``, so this branches on the strings, exactly as the inline comprehension
1225
+ it replaces did.
1226
+ """
1227
+ col: str = idxname(entry.column)
1228
+ comparison: str = entry.comparison
1229
+ if comparison in (Comparisons.JOIN.value, Comparisons.ANTIJOIN.value):
1230
+ if not isinstance(entry.comparator, list):
1231
+ raise TablassertError(
1232
+ f"`{comparison}` reindex requires a list of identifiers, got {type(entry.comparator).__name__}.",
1233
+ code="comparison-membership-comparator-required",
1234
+ )
1235
+ return (reindex_membership, (col, entry.comparator, comparison == Comparisons.ANTIJOIN.value))
1236
+ if entry.regex:
1237
+ # models.Reindex rejects `regex` off eq/ne, so this arm never needs the numeric cast or an
1238
+ # operator lookup.
1239
+ return (reindex_regex, (col, entry.comparator, comparison == Comparisons.NE.value))
1240
+ if comparison in (Comparisons.NE.value, Comparisons.EQ.value):
1241
+ return (reindex, (col, getattr(operator, comparison), entry.comparator, False))
1242
+ return (reindex, (col, getattr(operator, comparison), entry.comparator))
1243
+
1244
+
1073
1245
  def drop_not_significant(lf: pl.LazyFrame, col: str = "statistical_significance_qualifier") -> pl.LazyFrame:
1074
1246
  """Drop release-mode edges whose significance qualifier is ``not_significant``.
1075
1247
 
@@ -1338,14 +1510,7 @@ class Tcode(Section):
1338
1510
  (idx, ()),
1339
1511
  (crop, (self.source.row_slice,)) if self.source.row_slice else None,
1340
1512
  (pick, (self.source.rows,)) if self.source.rows else None,
1341
- [
1342
- (reindex, (idxname(x.column), getattr(operator, x.comparison), x.comparator))
1343
- if x.comparison not in ["ne", "eq"]
1344
- else (reindex, (idxname(x.column), getattr(operator, x.comparison), x.comparator, False))
1345
- for x in self.source.reindex
1346
- ]
1347
- if self.source.reindex
1348
- else None,
1513
+ [reindex_op(x) for x in self.source.reindex] if self.source.reindex else None,
1349
1514
  # --head preview: randomly sample min(HEAD_ROWS, height) rows before any encoding/resolve.
1350
1515
  (head, (HEAD_ROWS,)) if self.head else None,
1351
1516
  [
@@ -1588,6 +1753,8 @@ PHASE_OF: dict[Callable, str] = {
1588
1753
  crop: "filter",
1589
1754
  pick: "filter",
1590
1755
  reindex: "filter",
1756
+ reindex_membership: "filter",
1757
+ reindex_regex: "filter",
1591
1758
  head: "filter",
1592
1759
  coerce_columns: "clean",
1593
1760
  clean_numeric: "clean",
@@ -119,21 +119,119 @@ class TablaBase(BaseModel):
119
119
  class Reindex(TablaBase):
120
120
  column: str = Field(..., pattern=r"^[A-Z]{1,3}$", description="Source column letters used for row filtering.", examples=["A", "AA"])
121
121
  comparison: Comparisons = Field(
122
- Comparisons.NE, description="Comparison operator used in reindex filtering.", examples=[Comparisons.NE, Comparisons.EQ, Comparisons.GT]
122
+ Comparisons.NE,
123
+ description="Comparison operator used in reindex filtering.",
124
+ examples=[Comparisons.NE, Comparisons.EQ, Comparisons.GT, Comparisons.JOIN, Comparisons.ANTIJOIN],
125
+ )
126
+ comparator: str | int | float | list[str] = Field(
127
+ ...,
128
+ description=(
129
+ "Right-side value compared against the selected column. For `join`/`antijoin` this is the list of "
130
+ "identifiers matched exactly against the raw (pre-resolution) cell value."
131
+ ),
132
+ examples=["N/A", 0, 1.5, ["CHEBI:41774", "CHEBI:48217"]],
133
+ )
134
+ regex: bool = Field(
135
+ False,
136
+ description=(
137
+ "When true, `comparator` is a polars-compatible regex matched against the cell instead of a literal "
138
+ "value: `eq` keeps rows whose cell matches, `ne` keeps rows that do not. Matching is case-sensitive and "
139
+ "unanchored (add `^`/`$` to pin the whole cell), the column is compared as text, and a null cell never "
140
+ "matches, so null rows drop under both operators. Only valid with `eq`/`ne`; an invalid or empty pattern "
141
+ "is rejected at config time (`regex-bad-pattern`)."
142
+ ),
123
143
  )
124
- comparator: str | int | float = Field(..., description="Right-side value compared against the selected column.", examples=["N/A", 0, 1.5])
144
+
145
+ @model_validator(mode="before")
146
+ @classmethod
147
+ def membership_members_are_strings(cls, data: Any) -> Any:
148
+ """Reject an identifier list whose members are not strings, with a code rather than a field type error.
149
+
150
+ Args:
151
+ data: Raw config value for one ``Reindex`` entry.
152
+
153
+ Returns:
154
+ ``data`` unchanged when the entry is not a membership list, or when every member is a
155
+ string.
156
+
157
+ Raises:
158
+ TablassertValidationError: ``comparison-membership-member-not-a-string`` when a
159
+ ``join``/``antijoin`` list holds a non-string member.
160
+
161
+ Notes:
162
+ Without this guard the pydantic union rejects `comparator: [3, 4]` (the natural spelling
163
+ against an `Int64` column, whose string form is `3`) as a bare `string_type` field error
164
+ with no stable code and no docs URL, while every other failure in this family is coded.
165
+ The fix stays "quote the identifiers": membership is textual, so an unquoted number is a
166
+ config error rather than something to coerce silently.
167
+ """
168
+ if not isinstance(data, dict):
169
+ return data
170
+ comparison = data.get("comparison")
171
+ comparator = data.get("comparator")
172
+ if comparison not in (Comparisons.JOIN.value, Comparisons.ANTIJOIN.value) or not isinstance(comparator, list):
173
+ return data
174
+ for index, member in enumerate(comparator):
175
+ if not isinstance(member, str):
176
+ raise TablassertValidationError(
177
+ f"`join`/`antijoin` identifier lists must hold quoted strings: member {index} is {type(member).__name__} ({member!r}). "
178
+ f'Quote it (for example ["{member}"]) to match the cell as text.',
179
+ code="comparison-membership-member-not-a-string",
180
+ )
181
+ return data
125
182
 
126
183
  @model_validator(mode="after")
127
184
  def comparison_datatypes(self: Self) -> Self:
128
185
  x: Comparisons = self.comparison
129
- y: str | int | float = self.comparator
186
+ y: str | int | float | list[str] = self.comparator
130
187
 
131
- if x == Comparisons.NE or x == Comparisons.EQ:
188
+ if x == Comparisons.JOIN or x == Comparisons.ANTIJOIN:
189
+ if self.regex:
190
+ raise TablassertValidationError(
191
+ "`regex` is only meaningful with the `eq`/`ne` comparisons; `join`/`antijoin` match identifiers by exact string identity against a list.",
192
+ code="comparison-regex-requires-eq-ne",
193
+ )
194
+ if not isinstance(y, list):
195
+ raise TablassertValidationError(
196
+ f"`join`/`antijoin` comparisons require a list of identifier strings, got {type(y).__name__}.",
197
+ code="comparison-membership-comparator-required",
198
+ )
199
+ if not y:
200
+ raise TablassertValidationError(
201
+ "`join`/`antijoin` comparators must hold at least one identifier.", code="comparison-membership-empty"
202
+ )
203
+ elif x == Comparisons.NE or x == Comparisons.EQ:
132
204
  if not isinstance(y, str):
133
205
  raise TablassertValidationError(
134
206
  f"`eq`/`ne` comparisons require a str comparator, got {type(y).__name__}.", code="comparison-bad-comparator-type"
135
207
  )
208
+ if self.regex:
209
+ # An empty-or-everything pattern compiles, but it matches EVERY cell, so a `ne` filter
210
+ # would silently drop the whole table. Reject whitespace-only spellings and any pattern
211
+ # that can match the empty string (`^`, `()`, `.*`, `x*`) with the same code the
212
+ # `exclude_regex` probe uses.
213
+ if not y.strip():
214
+ raise TablassertValidationError(
215
+ f"`comparator` must be a non-empty pattern when `regex` is true (an empty pattern matches every cell), got {y!r}.",
216
+ code="regex-bad-pattern",
217
+ )
218
+ try:
219
+ pl.Series([""]).str.contains(y)
220
+ except Exception as e:
221
+ raise TablassertValidationError(
222
+ f"`comparator` must be a polars-compatible regex when `regex` is true, got {y!r}: {e}", code="regex-bad-pattern"
223
+ ) from e
224
+ if pl.Series([""]).str.contains(y).item():
225
+ raise TablassertValidationError(
226
+ f"`comparator` must not match an empty cell when `regex` is true (it would match every cell, so an `ne` filter would drop the whole table), got {y!r}.",
227
+ code="regex-bad-pattern",
228
+ )
136
229
  else:
230
+ if self.regex:
231
+ raise TablassertValidationError(
232
+ "`regex` is only meaningful with the `eq`/`ne` comparisons; ordering comparisons cast the column to Float64 and compare numerically.",
233
+ code="comparison-regex-requires-eq-ne",
234
+ )
137
235
  if not isinstance(y, (int, float)):
138
236
  raise TablassertValidationError(
139
237
  f"Comparisons other than `eq`/`ne` require a float or int comparator, got {type(y).__name__}.",
@@ -258,6 +258,9 @@ OP_COST: dict[Callable, int] = {
258
258
  lib.crop: 1,
259
259
  lib.pick: 1,
260
260
  lib.reindex: 1,
261
+ # Membership filter: one hash join against a small key frame, same single-predicate price.
262
+ lib.reindex_membership: 1,
263
+ lib.reindex_regex: 1,
261
264
  # head samples min(HEAD_ROWS, height) rows, so it needs a height pass before slicing.
262
265
  lib.head: 2,
263
266
  # Encode: each is one lazy expression over a single column.
@@ -20,6 +20,7 @@ _MESSAGES: dict[str, str] = {
20
20
  "unnamed-nodes": "nodes with no name or an empty name",
21
21
  "unidentified-nodes": "nodes with no id or an empty id",
22
22
  "incomplete-edges": "edges missing subject, predicate, or object",
23
+ "self-loop-edges": "edges whose subject and object are the same CURIE",
23
24
  "demoted-edges": "edges demoted to bare biolink:Association despite a declared category_override",
24
25
  "undeclared-nodes": "nodes referenced by edges but not declared in the nodes file",
25
26
  "isolated-nodes": "declared nodes participating in no edge",
@@ -61,6 +62,9 @@ class _FileScan:
61
62
  #: predicate. A demotion is only a violation when the build declared a
62
63
  #: ``category_override`` (see :func:`study_kgx`); the count is collected either way.
63
64
  demoted: Counter[str]
65
+ #: Edges that assert a concept relates to itself, keyed by the shared CURIE. See
66
+ #: :func:`study_kgx` for why a self-referential edge is a violation.
67
+ self_loops: Counter[str]
64
68
  malformed: int
65
69
  missing: bool
66
70
  path: Path
@@ -94,17 +98,18 @@ def _scan_ndjson(path: Path, *, edge: bool) -> _FileScan:
94
98
 
95
99
  Args:
96
100
  path: Path to a ``.nodes.ndjson`` or ``.edges.ndjson`` file.
97
- edge: ``True`` to collect referenced ids from ``subject``/``object`` and
98
- assert the three core edge slots are present; ``False`` to collect
99
- declared node ``id``s and track duplicates, nodes with no name or
100
- an empty name, and nodes with no id or an empty id.
101
+ edge: ``True`` to collect referenced ids from ``subject``/``object``, assert the
102
+ three core edge slots are present, and tally self-referential
103
+ (``subject == object``) edges; ``False`` to collect declared node ``id``s and
104
+ track duplicates, nodes with no name or an empty name, and nodes with no id
105
+ or an empty id.
101
106
 
102
107
  Returns:
103
108
  A :class:`_FileScan`; ``missing`` is set (and nothing else) when the
104
109
  file does not exist, so a typo'd path can never read as a clean pass.
105
110
  """
106
111
  scan: _FileScan = _FileScan(
107
- set(), set(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), 0, not path.is_file(), path
112
+ set(), set(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), 0, not path.is_file(), path
108
113
  )
109
114
  if scan.missing:
110
115
  return scan
@@ -162,6 +167,19 @@ def _scan_ndjson(path: Path, *, edge: bool) -> _FileScan:
162
167
  slot_value: object = record.get(slot)
163
168
  if slot_value is None or (isinstance(slot_value, str) and not slot_value.strip()):
164
169
  scan.incomplete[slot] += 1
170
+ # A self-referential edge states that a concept relates to itself. In a
171
+ # table-derived graph that is almost never the claim the author made: it is
172
+ # the signature of a resolution collapse, where two differently worded cells
173
+ # (or one cell duplicated into both slots) land on the same CURIE -- the same
174
+ # failure mode that makes two rows share an edge id. Such an edge carries no
175
+ # relational information, so downstream consumers either drop it silently or
176
+ # count it as evidence. Only a non-empty string pair is flagged; absent or
177
+ # blank slots are already reported by ``incomplete-edges``, and a blank-vs-blank
178
+ # pair must not read as "the same CURIE".
179
+ subject: object = record.get("subject")
180
+ object_: object = record.get("object")
181
+ if isinstance(subject, str) and isinstance(object_, str) and subject == object_ and subject.strip():
182
+ scan.self_loops[subject] += 1
165
183
  # Bare ``biolink:Association`` means the row lost every specific class:
166
184
  # ``lib.edge_category`` derives a class from the (subject role, object
167
185
  # role) pair -- optionally pinned per object category by the section's
@@ -215,7 +233,8 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
215
233
  Streams both files once each and checks: duplicate node ids, duplicate edge
216
234
  ids, nodes with no
217
235
  name or an empty name, nodes with no id or an empty id, edges missing any
218
- of ``subject``/``predicate``/``object``, nodes referenced by edges but
236
+ of ``subject``/``predicate``/``object``, edges whose ``subject`` and ``object`` are
237
+ the same CURIE (self-referential), nodes referenced by edges but
219
238
  never declared (``undeclared``), declared nodes participating in no edge
220
239
  (``isolated``), empty/malformed lines, string values carrying leading/trailing
221
240
  whitespace, and null or empty values in any field (a stronger contract than
@@ -229,6 +248,15 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
229
248
  ``has_supporting_studies``. Every check is an assertion -- the caller decides
230
249
  whether violations fail the build.
231
250
 
251
+ A self-referential edge (``subject`` and ``object`` resolved to one CURIE) is an
252
+ assertion that a concept relates to itself. Table-derived graphs produce them almost
253
+ always by accident: two differently worded cells collapsing onto the same identifier,
254
+ or one cell filling both statement slots, so the edge carries no relational content and
255
+ ships as noise a consumer has to filter. QC therefore reports it rather than quietly
256
+ deleting it, so the author sees which CURIEs collapsed and fixes the mapping (or drops
257
+ the row) in the config. Intentionally reflexive assertions are a known accepted caveat:
258
+ they fail a ``--qc`` build like every other violation.
259
+
232
260
  Args:
233
261
  nodes_path: Path to ``<name>_<version>.nodes.ndjson``.
234
262
  edges_path: Path to ``<name>_<version>.edges.ndjson``.
@@ -270,6 +298,9 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
270
298
  if edges.incomplete:
271
299
  examples = [f"{slot} ({n})" for slot, n in edges.incomplete.most_common(example_limit)]
272
300
  violations.append(StudyViolation("incomplete-edges", "edges", sum(edges.incomplete.values()), examples))
301
+ if edges.self_loops:
302
+ examples = [f"{curie} ({n})" for curie, n in edges.self_loops.most_common(example_limit)]
303
+ violations.append(StudyViolation("self-loop-edges", "edges", sum(edges.self_loops.values()), examples))
273
304
  if category_override_declared and edges.demoted:
274
305
  examples = [f"{predicate} ({n})" for predicate, n in edges.demoted.most_common(example_limit)]
275
306
  violations.append(StudyViolation("demoted-edges", "edges", sum(edges.demoted.values()), examples))
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes