tablassert 22.1.0__tar.gz → 22.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-22.1.0 → tablassert-22.2.0}/PKG-INFO +1 -1
- {tablassert-22.1.0 → tablassert-22.2.0}/pyproject.toml +1 -1
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/enums.py +4 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/errors.py +8 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/lib.py +177 -10
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/models.py +102 -4
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/runcache.py +3 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/study.py +37 -6
- {tablassert-22.1.0 → tablassert-22.2.0}/LICENSE +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/README.md +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/Cargo.lock +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/Cargo.toml +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/fullmap.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/json.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/lib.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/ndjson.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/nlp.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/src/uuid.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/__init__.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/biolink.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/cli.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/cli_entry.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/coerce.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/data/infores_catalog.yaml +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/extras.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/fullmap.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/infores_registry.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/ingests.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/log.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/net.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/nlp.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/progress.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/qc.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/rig.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-22.1.0 → tablassert-22.2.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "22.
|
|
3
|
+
version = "22.2.0"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -42,6 +42,10 @@ class Comparisons(str, Enum):
|
|
|
42
42
|
LE = "le"
|
|
43
43
|
EQ = "eq"
|
|
44
44
|
NE = "ne"
|
|
45
|
+
# Membership comparisons: the comparator is a list of identifiers, matched against the
|
|
46
|
+
# raw (pre-resolution) cell value. JOIN keeps matching rows, ANTIJOIN drops them.
|
|
47
|
+
JOIN = "join"
|
|
48
|
+
ANTIJOIN = "antijoin"
|
|
45
49
|
|
|
46
50
|
|
|
47
51
|
class Functions(str, Enum):
|
|
@@ -56,6 +56,14 @@ TablassertErrorCodes = Literal[
|
|
|
56
56
|
"config-rows-and-row-slice-conflict",
|
|
57
57
|
"comparison-bad-comparator-type",
|
|
58
58
|
"comparison-nonnumeric-comparator",
|
|
59
|
+
# Membership (join/antijoin) family: the first two are config-validation failures on the
|
|
60
|
+
# `comparator` field, the last two are op-level failures on the column being tested.
|
|
61
|
+
"comparison-membership-comparator-required",
|
|
62
|
+
"comparison-membership-empty",
|
|
63
|
+
"comparison-membership-member-not-a-string",
|
|
64
|
+
"reindex-membership-column-missing",
|
|
65
|
+
"reindex-membership-column-dtype-unsupported",
|
|
66
|
+
"comparison-regex-requires-eq-ne",
|
|
59
67
|
"encoding-bad-excel-column",
|
|
60
68
|
"regex-bad-pattern",
|
|
61
69
|
"regex-bad-replacement",
|
|
@@ -45,11 +45,11 @@ from tablassert.coerce import (
|
|
|
45
45
|
sig,
|
|
46
46
|
study_size_target,
|
|
47
47
|
)
|
|
48
|
-
from tablassert.enums import EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
|
|
48
|
+
from tablassert.enums import Comparisons, EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
|
|
49
49
|
from tablassert.errors import TablassertError
|
|
50
50
|
from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
|
|
51
51
|
from tablassert.log import cat
|
|
52
|
-
from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
|
|
52
|
+
from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, Reindex, RIGConfig, Section
|
|
53
53
|
from tablassert.nlp import level_one, level_two
|
|
54
54
|
from tablassert.qc import fullmap_audit
|
|
55
55
|
from tablassert.rig import (
|
|
@@ -1070,6 +1070,178 @@ def reindex(df: pl.LazyFrame, col: str, op: Callable, comp: str | int | float, c
|
|
|
1070
1070
|
return df.filter(op(expr, comp))
|
|
1071
1071
|
|
|
1072
1072
|
|
|
1073
|
+
MEMBERSHIP_KEY_COLUMN: str = "_reindex_membership_key"
|
|
1074
|
+
"""Base name for the throwaway join key column on the cast branch; extended with underscores until
|
|
1075
|
+
it is unused if a source frame already carries it, and dropped again before the op returns."""
|
|
1076
|
+
|
|
1077
|
+
MEMBERSHIP_UNCASTABLE_DTYPES: frozenset[str] = frozenset({"List", "Array", "Struct", "Object", "Binary"})
|
|
1078
|
+
"""polars dtype class NAMES whose cells have no single-string identity, so membership is refused.
|
|
1079
|
+
|
|
1080
|
+
Matched by name because a module-level ``pl.List`` reference would resolve the lazy proxy and import
|
|
1081
|
+
polars at ``import tablassert.lib``, which is exactly what :mod:`tablassert._lazy` exists to avoid.
|
|
1082
|
+
Every parametrized variant reports its base name (``List(String)`` is named ``List``).
|
|
1083
|
+
|
|
1084
|
+
A ``List``/``Array`` cell cannot be cast to ``String`` at all (``InvalidOperationError``), an
|
|
1085
|
+
``Object`` cell casts through python and never compares equal to an identifier, a ``Struct`` casts to
|
|
1086
|
+
a brace literal no identifier is spelled to match, and a ``Binary`` cell casts only when it happens to
|
|
1087
|
+
be valid UTF-8, so one list would match some rows and raise on the rest. Joining list cells on a
|
|
1088
|
+
separator instead would make a multi-value cell never match a single identifier, a silent wrong
|
|
1089
|
+
answer, so these dtypes raise a coded error.
|
|
1090
|
+
"""
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
def _membership_key_column(schema: pl.Schema) -> str:
|
|
1094
|
+
"""Return a join key column name the frame does not already use.
|
|
1095
|
+
|
|
1096
|
+
Args:
|
|
1097
|
+
schema: Collected schema of the frame being filtered.
|
|
1098
|
+
|
|
1099
|
+
Returns:
|
|
1100
|
+
:data:`MEMBERSHIP_KEY_COLUMN`, or that name extended with underscores until it is free.
|
|
1101
|
+
"""
|
|
1102
|
+
key_col: str = MEMBERSHIP_KEY_COLUMN
|
|
1103
|
+
while key_col in schema:
|
|
1104
|
+
key_col = f"{key_col}_"
|
|
1105
|
+
return key_col
|
|
1106
|
+
|
|
1107
|
+
|
|
1108
|
+
def reindex_membership(df: pl.LazyFrame, col: str, keys: list[str], anti: bool = False) -> pl.LazyFrame:
|
|
1109
|
+
"""Reindex a LazyFrame by exact membership in a list of identifiers (join / antijoin).
|
|
1110
|
+
|
|
1111
|
+
Args:
|
|
1112
|
+
df: Source LazyFrame.
|
|
1113
|
+
col: Column whose raw (pre-resolution) cell values are tested.
|
|
1114
|
+
keys: Identifier list. Duplicates are ignored: a semi/anti join cannot multiply left
|
|
1115
|
+
rows, and the join build side is cheaper once deduplicated.
|
|
1116
|
+
anti: ``False`` (join) keeps rows whose value is in ``keys``; ``True`` (antijoin) keeps
|
|
1117
|
+
the rows that are not.
|
|
1118
|
+
|
|
1119
|
+
Returns:
|
|
1120
|
+
Filtered LazyFrame keeping the surviving rows in left-frame order, with every original
|
|
1121
|
+
column and its original dtype.
|
|
1122
|
+
|
|
1123
|
+
Raises:
|
|
1124
|
+
TablassertError: ``comparison-membership-empty`` for an empty list,
|
|
1125
|
+
``reindex-membership-column-missing`` when the source lacks ``col``, and
|
|
1126
|
+
``reindex-membership-column-dtype-unsupported`` for cells with no single string form
|
|
1127
|
+
(``List``, ``Array``, ``Struct``, ``Object``, ``Binary``).
|
|
1128
|
+
|
|
1129
|
+
Notes:
|
|
1130
|
+
Why a join and not ``is_in``: on the pinned polars (``>=2.0.0rc2``) a semi/anti hash join
|
|
1131
|
+
against a once-built key frame is the faster measured path once the list is non-trivial (1M
|
|
1132
|
+
rows, median of three collects: 100k keys 0.018s vs 0.029s filtered; 1k keys 0.003s vs
|
|
1133
|
+
0.004s; a tie at ten keys), and it is the membership idiom already used in
|
|
1134
|
+
:mod:`tablassert.qc`. No ``is_in`` fast path exists for tiny lists: it propagates nulls
|
|
1135
|
+
differently from antijoin, and the gap is milliseconds per million rows.
|
|
1136
|
+
|
|
1137
|
+
``maintain_order="left"`` is explicit, not incidental: polars documents the default order as
|
|
1138
|
+
unspecified ("might differ across Polars versions or even between different runs"), and the
|
|
1139
|
+
dependency is an open range, so a future 2.x must not be able to reshuffle a section's rows.
|
|
1140
|
+
Measured free here (1M rows / 100k keys, five runs, median): semi 5.1ms -> 4.6ms, anti 5.4ms
|
|
1141
|
+
-> 5.4ms.
|
|
1142
|
+
|
|
1143
|
+
The string path joins straight on ``col``, because semi/anti joins emit only the left frame's
|
|
1144
|
+
columns. Only the cast path adds a throwaway helper column, and it is dropped again, so
|
|
1145
|
+
entity resolution downstream sees the cell exactly as the source produced it.
|
|
1146
|
+
|
|
1147
|
+
Null semantics are join semantics and deliberately differ from ``ne``: a null cell never
|
|
1148
|
+
matches, so join drops it while antijoin KEEPS it.
|
|
1149
|
+
|
|
1150
|
+
Membership is textual, so an identifier must be spelled the way polars' cast renders it.
|
|
1151
|
+
Measured forms: ``Int64`` 3 -> ``"3"``; ``Float64`` 3.0 -> ``"3.0"`` (1e+20 keeps scientific
|
|
1152
|
+
form); ``Bool`` -> ``"true"``/``"false"``; ``Date`` -> ``"2024-01-15"``; ``Datetime`` ->
|
|
1153
|
+
``"2024-01-15 06:24:30.000000"`` (space, not ``T``, with microsecond precision); ``Time`` ->
|
|
1154
|
+
``"01:02:03"``; ``Decimal(10,2)`` 1.5 -> ``"1.50"`` (scale kept). A misspelled list is not an
|
|
1155
|
+
error: join emits an empty section and antijoin emits every row. No zero-match warning is
|
|
1156
|
+
emitted, because detecting it would cost an extra pass over the frame on every build.
|
|
1157
|
+
"""
|
|
1158
|
+
if not keys:
|
|
1159
|
+
raise TablassertError("`join`/`antijoin` reindex requires at least one identifier.", code="comparison-membership-empty")
|
|
1160
|
+
|
|
1161
|
+
schema: pl.Schema = df.collect_schema()
|
|
1162
|
+
if col not in schema:
|
|
1163
|
+
raise TablassertError(f"`join`/`antijoin` reindex targets `{col}`, which the source does not have.", code="reindex-membership-column-missing")
|
|
1164
|
+
if type(schema[col]).__name__ in MEMBERSHIP_UNCASTABLE_DTYPES:
|
|
1165
|
+
raise TablassertError(
|
|
1166
|
+
f"`join`/`antijoin` reindex cannot test `{col}`: {schema[col]} cells have no single string form to match identifiers against.",
|
|
1167
|
+
code="reindex-membership-column-dtype-unsupported",
|
|
1168
|
+
)
|
|
1169
|
+
|
|
1170
|
+
unique: list[str] = list(dict.fromkeys(keys))
|
|
1171
|
+
how: Literal["semi", "anti"] = "anti" if anti else "semi"
|
|
1172
|
+
if isinstance(schema[col], pl.String):
|
|
1173
|
+
# No helper column: the join returns the left frame's columns unchanged.
|
|
1174
|
+
return df.join(pl.DataFrame({col: unique}).lazy(), on=col, how=how, maintain_order="left")
|
|
1175
|
+
key_col: str = _membership_key_column(schema)
|
|
1176
|
+
left: pl.LazyFrame = df.with_columns(pl.col(col).cast(pl.String, strict=False).alias(key_col))
|
|
1177
|
+
return left.join(pl.DataFrame({key_col: unique}).lazy(), on=key_col, how=how, maintain_order="left").drop(key_col)
|
|
1178
|
+
|
|
1179
|
+
|
|
1180
|
+
def reindex_regex(df: pl.LazyFrame, col: str, pattern: str, negate: bool = False) -> pl.LazyFrame:
|
|
1181
|
+
"""Reindex a LazyFrame by filtering rows on a polars regex match against a column.
|
|
1182
|
+
|
|
1183
|
+
Args:
|
|
1184
|
+
df: Source LazyFrame.
|
|
1185
|
+
col: Column to test.
|
|
1186
|
+
pattern: Polars-compatible regex, matched unanchored and case-sensitively (write `^`/`$`
|
|
1187
|
+
to pin a whole cell).
|
|
1188
|
+
negate: When True keep the rows that do NOT match (the `ne` filter); when False keep the
|
|
1189
|
+
matches (the `eq` filter).
|
|
1190
|
+
|
|
1191
|
+
Returns:
|
|
1192
|
+
LazyFrame containing only rows whose predicate is true.
|
|
1193
|
+
|
|
1194
|
+
Notes:
|
|
1195
|
+
The column is cast to String first, so a typed source (parquet, or a numeric column in a
|
|
1196
|
+
text table) filters on its text form instead of raising polars' ``InvalidOperationError:
|
|
1197
|
+
expected String type``. A null cell matches nothing and ``not_()`` leaves it null, so null
|
|
1198
|
+
rows drop under both directions -- the same null rule literal `eq`/`ne` already have, since
|
|
1199
|
+
a null comparison is never true.
|
|
1200
|
+
"""
|
|
1201
|
+
match: pl.Expr = pl.col(col).cast(pl.String).str.contains(pattern)
|
|
1202
|
+
return df.filter(match.not_() if negate else match)
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
def reindex_op(entry: Reindex) -> tuple[Callable, tuple[Any, ...]]:
|
|
1206
|
+
"""Compile one ``Reindex`` config entry into its Tcode op tuple.
|
|
1207
|
+
|
|
1208
|
+
Args:
|
|
1209
|
+
entry: Validated ``Reindex`` model. The validator guarantees the pairing: a membership entry
|
|
1210
|
+
carries a non-empty ``list[str]`` comparator, ``regex`` only ever sits on ``eq``/``ne``,
|
|
1211
|
+
and a scalar entry carries a scalar comparator.
|
|
1212
|
+
|
|
1213
|
+
Returns:
|
|
1214
|
+
``(callable, args)`` for the pipeline: :func:`reindex_membership` for ``join``/``antijoin``,
|
|
1215
|
+
:func:`reindex_regex` for a ``regex`` entry, otherwise :func:`reindex` with the ``operator``
|
|
1216
|
+
module callable and the same ``cast`` choice it has always used (numeric comparisons cast to
|
|
1217
|
+
Float64, ``eq``/``ne`` do not).
|
|
1218
|
+
|
|
1219
|
+
Notes:
|
|
1220
|
+
Legacy entries keep emitting exactly the tuples they always did, so their run-cache op digests
|
|
1221
|
+
do not move; membership is the only new shape.
|
|
1222
|
+
|
|
1223
|
+
``Reindex.comparison`` carries the plain enum *value* at runtime because ``TablaBase`` sets
|
|
1224
|
+
``use_enum_values=True``, so this branches on the strings, exactly as the inline comprehension
|
|
1225
|
+
it replaces did.
|
|
1226
|
+
"""
|
|
1227
|
+
col: str = idxname(entry.column)
|
|
1228
|
+
comparison: str = entry.comparison
|
|
1229
|
+
if comparison in (Comparisons.JOIN.value, Comparisons.ANTIJOIN.value):
|
|
1230
|
+
if not isinstance(entry.comparator, list):
|
|
1231
|
+
raise TablassertError(
|
|
1232
|
+
f"`{comparison}` reindex requires a list of identifiers, got {type(entry.comparator).__name__}.",
|
|
1233
|
+
code="comparison-membership-comparator-required",
|
|
1234
|
+
)
|
|
1235
|
+
return (reindex_membership, (col, entry.comparator, comparison == Comparisons.ANTIJOIN.value))
|
|
1236
|
+
if entry.regex:
|
|
1237
|
+
# models.Reindex rejects `regex` off eq/ne, so this arm never needs the numeric cast or an
|
|
1238
|
+
# operator lookup.
|
|
1239
|
+
return (reindex_regex, (col, entry.comparator, comparison == Comparisons.NE.value))
|
|
1240
|
+
if comparison in (Comparisons.NE.value, Comparisons.EQ.value):
|
|
1241
|
+
return (reindex, (col, getattr(operator, comparison), entry.comparator, False))
|
|
1242
|
+
return (reindex, (col, getattr(operator, comparison), entry.comparator))
|
|
1243
|
+
|
|
1244
|
+
|
|
1073
1245
|
def drop_not_significant(lf: pl.LazyFrame, col: str = "statistical_significance_qualifier") -> pl.LazyFrame:
|
|
1074
1246
|
"""Drop release-mode edges whose significance qualifier is ``not_significant``.
|
|
1075
1247
|
|
|
@@ -1338,14 +1510,7 @@ class Tcode(Section):
|
|
|
1338
1510
|
(idx, ()),
|
|
1339
1511
|
(crop, (self.source.row_slice,)) if self.source.row_slice else None,
|
|
1340
1512
|
(pick, (self.source.rows,)) if self.source.rows else None,
|
|
1341
|
-
[
|
|
1342
|
-
(reindex, (idxname(x.column), getattr(operator, x.comparison), x.comparator))
|
|
1343
|
-
if x.comparison not in ["ne", "eq"]
|
|
1344
|
-
else (reindex, (idxname(x.column), getattr(operator, x.comparison), x.comparator, False))
|
|
1345
|
-
for x in self.source.reindex
|
|
1346
|
-
]
|
|
1347
|
-
if self.source.reindex
|
|
1348
|
-
else None,
|
|
1513
|
+
[reindex_op(x) for x in self.source.reindex] if self.source.reindex else None,
|
|
1349
1514
|
# --head preview: randomly sample min(HEAD_ROWS, height) rows before any encoding/resolve.
|
|
1350
1515
|
(head, (HEAD_ROWS,)) if self.head else None,
|
|
1351
1516
|
[
|
|
@@ -1588,6 +1753,8 @@ PHASE_OF: dict[Callable, str] = {
|
|
|
1588
1753
|
crop: "filter",
|
|
1589
1754
|
pick: "filter",
|
|
1590
1755
|
reindex: "filter",
|
|
1756
|
+
reindex_membership: "filter",
|
|
1757
|
+
reindex_regex: "filter",
|
|
1591
1758
|
head: "filter",
|
|
1592
1759
|
coerce_columns: "clean",
|
|
1593
1760
|
clean_numeric: "clean",
|
|
@@ -119,21 +119,119 @@ class TablaBase(BaseModel):
|
|
|
119
119
|
class Reindex(TablaBase):
|
|
120
120
|
column: str = Field(..., pattern=r"^[A-Z]{1,3}$", description="Source column letters used for row filtering.", examples=["A", "AA"])
|
|
121
121
|
comparison: Comparisons = Field(
|
|
122
|
-
Comparisons.NE,
|
|
122
|
+
Comparisons.NE,
|
|
123
|
+
description="Comparison operator used in reindex filtering.",
|
|
124
|
+
examples=[Comparisons.NE, Comparisons.EQ, Comparisons.GT, Comparisons.JOIN, Comparisons.ANTIJOIN],
|
|
125
|
+
)
|
|
126
|
+
comparator: str | int | float | list[str] = Field(
|
|
127
|
+
...,
|
|
128
|
+
description=(
|
|
129
|
+
"Right-side value compared against the selected column. For `join`/`antijoin` this is the list of "
|
|
130
|
+
"identifiers matched exactly against the raw (pre-resolution) cell value."
|
|
131
|
+
),
|
|
132
|
+
examples=["N/A", 0, 1.5, ["CHEBI:41774", "CHEBI:48217"]],
|
|
133
|
+
)
|
|
134
|
+
regex: bool = Field(
|
|
135
|
+
False,
|
|
136
|
+
description=(
|
|
137
|
+
"When true, `comparator` is a polars-compatible regex matched against the cell instead of a literal "
|
|
138
|
+
"value: `eq` keeps rows whose cell matches, `ne` keeps rows that do not. Matching is case-sensitive and "
|
|
139
|
+
"unanchored (add `^`/`$` to pin the whole cell), the column is compared as text, and a null cell never "
|
|
140
|
+
"matches, so null rows drop under both operators. Only valid with `eq`/`ne`; an invalid or empty pattern "
|
|
141
|
+
"is rejected at config time (`regex-bad-pattern`)."
|
|
142
|
+
),
|
|
123
143
|
)
|
|
124
|
-
|
|
144
|
+
|
|
145
|
+
@model_validator(mode="before")
|
|
146
|
+
@classmethod
|
|
147
|
+
def membership_members_are_strings(cls, data: Any) -> Any:
|
|
148
|
+
"""Reject an identifier list whose members are not strings, with a code rather than a field type error.
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
data: Raw config value for one ``Reindex`` entry.
|
|
152
|
+
|
|
153
|
+
Returns:
|
|
154
|
+
``data`` unchanged when the entry is not a membership list, or when every member is a
|
|
155
|
+
string.
|
|
156
|
+
|
|
157
|
+
Raises:
|
|
158
|
+
TablassertValidationError: ``comparison-membership-member-not-a-string`` when a
|
|
159
|
+
``join``/``antijoin`` list holds a non-string member.
|
|
160
|
+
|
|
161
|
+
Notes:
|
|
162
|
+
Without this guard the pydantic union rejects `comparator: [3, 4]` (the natural spelling
|
|
163
|
+
against an `Int64` column, whose string form is `3`) as a bare `string_type` field error
|
|
164
|
+
with no stable code and no docs URL, while every other failure in this family is coded.
|
|
165
|
+
The fix stays "quote the identifiers": membership is textual, so an unquoted number is a
|
|
166
|
+
config error rather than something to coerce silently.
|
|
167
|
+
"""
|
|
168
|
+
if not isinstance(data, dict):
|
|
169
|
+
return data
|
|
170
|
+
comparison = data.get("comparison")
|
|
171
|
+
comparator = data.get("comparator")
|
|
172
|
+
if comparison not in (Comparisons.JOIN.value, Comparisons.ANTIJOIN.value) or not isinstance(comparator, list):
|
|
173
|
+
return data
|
|
174
|
+
for index, member in enumerate(comparator):
|
|
175
|
+
if not isinstance(member, str):
|
|
176
|
+
raise TablassertValidationError(
|
|
177
|
+
f"`join`/`antijoin` identifier lists must hold quoted strings: member {index} is {type(member).__name__} ({member!r}). "
|
|
178
|
+
f'Quote it (for example ["{member}"]) to match the cell as text.',
|
|
179
|
+
code="comparison-membership-member-not-a-string",
|
|
180
|
+
)
|
|
181
|
+
return data
|
|
125
182
|
|
|
126
183
|
@model_validator(mode="after")
|
|
127
184
|
def comparison_datatypes(self: Self) -> Self:
|
|
128
185
|
x: Comparisons = self.comparison
|
|
129
|
-
y: str | int | float = self.comparator
|
|
186
|
+
y: str | int | float | list[str] = self.comparator
|
|
130
187
|
|
|
131
|
-
if x == Comparisons.
|
|
188
|
+
if x == Comparisons.JOIN or x == Comparisons.ANTIJOIN:
|
|
189
|
+
if self.regex:
|
|
190
|
+
raise TablassertValidationError(
|
|
191
|
+
"`regex` is only meaningful with the `eq`/`ne` comparisons; `join`/`antijoin` match identifiers by exact string identity against a list.",
|
|
192
|
+
code="comparison-regex-requires-eq-ne",
|
|
193
|
+
)
|
|
194
|
+
if not isinstance(y, list):
|
|
195
|
+
raise TablassertValidationError(
|
|
196
|
+
f"`join`/`antijoin` comparisons require a list of identifier strings, got {type(y).__name__}.",
|
|
197
|
+
code="comparison-membership-comparator-required",
|
|
198
|
+
)
|
|
199
|
+
if not y:
|
|
200
|
+
raise TablassertValidationError(
|
|
201
|
+
"`join`/`antijoin` comparators must hold at least one identifier.", code="comparison-membership-empty"
|
|
202
|
+
)
|
|
203
|
+
elif x == Comparisons.NE or x == Comparisons.EQ:
|
|
132
204
|
if not isinstance(y, str):
|
|
133
205
|
raise TablassertValidationError(
|
|
134
206
|
f"`eq`/`ne` comparisons require a str comparator, got {type(y).__name__}.", code="comparison-bad-comparator-type"
|
|
135
207
|
)
|
|
208
|
+
if self.regex:
|
|
209
|
+
# An empty-or-everything pattern compiles, but it matches EVERY cell, so a `ne` filter
|
|
210
|
+
# would silently drop the whole table. Reject whitespace-only spellings and any pattern
|
|
211
|
+
# that can match the empty string (`^`, `()`, `.*`, `x*`) with the same code the
|
|
212
|
+
# `exclude_regex` probe uses.
|
|
213
|
+
if not y.strip():
|
|
214
|
+
raise TablassertValidationError(
|
|
215
|
+
f"`comparator` must be a non-empty pattern when `regex` is true (an empty pattern matches every cell), got {y!r}.",
|
|
216
|
+
code="regex-bad-pattern",
|
|
217
|
+
)
|
|
218
|
+
try:
|
|
219
|
+
pl.Series([""]).str.contains(y)
|
|
220
|
+
except Exception as e:
|
|
221
|
+
raise TablassertValidationError(
|
|
222
|
+
f"`comparator` must be a polars-compatible regex when `regex` is true, got {y!r}: {e}", code="regex-bad-pattern"
|
|
223
|
+
) from e
|
|
224
|
+
if pl.Series([""]).str.contains(y).item():
|
|
225
|
+
raise TablassertValidationError(
|
|
226
|
+
f"`comparator` must not match an empty cell when `regex` is true (it would match every cell, so an `ne` filter would drop the whole table), got {y!r}.",
|
|
227
|
+
code="regex-bad-pattern",
|
|
228
|
+
)
|
|
136
229
|
else:
|
|
230
|
+
if self.regex:
|
|
231
|
+
raise TablassertValidationError(
|
|
232
|
+
"`regex` is only meaningful with the `eq`/`ne` comparisons; ordering comparisons cast the column to Float64 and compare numerically.",
|
|
233
|
+
code="comparison-regex-requires-eq-ne",
|
|
234
|
+
)
|
|
137
235
|
if not isinstance(y, (int, float)):
|
|
138
236
|
raise TablassertValidationError(
|
|
139
237
|
f"Comparisons other than `eq`/`ne` require a float or int comparator, got {type(y).__name__}.",
|
|
@@ -258,6 +258,9 @@ OP_COST: dict[Callable, int] = {
|
|
|
258
258
|
lib.crop: 1,
|
|
259
259
|
lib.pick: 1,
|
|
260
260
|
lib.reindex: 1,
|
|
261
|
+
# Membership filter: one hash join against a small key frame, same single-predicate price.
|
|
262
|
+
lib.reindex_membership: 1,
|
|
263
|
+
lib.reindex_regex: 1,
|
|
261
264
|
# head samples min(HEAD_ROWS, height) rows, so it needs a height pass before slicing.
|
|
262
265
|
lib.head: 2,
|
|
263
266
|
# Encode: each is one lazy expression over a single column.
|
|
@@ -20,6 +20,7 @@ _MESSAGES: dict[str, str] = {
|
|
|
20
20
|
"unnamed-nodes": "nodes with no name or an empty name",
|
|
21
21
|
"unidentified-nodes": "nodes with no id or an empty id",
|
|
22
22
|
"incomplete-edges": "edges missing subject, predicate, or object",
|
|
23
|
+
"self-loop-edges": "edges whose subject and object are the same CURIE",
|
|
23
24
|
"demoted-edges": "edges demoted to bare biolink:Association despite a declared category_override",
|
|
24
25
|
"undeclared-nodes": "nodes referenced by edges but not declared in the nodes file",
|
|
25
26
|
"isolated-nodes": "declared nodes participating in no edge",
|
|
@@ -61,6 +62,9 @@ class _FileScan:
|
|
|
61
62
|
#: predicate. A demotion is only a violation when the build declared a
|
|
62
63
|
#: ``category_override`` (see :func:`study_kgx`); the count is collected either way.
|
|
63
64
|
demoted: Counter[str]
|
|
65
|
+
#: Edges that assert a concept relates to itself, keyed by the shared CURIE. See
|
|
66
|
+
#: :func:`study_kgx` for why a self-referential edge is a violation.
|
|
67
|
+
self_loops: Counter[str]
|
|
64
68
|
malformed: int
|
|
65
69
|
missing: bool
|
|
66
70
|
path: Path
|
|
@@ -94,17 +98,18 @@ def _scan_ndjson(path: Path, *, edge: bool) -> _FileScan:
|
|
|
94
98
|
|
|
95
99
|
Args:
|
|
96
100
|
path: Path to a ``.nodes.ndjson`` or ``.edges.ndjson`` file.
|
|
97
|
-
edge: ``True`` to collect referenced ids from ``subject``/``object
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
+
edge: ``True`` to collect referenced ids from ``subject``/``object``, assert the
|
|
102
|
+
three core edge slots are present, and tally self-referential
|
|
103
|
+
(``subject == object``) edges; ``False`` to collect declared node ``id``s and
|
|
104
|
+
track duplicates, nodes with no name or an empty name, and nodes with no id
|
|
105
|
+
or an empty id.
|
|
101
106
|
|
|
102
107
|
Returns:
|
|
103
108
|
A :class:`_FileScan`; ``missing`` is set (and nothing else) when the
|
|
104
109
|
file does not exist, so a typo'd path can never read as a clean pass.
|
|
105
110
|
"""
|
|
106
111
|
scan: _FileScan = _FileScan(
|
|
107
|
-
set(), set(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), 0, not path.is_file(), path
|
|
112
|
+
set(), set(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), 0, not path.is_file(), path
|
|
108
113
|
)
|
|
109
114
|
if scan.missing:
|
|
110
115
|
return scan
|
|
@@ -162,6 +167,19 @@ def _scan_ndjson(path: Path, *, edge: bool) -> _FileScan:
|
|
|
162
167
|
slot_value: object = record.get(slot)
|
|
163
168
|
if slot_value is None or (isinstance(slot_value, str) and not slot_value.strip()):
|
|
164
169
|
scan.incomplete[slot] += 1
|
|
170
|
+
# A self-referential edge states that a concept relates to itself. In a
|
|
171
|
+
# table-derived graph that is almost never the claim the author made: it is
|
|
172
|
+
# the signature of a resolution collapse, where two differently worded cells
|
|
173
|
+
# (or one cell duplicated into both slots) land on the same CURIE -- the same
|
|
174
|
+
# failure mode that makes two rows share an edge id. Such an edge carries no
|
|
175
|
+
# relational information, so downstream consumers either drop it silently or
|
|
176
|
+
# count it as evidence. Only a non-empty string pair is flagged; absent or
|
|
177
|
+
# blank slots are already reported by ``incomplete-edges``, and a blank-vs-blank
|
|
178
|
+
# pair must not read as "the same CURIE".
|
|
179
|
+
subject: object = record.get("subject")
|
|
180
|
+
object_: object = record.get("object")
|
|
181
|
+
if isinstance(subject, str) and isinstance(object_, str) and subject == object_ and subject.strip():
|
|
182
|
+
scan.self_loops[subject] += 1
|
|
165
183
|
# Bare ``biolink:Association`` means the row lost every specific class:
|
|
166
184
|
# ``lib.edge_category`` derives a class from the (subject role, object
|
|
167
185
|
# role) pair -- optionally pinned per object category by the section's
|
|
@@ -215,7 +233,8 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
|
|
|
215
233
|
Streams both files once each and checks: duplicate node ids, duplicate edge
|
|
216
234
|
ids, nodes with no
|
|
217
235
|
name or an empty name, nodes with no id or an empty id, edges missing any
|
|
218
|
-
of ``subject``/``predicate``/``object``,
|
|
236
|
+
of ``subject``/``predicate``/``object``, edges whose ``subject`` and ``object`` are
|
|
237
|
+
the same CURIE (self-referential), nodes referenced by edges but
|
|
219
238
|
never declared (``undeclared``), declared nodes participating in no edge
|
|
220
239
|
(``isolated``), empty/malformed lines, string values carrying leading/trailing
|
|
221
240
|
whitespace, and null or empty values in any field (a stronger contract than
|
|
@@ -229,6 +248,15 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
|
|
|
229
248
|
``has_supporting_studies``. Every check is an assertion -- the caller decides
|
|
230
249
|
whether violations fail the build.
|
|
231
250
|
|
|
251
|
+
A self-referential edge (``subject`` and ``object`` resolved to one CURIE) is an
|
|
252
|
+
assertion that a concept relates to itself. Table-derived graphs produce them almost
|
|
253
|
+
always by accident: two differently worded cells collapsing onto the same identifier,
|
|
254
|
+
or one cell filling both statement slots, so the edge carries no relational content and
|
|
255
|
+
ships as noise a consumer has to filter. QC therefore reports it rather than quietly
|
|
256
|
+
deleting it, so the author sees which CURIEs collapsed and fixes the mapping (or drops
|
|
257
|
+
the row) in the config. Intentionally reflexive assertions are a known accepted caveat:
|
|
258
|
+
they fail a ``--qc`` build like every other violation.
|
|
259
|
+
|
|
232
260
|
Args:
|
|
233
261
|
nodes_path: Path to ``<name>_<version>.nodes.ndjson``.
|
|
234
262
|
edges_path: Path to ``<name>_<version>.edges.ndjson``.
|
|
@@ -270,6 +298,9 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
|
|
|
270
298
|
if edges.incomplete:
|
|
271
299
|
examples = [f"{slot} ({n})" for slot, n in edges.incomplete.most_common(example_limit)]
|
|
272
300
|
violations.append(StudyViolation("incomplete-edges", "edges", sum(edges.incomplete.values()), examples))
|
|
301
|
+
if edges.self_loops:
|
|
302
|
+
examples = [f"{curie} ({n})" for curie, n in edges.self_loops.most_common(example_limit)]
|
|
303
|
+
violations.append(StudyViolation("self-loop-edges", "edges", sum(edges.self_loops.values()), examples))
|
|
273
304
|
if category_override_declared and edges.demoted:
|
|
274
305
|
examples = [f"{predicate} ({n})" for predicate, n in edges.demoted.most_common(example_limit)]
|
|
275
306
|
violations.append(StudyViolation("demoted-edges", "edges", sum(edges.demoted.values()), examples))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|