tablassert 22.1.0__tar.gz → 22.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-22.1.0 → tablassert-22.2.1}/PKG-INFO +2 -7
- {tablassert-22.1.0 → tablassert-22.2.1}/README.md +1 -6
- {tablassert-22.1.0 → tablassert-22.2.1}/pyproject.toml +1 -1
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/Cargo.lock +52 -57
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/Cargo.toml +3 -3
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/enums.py +4 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/errors.py +8 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/lib.py +177 -10
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/models.py +102 -4
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/runcache.py +3 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/study.py +37 -6
- {tablassert-22.1.0 → tablassert-22.2.1}/LICENSE +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/examples/count_tables.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/src/fullmap.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/src/json.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/src/lib.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/src/ndjson.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/src/nlp.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/src/uuid.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/tests/build_golden.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/tests/common/mod.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/__init__.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/_lazy.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/biolink.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/cli.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/cli_entry.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/coerce.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/data/infores_catalog.yaml +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/extras.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/fullmap.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/infores_registry.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/ingests.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/log.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/net.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/nlp.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/progress.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/qc.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/rig.py +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/rs.pyi +0 -0
- {tablassert-22.1.0 → tablassert-22.2.1}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version: 22.1
|
|
3
|
+
Version: 22.2.1
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -220,12 +220,7 @@ per-command preflight behavior, and the
|
|
|
220
220
|
from pathlib import Path
|
|
221
221
|
from tablassert.lib import resolve_many
|
|
222
222
|
|
|
223
|
-
results = resolve_many(
|
|
224
|
-
col="gene",
|
|
225
|
-
entities=["TP53", "BRCA1"],
|
|
226
|
-
fullmap=Path("/path/to/fullmap"),
|
|
227
|
-
taxon="9606",
|
|
228
|
-
)
|
|
223
|
+
results = resolve_many(col="gene", entities=["TP53", "BRCA1"], fullmap=Path("/path/to/fullmap"), taxon="9606")
|
|
229
224
|
# [{"original_gene": "TP53", "gene": "HGNC:11998", "gene_name": "TP53", ...}, ...]
|
|
230
225
|
```
|
|
231
226
|
|
|
@@ -166,12 +166,7 @@ per-command preflight behavior, and the
|
|
|
166
166
|
from pathlib import Path
|
|
167
167
|
from tablassert.lib import resolve_many
|
|
168
168
|
|
|
169
|
-
results = resolve_many(
|
|
170
|
-
col="gene",
|
|
171
|
-
entities=["TP53", "BRCA1"],
|
|
172
|
-
fullmap=Path("/path/to/fullmap"),
|
|
173
|
-
taxon="9606",
|
|
174
|
-
)
|
|
169
|
+
results = resolve_many(col="gene", entities=["TP53", "BRCA1"], fullmap=Path("/path/to/fullmap"), taxon="9606")
|
|
175
170
|
# [{"original_gene": "TP53", "gene": "HGNC:11998", "gene_name": "TP53", ...}, ...]
|
|
176
171
|
```
|
|
177
172
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "22.1
|
|
3
|
+
version = "22.2.1"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -2,12 +2,6 @@
|
|
|
2
2
|
# It is not intended for manual editing.
|
|
3
3
|
version = 4
|
|
4
4
|
|
|
5
|
-
[[package]]
|
|
6
|
-
name = "adler2"
|
|
7
|
-
version = "2.0.1"
|
|
8
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
-
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
|
10
|
-
|
|
11
5
|
[[package]]
|
|
12
6
|
name = "bincode"
|
|
13
7
|
version = "1.3.3"
|
|
@@ -147,14 +141,19 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
|
|
147
141
|
|
|
148
142
|
[[package]]
|
|
149
143
|
name = "flate2"
|
|
150
|
-
version = "1.1.
|
|
144
|
+
version = "1.1.10"
|
|
151
145
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
152
|
-
checksum = "
|
|
146
|
+
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
|
153
147
|
dependencies = [
|
|
154
|
-
"miniz_oxide",
|
|
155
148
|
"zlib-rs",
|
|
156
149
|
]
|
|
157
150
|
|
|
151
|
+
[[package]]
|
|
152
|
+
name = "foldhash"
|
|
153
|
+
version = "0.2.0"
|
|
154
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
155
|
+
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
|
156
|
+
|
|
158
157
|
[[package]]
|
|
159
158
|
name = "futures-core"
|
|
160
159
|
version = "0.3.32"
|
|
@@ -288,9 +287,9 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
|
288
287
|
|
|
289
288
|
[[package]]
|
|
290
289
|
name = "memmap2"
|
|
291
|
-
version = "0.9.
|
|
290
|
+
version = "0.9.12"
|
|
292
291
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
293
|
-
checksum = "
|
|
292
|
+
checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157"
|
|
294
293
|
dependencies = [
|
|
295
294
|
"libc",
|
|
296
295
|
]
|
|
@@ -304,16 +303,6 @@ dependencies = [
|
|
|
304
303
|
"libmimalloc-sys",
|
|
305
304
|
]
|
|
306
305
|
|
|
307
|
-
[[package]]
|
|
308
|
-
name = "miniz_oxide"
|
|
309
|
-
version = "0.8.9"
|
|
310
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
311
|
-
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
|
312
|
-
dependencies = [
|
|
313
|
-
"adler2",
|
|
314
|
-
"simd-adler32",
|
|
315
|
-
]
|
|
316
|
-
|
|
317
306
|
[[package]]
|
|
318
307
|
name = "once_cell"
|
|
319
308
|
version = "1.21.4"
|
|
@@ -349,9 +338,9 @@ dependencies = [
|
|
|
349
338
|
|
|
350
339
|
[[package]]
|
|
351
340
|
name = "pyo3"
|
|
352
|
-
version = "0.
|
|
341
|
+
version = "0.30.1"
|
|
353
342
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
354
|
-
checksum = "
|
|
343
|
+
checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4"
|
|
355
344
|
dependencies = [
|
|
356
345
|
"libc",
|
|
357
346
|
"once_cell",
|
|
@@ -363,18 +352,18 @@ dependencies = [
|
|
|
363
352
|
|
|
364
353
|
[[package]]
|
|
365
354
|
name = "pyo3-build-config"
|
|
366
|
-
version = "0.
|
|
355
|
+
version = "0.30.1"
|
|
367
356
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
368
|
-
checksum = "
|
|
357
|
+
checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700"
|
|
369
358
|
dependencies = [
|
|
370
359
|
"target-lexicon",
|
|
371
360
|
]
|
|
372
361
|
|
|
373
362
|
[[package]]
|
|
374
363
|
name = "pyo3-ffi"
|
|
375
|
-
version = "0.
|
|
364
|
+
version = "0.30.1"
|
|
376
365
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
377
|
-
checksum = "
|
|
366
|
+
checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4"
|
|
378
367
|
dependencies = [
|
|
379
368
|
"libc",
|
|
380
369
|
"pyo3-build-config",
|
|
@@ -382,26 +371,26 @@ dependencies = [
|
|
|
382
371
|
|
|
383
372
|
[[package]]
|
|
384
373
|
name = "pyo3-macros"
|
|
385
|
-
version = "0.
|
|
374
|
+
version = "0.30.1"
|
|
386
375
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
387
|
-
checksum = "
|
|
376
|
+
checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60"
|
|
388
377
|
dependencies = [
|
|
389
378
|
"proc-macro2",
|
|
390
379
|
"pyo3-macros-backend",
|
|
391
380
|
"quote",
|
|
392
|
-
"syn",
|
|
381
|
+
"syn 3.0.9",
|
|
393
382
|
]
|
|
394
383
|
|
|
395
384
|
[[package]]
|
|
396
385
|
name = "pyo3-macros-backend"
|
|
397
|
-
version = "0.
|
|
386
|
+
version = "0.30.1"
|
|
398
387
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
399
|
-
checksum = "
|
|
388
|
+
checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b"
|
|
400
389
|
dependencies = [
|
|
401
390
|
"heck",
|
|
402
391
|
"proc-macro2",
|
|
403
392
|
"quote",
|
|
404
|
-
"syn",
|
|
393
|
+
"syn 3.0.9",
|
|
405
394
|
]
|
|
406
395
|
|
|
407
396
|
[[package]]
|
|
@@ -449,9 +438,9 @@ dependencies = [
|
|
|
449
438
|
|
|
450
439
|
[[package]]
|
|
451
440
|
name = "rlimit"
|
|
452
|
-
version = "0.
|
|
441
|
+
version = "0.11.0"
|
|
453
442
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
454
|
-
checksum = "
|
|
443
|
+
checksum = "f35ee2729c56bb610f6dba436bf78135f728b7373bdffae2ec815b2d3eb98cc3"
|
|
455
444
|
dependencies = [
|
|
456
445
|
"libc",
|
|
457
446
|
]
|
|
@@ -468,9 +457,9 @@ dependencies = [
|
|
|
468
457
|
|
|
469
458
|
[[package]]
|
|
470
459
|
name = "rustc-hash"
|
|
471
|
-
version = "
|
|
460
|
+
version = "2.1.3"
|
|
472
461
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
473
|
-
checksum = "
|
|
462
|
+
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
|
474
463
|
|
|
475
464
|
[[package]]
|
|
476
465
|
name = "rustix"
|
|
@@ -493,9 +482,9 @@ checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
|
|
493
482
|
|
|
494
483
|
[[package]]
|
|
495
484
|
name = "serde"
|
|
496
|
-
version = "1.0.
|
|
485
|
+
version = "1.0.229"
|
|
497
486
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
498
|
-
checksum = "
|
|
487
|
+
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
|
499
488
|
dependencies = [
|
|
500
489
|
"serde_core",
|
|
501
490
|
"serde_derive",
|
|
@@ -503,30 +492,31 @@ dependencies = [
|
|
|
503
492
|
|
|
504
493
|
[[package]]
|
|
505
494
|
name = "serde_core"
|
|
506
|
-
version = "1.0.
|
|
495
|
+
version = "1.0.229"
|
|
507
496
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
508
|
-
checksum = "
|
|
497
|
+
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
|
509
498
|
dependencies = [
|
|
510
499
|
"serde_derive",
|
|
511
500
|
]
|
|
512
501
|
|
|
513
502
|
[[package]]
|
|
514
503
|
name = "serde_derive"
|
|
515
|
-
version = "1.0.
|
|
504
|
+
version = "1.0.229"
|
|
516
505
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
517
|
-
checksum = "
|
|
506
|
+
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|
518
507
|
dependencies = [
|
|
519
508
|
"proc-macro2",
|
|
520
509
|
"quote",
|
|
521
|
-
"syn",
|
|
510
|
+
"syn 3.0.9",
|
|
522
511
|
]
|
|
523
512
|
|
|
524
513
|
[[package]]
|
|
525
514
|
name = "serde_json"
|
|
526
|
-
version = "1.0.
|
|
515
|
+
version = "1.0.154"
|
|
527
516
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
528
|
-
checksum = "
|
|
517
|
+
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
|
529
518
|
dependencies = [
|
|
519
|
+
"foldhash",
|
|
530
520
|
"indexmap",
|
|
531
521
|
"itoa",
|
|
532
522
|
"memchr",
|
|
@@ -541,12 +531,6 @@ version = "2.0.1"
|
|
|
541
531
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
542
532
|
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
|
543
533
|
|
|
544
|
-
[[package]]
|
|
545
|
-
name = "simd-adler32"
|
|
546
|
-
version = "0.3.10"
|
|
547
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
548
|
-
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
|
549
|
-
|
|
550
534
|
[[package]]
|
|
551
535
|
name = "slab"
|
|
552
536
|
version = "0.4.12"
|
|
@@ -564,6 +548,17 @@ dependencies = [
|
|
|
564
548
|
"unicode-ident",
|
|
565
549
|
]
|
|
566
550
|
|
|
551
|
+
[[package]]
|
|
552
|
+
name = "syn"
|
|
553
|
+
version = "3.0.9"
|
|
554
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
555
|
+
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
|
556
|
+
dependencies = [
|
|
557
|
+
"proc-macro2",
|
|
558
|
+
"quote",
|
|
559
|
+
"unicode-ident",
|
|
560
|
+
]
|
|
561
|
+
|
|
567
562
|
[[package]]
|
|
568
563
|
name = "tablassert-rs"
|
|
569
564
|
version = "0.1.0"
|
|
@@ -631,9 +626,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
|
|
631
626
|
|
|
632
627
|
[[package]]
|
|
633
628
|
name = "uuid"
|
|
634
|
-
version = "1.
|
|
629
|
+
version = "1.29.0"
|
|
635
630
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
636
|
-
checksum = "
|
|
631
|
+
checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514"
|
|
637
632
|
dependencies = [
|
|
638
633
|
"js-sys",
|
|
639
634
|
"md-5",
|
|
@@ -678,7 +673,7 @@ dependencies = [
|
|
|
678
673
|
"bumpalo",
|
|
679
674
|
"proc-macro2",
|
|
680
675
|
"quote",
|
|
681
|
-
"syn",
|
|
676
|
+
"syn 2.0.119",
|
|
682
677
|
"wasm-bindgen-shared",
|
|
683
678
|
]
|
|
684
679
|
|
|
@@ -718,9 +713,9 @@ dependencies = [
|
|
|
718
713
|
|
|
719
714
|
[[package]]
|
|
720
715
|
name = "xxhash-rust"
|
|
721
|
-
version = "0.8.
|
|
716
|
+
version = "0.8.19"
|
|
722
717
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
723
|
-
checksum = "
|
|
718
|
+
checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7"
|
|
724
719
|
|
|
725
720
|
[[package]]
|
|
726
721
|
name = "zlib-rs"
|
|
@@ -21,7 +21,7 @@ bincode = "1"
|
|
|
21
21
|
flate2 = { version = "1", features = ["zlib-rs"], default-features = false }
|
|
22
22
|
memmap2 = "0.9"
|
|
23
23
|
mimalloc = { version = "0.1", default-features = false }
|
|
24
|
-
pyo3 = "0.
|
|
24
|
+
pyo3 = "0.30"
|
|
25
25
|
rayon = "1"
|
|
26
26
|
# Pinned to cberner/redb master (the 4.2.0-to-be, rev a35e7cc 2026-08-10) for
|
|
27
27
|
# two Phase-4 wins not yet published on crates.io (latest stable is 4.1.0):
|
|
@@ -33,8 +33,8 @@ rayon = "1"
|
|
|
33
33
|
# bump, existing DBs keep opening). Swap back to `redb = "4.2"` once 4.2.0
|
|
34
34
|
# publishes and re-verify the changelog.
|
|
35
35
|
redb = { git = "https://github.com/cberner/redb.git", rev = "a35e7cc86f191d08a444d5973469b34673d586fc", features = ["experimental_cursor"] }
|
|
36
|
-
rlimit = "0.
|
|
37
|
-
rustc-hash = "
|
|
36
|
+
rlimit = "0.11"
|
|
37
|
+
rustc-hash = "2"
|
|
38
38
|
# English (Porter2) stemmer for `nlp::normalize_l1` term normalization: pure
|
|
39
39
|
# Rust (no dictionaries, no C deps) and fast enough for the per-token hot path;
|
|
40
40
|
# it requires already-lowercased input, which the pipeline guarantees by
|
|
@@ -42,6 +42,10 @@ class Comparisons(str, Enum):
|
|
|
42
42
|
LE = "le"
|
|
43
43
|
EQ = "eq"
|
|
44
44
|
NE = "ne"
|
|
45
|
+
# Membership comparisons: the comparator is a list of identifiers, matched against the
|
|
46
|
+
# raw (pre-resolution) cell value. JOIN keeps matching rows, ANTIJOIN drops them.
|
|
47
|
+
JOIN = "join"
|
|
48
|
+
ANTIJOIN = "antijoin"
|
|
45
49
|
|
|
46
50
|
|
|
47
51
|
class Functions(str, Enum):
|
|
@@ -56,6 +56,14 @@ TablassertErrorCodes = Literal[
|
|
|
56
56
|
"config-rows-and-row-slice-conflict",
|
|
57
57
|
"comparison-bad-comparator-type",
|
|
58
58
|
"comparison-nonnumeric-comparator",
|
|
59
|
+
# Membership (join/antijoin) family: the first two are config-validation failures on the
|
|
60
|
+
# `comparator` field, the last two are op-level failures on the column being tested.
|
|
61
|
+
"comparison-membership-comparator-required",
|
|
62
|
+
"comparison-membership-empty",
|
|
63
|
+
"comparison-membership-member-not-a-string",
|
|
64
|
+
"reindex-membership-column-missing",
|
|
65
|
+
"reindex-membership-column-dtype-unsupported",
|
|
66
|
+
"comparison-regex-requires-eq-ne",
|
|
59
67
|
"encoding-bad-excel-column",
|
|
60
68
|
"regex-bad-pattern",
|
|
61
69
|
"regex-bad-replacement",
|
|
@@ -45,11 +45,11 @@ from tablassert.coerce import (
|
|
|
45
45
|
sig,
|
|
46
46
|
study_size_target,
|
|
47
47
|
)
|
|
48
|
-
from tablassert.enums import EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
|
|
48
|
+
from tablassert.enums import Comparisons, EncodingMethods, Files, Functions, InformationResources, Repositories, Tokens
|
|
49
49
|
from tablassert.errors import TablassertError
|
|
50
50
|
from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
|
|
51
51
|
from tablassert.log import cat
|
|
52
|
-
from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
|
|
52
|
+
from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, Reindex, RIGConfig, Section
|
|
53
53
|
from tablassert.nlp import level_one, level_two
|
|
54
54
|
from tablassert.qc import fullmap_audit
|
|
55
55
|
from tablassert.rig import (
|
|
@@ -1070,6 +1070,178 @@ def reindex(df: pl.LazyFrame, col: str, op: Callable, comp: str | int | float, c
|
|
|
1070
1070
|
return df.filter(op(expr, comp))
|
|
1071
1071
|
|
|
1072
1072
|
|
|
1073
|
+
MEMBERSHIP_KEY_COLUMN: str = "_reindex_membership_key"
|
|
1074
|
+
"""Base name for the throwaway join key column on the cast branch; extended with underscores until
|
|
1075
|
+
it is unused if a source frame already carries it, and dropped again before the op returns."""
|
|
1076
|
+
|
|
1077
|
+
MEMBERSHIP_UNCASTABLE_DTYPES: frozenset[str] = frozenset({"List", "Array", "Struct", "Object", "Binary"})
|
|
1078
|
+
"""polars dtype class NAMES whose cells have no single-string identity, so membership is refused.
|
|
1079
|
+
|
|
1080
|
+
Matched by name because a module-level ``pl.List`` reference would resolve the lazy proxy and import
|
|
1081
|
+
polars at ``import tablassert.lib``, which is exactly what :mod:`tablassert._lazy` exists to avoid.
|
|
1082
|
+
Every parametrized variant reports its base name (``List(String)`` is named ``List``).
|
|
1083
|
+
|
|
1084
|
+
A ``List``/``Array`` cell cannot be cast to ``String`` at all (``InvalidOperationError``), an
|
|
1085
|
+
``Object`` cell casts through python and never compares equal to an identifier, a ``Struct`` casts to
|
|
1086
|
+
a brace literal no identifier is spelled to match, and a ``Binary`` cell casts only when it happens to
|
|
1087
|
+
be valid UTF-8, so one list would match some rows and raise on the rest. Joining list cells on a
|
|
1088
|
+
separator instead would make a multi-value cell never match a single identifier, a silent wrong
|
|
1089
|
+
answer, so these dtypes raise a coded error.
|
|
1090
|
+
"""
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
def _membership_key_column(schema: pl.Schema) -> str:
|
|
1094
|
+
"""Return a join key column name the frame does not already use.
|
|
1095
|
+
|
|
1096
|
+
Args:
|
|
1097
|
+
schema: Collected schema of the frame being filtered.
|
|
1098
|
+
|
|
1099
|
+
Returns:
|
|
1100
|
+
:data:`MEMBERSHIP_KEY_COLUMN`, or that name extended with underscores until it is free.
|
|
1101
|
+
"""
|
|
1102
|
+
key_col: str = MEMBERSHIP_KEY_COLUMN
|
|
1103
|
+
while key_col in schema:
|
|
1104
|
+
key_col = f"{key_col}_"
|
|
1105
|
+
return key_col
|
|
1106
|
+
|
|
1107
|
+
|
|
1108
|
+
def reindex_membership(df: pl.LazyFrame, col: str, keys: list[str], anti: bool = False) -> pl.LazyFrame:
|
|
1109
|
+
"""Reindex a LazyFrame by exact membership in a list of identifiers (join / antijoin).
|
|
1110
|
+
|
|
1111
|
+
Args:
|
|
1112
|
+
df: Source LazyFrame.
|
|
1113
|
+
col: Column whose raw (pre-resolution) cell values are tested.
|
|
1114
|
+
keys: Identifier list. Duplicates are ignored: a semi/anti join cannot multiply left
|
|
1115
|
+
rows, and the join build side is cheaper once deduplicated.
|
|
1116
|
+
anti: ``False`` (join) keeps rows whose value is in ``keys``; ``True`` (antijoin) keeps
|
|
1117
|
+
the rows that are not.
|
|
1118
|
+
|
|
1119
|
+
Returns:
|
|
1120
|
+
Filtered LazyFrame keeping the surviving rows in left-frame order, with every original
|
|
1121
|
+
column and its original dtype.
|
|
1122
|
+
|
|
1123
|
+
Raises:
|
|
1124
|
+
TablassertError: ``comparison-membership-empty`` for an empty list,
|
|
1125
|
+
``reindex-membership-column-missing`` when the source lacks ``col``, and
|
|
1126
|
+
``reindex-membership-column-dtype-unsupported`` for cells with no single string form
|
|
1127
|
+
(``List``, ``Array``, ``Struct``, ``Object``, ``Binary``).
|
|
1128
|
+
|
|
1129
|
+
Notes:
|
|
1130
|
+
Why a join and not ``is_in``: on the pinned polars (``>=2.0.0rc2``) a semi/anti hash join
|
|
1131
|
+
against a once-built key frame is the faster measured path once the list is non-trivial (1M
|
|
1132
|
+
rows, median of three collects: 100k keys 0.018s vs 0.029s filtered; 1k keys 0.003s vs
|
|
1133
|
+
0.004s; a tie at ten keys), and it is the membership idiom already used in
|
|
1134
|
+
:mod:`tablassert.qc`. No ``is_in`` fast path exists for tiny lists: it propagates nulls
|
|
1135
|
+
differently from antijoin, and the gap is milliseconds per million rows.
|
|
1136
|
+
|
|
1137
|
+
``maintain_order="left"`` is explicit, not incidental: polars documents the default order as
|
|
1138
|
+
unspecified ("might differ across Polars versions or even between different runs"), and the
|
|
1139
|
+
dependency is an open range, so a future 2.x must not be able to reshuffle a section's rows.
|
|
1140
|
+
Measured free here (1M rows / 100k keys, five runs, median): semi 5.1ms -> 4.6ms, anti 5.4ms
|
|
1141
|
+
-> 5.4ms.
|
|
1142
|
+
|
|
1143
|
+
The string path joins straight on ``col``, because semi/anti joins emit only the left frame's
|
|
1144
|
+
columns. Only the cast path adds a throwaway helper column, and it is dropped again, so
|
|
1145
|
+
entity resolution downstream sees the cell exactly as the source produced it.
|
|
1146
|
+
|
|
1147
|
+
Null semantics are join semantics and deliberately differ from ``ne``: a null cell never
|
|
1148
|
+
matches, so join drops it while antijoin KEEPS it.
|
|
1149
|
+
|
|
1150
|
+
Membership is textual, so an identifier must be spelled the way polars' cast renders it.
|
|
1151
|
+
Measured forms: ``Int64`` 3 -> ``"3"``; ``Float64`` 3.0 -> ``"3.0"`` (1e+20 keeps scientific
|
|
1152
|
+
form); ``Bool`` -> ``"true"``/``"false"``; ``Date`` -> ``"2024-01-15"``; ``Datetime`` ->
|
|
1153
|
+
``"2024-01-15 06:24:30.000000"`` (space, not ``T``, with microsecond precision); ``Time`` ->
|
|
1154
|
+
``"01:02:03"``; ``Decimal(10,2)`` 1.5 -> ``"1.50"`` (scale kept). A misspelled list is not an
|
|
1155
|
+
error: join emits an empty section and antijoin emits every row. No zero-match warning is
|
|
1156
|
+
emitted, because detecting it would cost an extra pass over the frame on every build.
|
|
1157
|
+
"""
|
|
1158
|
+
if not keys:
|
|
1159
|
+
raise TablassertError("`join`/`antijoin` reindex requires at least one identifier.", code="comparison-membership-empty")
|
|
1160
|
+
|
|
1161
|
+
schema: pl.Schema = df.collect_schema()
|
|
1162
|
+
if col not in schema:
|
|
1163
|
+
raise TablassertError(f"`join`/`antijoin` reindex targets `{col}`, which the source does not have.", code="reindex-membership-column-missing")
|
|
1164
|
+
if type(schema[col]).__name__ in MEMBERSHIP_UNCASTABLE_DTYPES:
|
|
1165
|
+
raise TablassertError(
|
|
1166
|
+
f"`join`/`antijoin` reindex cannot test `{col}`: {schema[col]} cells have no single string form to match identifiers against.",
|
|
1167
|
+
code="reindex-membership-column-dtype-unsupported",
|
|
1168
|
+
)
|
|
1169
|
+
|
|
1170
|
+
unique: list[str] = list(dict.fromkeys(keys))
|
|
1171
|
+
how: Literal["semi", "anti"] = "anti" if anti else "semi"
|
|
1172
|
+
if isinstance(schema[col], pl.String):
|
|
1173
|
+
# No helper column: the join returns the left frame's columns unchanged.
|
|
1174
|
+
return df.join(pl.DataFrame({col: unique}).lazy(), on=col, how=how, maintain_order="left")
|
|
1175
|
+
key_col: str = _membership_key_column(schema)
|
|
1176
|
+
left: pl.LazyFrame = df.with_columns(pl.col(col).cast(pl.String, strict=False).alias(key_col))
|
|
1177
|
+
return left.join(pl.DataFrame({key_col: unique}).lazy(), on=key_col, how=how, maintain_order="left").drop(key_col)
|
|
1178
|
+
|
|
1179
|
+
|
|
1180
|
+
def reindex_regex(df: pl.LazyFrame, col: str, pattern: str, negate: bool = False) -> pl.LazyFrame:
|
|
1181
|
+
"""Reindex a LazyFrame by filtering rows on a polars regex match against a column.
|
|
1182
|
+
|
|
1183
|
+
Args:
|
|
1184
|
+
df: Source LazyFrame.
|
|
1185
|
+
col: Column to test.
|
|
1186
|
+
pattern: Polars-compatible regex, matched unanchored and case-sensitively (write `^`/`$`
|
|
1187
|
+
to pin a whole cell).
|
|
1188
|
+
negate: When True keep the rows that do NOT match (the `ne` filter); when False keep the
|
|
1189
|
+
matches (the `eq` filter).
|
|
1190
|
+
|
|
1191
|
+
Returns:
|
|
1192
|
+
LazyFrame containing only rows whose predicate is true.
|
|
1193
|
+
|
|
1194
|
+
Notes:
|
|
1195
|
+
The column is cast to String first, so a typed source (parquet, or a numeric column in a
|
|
1196
|
+
text table) filters on its text form instead of raising polars' ``InvalidOperationError:
|
|
1197
|
+
expected String type``. A null cell matches nothing and ``not_()`` leaves it null, so null
|
|
1198
|
+
rows drop under both directions -- the same null rule literal `eq`/`ne` already have, since
|
|
1199
|
+
a null comparison is never true.
|
|
1200
|
+
"""
|
|
1201
|
+
match: pl.Expr = pl.col(col).cast(pl.String).str.contains(pattern)
|
|
1202
|
+
return df.filter(match.not_() if negate else match)
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
def reindex_op(entry: Reindex) -> tuple[Callable, tuple[Any, ...]]:
|
|
1206
|
+
"""Compile one ``Reindex`` config entry into its Tcode op tuple.
|
|
1207
|
+
|
|
1208
|
+
Args:
|
|
1209
|
+
entry: Validated ``Reindex`` model. The validator guarantees the pairing: a membership entry
|
|
1210
|
+
carries a non-empty ``list[str]`` comparator, ``regex`` only ever sits on ``eq``/``ne``,
|
|
1211
|
+
and a scalar entry carries a scalar comparator.
|
|
1212
|
+
|
|
1213
|
+
Returns:
|
|
1214
|
+
``(callable, args)`` for the pipeline: :func:`reindex_membership` for ``join``/``antijoin``,
|
|
1215
|
+
:func:`reindex_regex` for a ``regex`` entry, otherwise :func:`reindex` with the ``operator``
|
|
1216
|
+
module callable and the same ``cast`` choice it has always used (numeric comparisons cast to
|
|
1217
|
+
Float64, ``eq``/``ne`` do not).
|
|
1218
|
+
|
|
1219
|
+
Notes:
|
|
1220
|
+
Legacy entries keep emitting exactly the tuples they always did, so their run-cache op digests
|
|
1221
|
+
do not move; membership is the only new shape.
|
|
1222
|
+
|
|
1223
|
+
``Reindex.comparison`` carries the plain enum *value* at runtime because ``TablaBase`` sets
|
|
1224
|
+
``use_enum_values=True``, so this branches on the strings, exactly as the inline comprehension
|
|
1225
|
+
it replaces did.
|
|
1226
|
+
"""
|
|
1227
|
+
col: str = idxname(entry.column)
|
|
1228
|
+
comparison: str = entry.comparison
|
|
1229
|
+
if comparison in (Comparisons.JOIN.value, Comparisons.ANTIJOIN.value):
|
|
1230
|
+
if not isinstance(entry.comparator, list):
|
|
1231
|
+
raise TablassertError(
|
|
1232
|
+
f"`{comparison}` reindex requires a list of identifiers, got {type(entry.comparator).__name__}.",
|
|
1233
|
+
code="comparison-membership-comparator-required",
|
|
1234
|
+
)
|
|
1235
|
+
return (reindex_membership, (col, entry.comparator, comparison == Comparisons.ANTIJOIN.value))
|
|
1236
|
+
if entry.regex:
|
|
1237
|
+
# models.Reindex rejects `regex` off eq/ne, so this arm never needs the numeric cast or an
|
|
1238
|
+
# operator lookup.
|
|
1239
|
+
return (reindex_regex, (col, entry.comparator, comparison == Comparisons.NE.value))
|
|
1240
|
+
if comparison in (Comparisons.NE.value, Comparisons.EQ.value):
|
|
1241
|
+
return (reindex, (col, getattr(operator, comparison), entry.comparator, False))
|
|
1242
|
+
return (reindex, (col, getattr(operator, comparison), entry.comparator))
|
|
1243
|
+
|
|
1244
|
+
|
|
1073
1245
|
def drop_not_significant(lf: pl.LazyFrame, col: str = "statistical_significance_qualifier") -> pl.LazyFrame:
|
|
1074
1246
|
"""Drop release-mode edges whose significance qualifier is ``not_significant``.
|
|
1075
1247
|
|
|
@@ -1338,14 +1510,7 @@ class Tcode(Section):
|
|
|
1338
1510
|
(idx, ()),
|
|
1339
1511
|
(crop, (self.source.row_slice,)) if self.source.row_slice else None,
|
|
1340
1512
|
(pick, (self.source.rows,)) if self.source.rows else None,
|
|
1341
|
-
[
|
|
1342
|
-
(reindex, (idxname(x.column), getattr(operator, x.comparison), x.comparator))
|
|
1343
|
-
if x.comparison not in ["ne", "eq"]
|
|
1344
|
-
else (reindex, (idxname(x.column), getattr(operator, x.comparison), x.comparator, False))
|
|
1345
|
-
for x in self.source.reindex
|
|
1346
|
-
]
|
|
1347
|
-
if self.source.reindex
|
|
1348
|
-
else None,
|
|
1513
|
+
[reindex_op(x) for x in self.source.reindex] if self.source.reindex else None,
|
|
1349
1514
|
# --head preview: randomly sample min(HEAD_ROWS, height) rows before any encoding/resolve.
|
|
1350
1515
|
(head, (HEAD_ROWS,)) if self.head else None,
|
|
1351
1516
|
[
|
|
@@ -1588,6 +1753,8 @@ PHASE_OF: dict[Callable, str] = {
|
|
|
1588
1753
|
crop: "filter",
|
|
1589
1754
|
pick: "filter",
|
|
1590
1755
|
reindex: "filter",
|
|
1756
|
+
reindex_membership: "filter",
|
|
1757
|
+
reindex_regex: "filter",
|
|
1591
1758
|
head: "filter",
|
|
1592
1759
|
coerce_columns: "clean",
|
|
1593
1760
|
clean_numeric: "clean",
|
|
@@ -119,21 +119,119 @@ class TablaBase(BaseModel):
|
|
|
119
119
|
class Reindex(TablaBase):
|
|
120
120
|
column: str = Field(..., pattern=r"^[A-Z]{1,3}$", description="Source column letters used for row filtering.", examples=["A", "AA"])
|
|
121
121
|
comparison: Comparisons = Field(
|
|
122
|
-
Comparisons.NE,
|
|
122
|
+
Comparisons.NE,
|
|
123
|
+
description="Comparison operator used in reindex filtering.",
|
|
124
|
+
examples=[Comparisons.NE, Comparisons.EQ, Comparisons.GT, Comparisons.JOIN, Comparisons.ANTIJOIN],
|
|
125
|
+
)
|
|
126
|
+
comparator: str | int | float | list[str] = Field(
|
|
127
|
+
...,
|
|
128
|
+
description=(
|
|
129
|
+
"Right-side value compared against the selected column. For `join`/`antijoin` this is the list of "
|
|
130
|
+
"identifiers matched exactly against the raw (pre-resolution) cell value."
|
|
131
|
+
),
|
|
132
|
+
examples=["N/A", 0, 1.5, ["CHEBI:41774", "CHEBI:48217"]],
|
|
133
|
+
)
|
|
134
|
+
regex: bool = Field(
|
|
135
|
+
False,
|
|
136
|
+
description=(
|
|
137
|
+
"When true, `comparator` is a polars-compatible regex matched against the cell instead of a literal "
|
|
138
|
+
"value: `eq` keeps rows whose cell matches, `ne` keeps rows that do not. Matching is case-sensitive and "
|
|
139
|
+
"unanchored (add `^`/`$` to pin the whole cell), the column is compared as text, and a null cell never "
|
|
140
|
+
"matches, so null rows drop under both operators. Only valid with `eq`/`ne`; an invalid or empty pattern "
|
|
141
|
+
"is rejected at config time (`regex-bad-pattern`)."
|
|
142
|
+
),
|
|
123
143
|
)
|
|
124
|
-
|
|
144
|
+
|
|
145
|
+
@model_validator(mode="before")
|
|
146
|
+
@classmethod
|
|
147
|
+
def membership_members_are_strings(cls, data: Any) -> Any:
|
|
148
|
+
"""Reject an identifier list whose members are not strings, with a code rather than a field type error.
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
data: Raw config value for one ``Reindex`` entry.
|
|
152
|
+
|
|
153
|
+
Returns:
|
|
154
|
+
``data`` unchanged when the entry is not a membership list, or when every member is a
|
|
155
|
+
string.
|
|
156
|
+
|
|
157
|
+
Raises:
|
|
158
|
+
TablassertValidationError: ``comparison-membership-member-not-a-string`` when a
|
|
159
|
+
``join``/``antijoin`` list holds a non-string member.
|
|
160
|
+
|
|
161
|
+
Notes:
|
|
162
|
+
Without this guard the pydantic union rejects `comparator: [3, 4]` (the natural spelling
|
|
163
|
+
against an `Int64` column, whose string form is `3`) as a bare `string_type` field error
|
|
164
|
+
with no stable code and no docs URL, while every other failure in this family is coded.
|
|
165
|
+
The fix stays "quote the identifiers": membership is textual, so an unquoted number is a
|
|
166
|
+
config error rather than something to coerce silently.
|
|
167
|
+
"""
|
|
168
|
+
if not isinstance(data, dict):
|
|
169
|
+
return data
|
|
170
|
+
comparison = data.get("comparison")
|
|
171
|
+
comparator = data.get("comparator")
|
|
172
|
+
if comparison not in (Comparisons.JOIN.value, Comparisons.ANTIJOIN.value) or not isinstance(comparator, list):
|
|
173
|
+
return data
|
|
174
|
+
for index, member in enumerate(comparator):
|
|
175
|
+
if not isinstance(member, str):
|
|
176
|
+
raise TablassertValidationError(
|
|
177
|
+
f"`join`/`antijoin` identifier lists must hold quoted strings: member {index} is {type(member).__name__} ({member!r}). "
|
|
178
|
+
f'Quote it (for example ["{member}"]) to match the cell as text.',
|
|
179
|
+
code="comparison-membership-member-not-a-string",
|
|
180
|
+
)
|
|
181
|
+
return data
|
|
125
182
|
|
|
126
183
|
@model_validator(mode="after")
|
|
127
184
|
def comparison_datatypes(self: Self) -> Self:
|
|
128
185
|
x: Comparisons = self.comparison
|
|
129
|
-
y: str | int | float = self.comparator
|
|
186
|
+
y: str | int | float | list[str] = self.comparator
|
|
130
187
|
|
|
131
|
-
if x == Comparisons.
|
|
188
|
+
if x == Comparisons.JOIN or x == Comparisons.ANTIJOIN:
|
|
189
|
+
if self.regex:
|
|
190
|
+
raise TablassertValidationError(
|
|
191
|
+
"`regex` is only meaningful with the `eq`/`ne` comparisons; `join`/`antijoin` match identifiers by exact string identity against a list.",
|
|
192
|
+
code="comparison-regex-requires-eq-ne",
|
|
193
|
+
)
|
|
194
|
+
if not isinstance(y, list):
|
|
195
|
+
raise TablassertValidationError(
|
|
196
|
+
f"`join`/`antijoin` comparisons require a list of identifier strings, got {type(y).__name__}.",
|
|
197
|
+
code="comparison-membership-comparator-required",
|
|
198
|
+
)
|
|
199
|
+
if not y:
|
|
200
|
+
raise TablassertValidationError(
|
|
201
|
+
"`join`/`antijoin` comparators must hold at least one identifier.", code="comparison-membership-empty"
|
|
202
|
+
)
|
|
203
|
+
elif x == Comparisons.NE or x == Comparisons.EQ:
|
|
132
204
|
if not isinstance(y, str):
|
|
133
205
|
raise TablassertValidationError(
|
|
134
206
|
f"`eq`/`ne` comparisons require a str comparator, got {type(y).__name__}.", code="comparison-bad-comparator-type"
|
|
135
207
|
)
|
|
208
|
+
if self.regex:
|
|
209
|
+
# An empty-or-everything pattern compiles, but it matches EVERY cell, so a `ne` filter
|
|
210
|
+
# would silently drop the whole table. Reject whitespace-only spellings and any pattern
|
|
211
|
+
# that can match the empty string (`^`, `()`, `.*`, `x*`) with the same code the
|
|
212
|
+
# `exclude_regex` probe uses.
|
|
213
|
+
if not y.strip():
|
|
214
|
+
raise TablassertValidationError(
|
|
215
|
+
f"`comparator` must be a non-empty pattern when `regex` is true (an empty pattern matches every cell), got {y!r}.",
|
|
216
|
+
code="regex-bad-pattern",
|
|
217
|
+
)
|
|
218
|
+
try:
|
|
219
|
+
pl.Series([""]).str.contains(y)
|
|
220
|
+
except Exception as e:
|
|
221
|
+
raise TablassertValidationError(
|
|
222
|
+
f"`comparator` must be a polars-compatible regex when `regex` is true, got {y!r}: {e}", code="regex-bad-pattern"
|
|
223
|
+
) from e
|
|
224
|
+
if pl.Series([""]).str.contains(y).item():
|
|
225
|
+
raise TablassertValidationError(
|
|
226
|
+
f"`comparator` must not match an empty cell when `regex` is true (it would match every cell, so an `ne` filter would drop the whole table), got {y!r}.",
|
|
227
|
+
code="regex-bad-pattern",
|
|
228
|
+
)
|
|
136
229
|
else:
|
|
230
|
+
if self.regex:
|
|
231
|
+
raise TablassertValidationError(
|
|
232
|
+
"`regex` is only meaningful with the `eq`/`ne` comparisons; ordering comparisons cast the column to Float64 and compare numerically.",
|
|
233
|
+
code="comparison-regex-requires-eq-ne",
|
|
234
|
+
)
|
|
137
235
|
if not isinstance(y, (int, float)):
|
|
138
236
|
raise TablassertValidationError(
|
|
139
237
|
f"Comparisons other than `eq`/`ne` require a float or int comparator, got {type(y).__name__}.",
|
|
@@ -258,6 +258,9 @@ OP_COST: dict[Callable, int] = {
|
|
|
258
258
|
lib.crop: 1,
|
|
259
259
|
lib.pick: 1,
|
|
260
260
|
lib.reindex: 1,
|
|
261
|
+
# Membership filter: one hash join against a small key frame, same single-predicate price.
|
|
262
|
+
lib.reindex_membership: 1,
|
|
263
|
+
lib.reindex_regex: 1,
|
|
261
264
|
# head samples min(HEAD_ROWS, height) rows, so it needs a height pass before slicing.
|
|
262
265
|
lib.head: 2,
|
|
263
266
|
# Encode: each is one lazy expression over a single column.
|
|
@@ -20,6 +20,7 @@ _MESSAGES: dict[str, str] = {
|
|
|
20
20
|
"unnamed-nodes": "nodes with no name or an empty name",
|
|
21
21
|
"unidentified-nodes": "nodes with no id or an empty id",
|
|
22
22
|
"incomplete-edges": "edges missing subject, predicate, or object",
|
|
23
|
+
"self-loop-edges": "edges whose subject and object are the same CURIE",
|
|
23
24
|
"demoted-edges": "edges demoted to bare biolink:Association despite a declared category_override",
|
|
24
25
|
"undeclared-nodes": "nodes referenced by edges but not declared in the nodes file",
|
|
25
26
|
"isolated-nodes": "declared nodes participating in no edge",
|
|
@@ -61,6 +62,9 @@ class _FileScan:
|
|
|
61
62
|
#: predicate. A demotion is only a violation when the build declared a
|
|
62
63
|
#: ``category_override`` (see :func:`study_kgx`); the count is collected either way.
|
|
63
64
|
demoted: Counter[str]
|
|
65
|
+
#: Edges that assert a concept relates to itself, keyed by the shared CURIE. See
|
|
66
|
+
#: :func:`study_kgx` for why a self-referential edge is a violation.
|
|
67
|
+
self_loops: Counter[str]
|
|
64
68
|
malformed: int
|
|
65
69
|
missing: bool
|
|
66
70
|
path: Path
|
|
@@ -94,17 +98,18 @@ def _scan_ndjson(path: Path, *, edge: bool) -> _FileScan:
|
|
|
94
98
|
|
|
95
99
|
Args:
|
|
96
100
|
path: Path to a ``.nodes.ndjson`` or ``.edges.ndjson`` file.
|
|
97
|
-
edge: ``True`` to collect referenced ids from ``subject``/``object
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
+
edge: ``True`` to collect referenced ids from ``subject``/``object``, assert the
|
|
102
|
+
three core edge slots are present, and tally self-referential
|
|
103
|
+
(``subject == object``) edges; ``False`` to collect declared node ``id``s and
|
|
104
|
+
track duplicates, nodes with no name or an empty name, and nodes with no id
|
|
105
|
+
or an empty id.
|
|
101
106
|
|
|
102
107
|
Returns:
|
|
103
108
|
A :class:`_FileScan`; ``missing`` is set (and nothing else) when the
|
|
104
109
|
file does not exist, so a typo'd path can never read as a clean pass.
|
|
105
110
|
"""
|
|
106
111
|
scan: _FileScan = _FileScan(
|
|
107
|
-
set(), set(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), 0, not path.is_file(), path
|
|
112
|
+
set(), set(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), Counter(), 0, not path.is_file(), path
|
|
108
113
|
)
|
|
109
114
|
if scan.missing:
|
|
110
115
|
return scan
|
|
@@ -162,6 +167,19 @@ def _scan_ndjson(path: Path, *, edge: bool) -> _FileScan:
|
|
|
162
167
|
slot_value: object = record.get(slot)
|
|
163
168
|
if slot_value is None or (isinstance(slot_value, str) and not slot_value.strip()):
|
|
164
169
|
scan.incomplete[slot] += 1
|
|
170
|
+
# A self-referential edge states that a concept relates to itself. In a
|
|
171
|
+
# table-derived graph that is almost never the claim the author made: it is
|
|
172
|
+
# the signature of a resolution collapse, where two differently worded cells
|
|
173
|
+
# (or one cell duplicated into both slots) land on the same CURIE -- the same
|
|
174
|
+
# failure mode that makes two rows share an edge id. Such an edge carries no
|
|
175
|
+
# relational information, so downstream consumers either drop it silently or
|
|
176
|
+
# count it as evidence. Only a non-empty string pair is flagged; absent or
|
|
177
|
+
# blank slots are already reported by ``incomplete-edges``, and a blank-vs-blank
|
|
178
|
+
# pair must not read as "the same CURIE".
|
|
179
|
+
subject: object = record.get("subject")
|
|
180
|
+
object_: object = record.get("object")
|
|
181
|
+
if isinstance(subject, str) and isinstance(object_, str) and subject == object_ and subject.strip():
|
|
182
|
+
scan.self_loops[subject] += 1
|
|
165
183
|
# Bare ``biolink:Association`` means the row lost every specific class:
|
|
166
184
|
# ``lib.edge_category`` derives a class from the (subject role, object
|
|
167
185
|
# role) pair -- optionally pinned per object category by the section's
|
|
@@ -215,7 +233,8 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
|
|
|
215
233
|
Streams both files once each and checks: duplicate node ids, duplicate edge
|
|
216
234
|
ids, nodes with no
|
|
217
235
|
name or an empty name, nodes with no id or an empty id, edges missing any
|
|
218
|
-
of ``subject``/``predicate``/``object``,
|
|
236
|
+
of ``subject``/``predicate``/``object``, edges whose ``subject`` and ``object`` are
|
|
237
|
+
the same CURIE (self-referential), nodes referenced by edges but
|
|
219
238
|
never declared (``undeclared``), declared nodes participating in no edge
|
|
220
239
|
(``isolated``), empty/malformed lines, string values carrying leading/trailing
|
|
221
240
|
whitespace, and null or empty values in any field (a stronger contract than
|
|
@@ -229,6 +248,15 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
|
|
|
229
248
|
``has_supporting_studies``. Every check is an assertion -- the caller decides
|
|
230
249
|
whether violations fail the build.
|
|
231
250
|
|
|
251
|
+
A self-referential edge (``subject`` and ``object`` resolved to one CURIE) is an
|
|
252
|
+
assertion that a concept relates to itself. Table-derived graphs produce them almost
|
|
253
|
+
always by accident: two differently worded cells collapsing onto the same identifier,
|
|
254
|
+
or one cell filling both statement slots, so the edge carries no relational content and
|
|
255
|
+
ships as noise a consumer has to filter. QC therefore reports it rather than quietly
|
|
256
|
+
deleting it, so the author sees which CURIEs collapsed and fixes the mapping (or drops
|
|
257
|
+
the row) in the config. Intentionally reflexive assertions are a known accepted caveat:
|
|
258
|
+
they fail a ``--qc`` build like every other violation.
|
|
259
|
+
|
|
232
260
|
Args:
|
|
233
261
|
nodes_path: Path to ``<name>_<version>.nodes.ndjson``.
|
|
234
262
|
edges_path: Path to ``<name>_<version>.edges.ndjson``.
|
|
@@ -270,6 +298,9 @@ def study_kgx(nodes_path: Path, edges_path: Path, *, example_limit: int = 10, ca
|
|
|
270
298
|
if edges.incomplete:
|
|
271
299
|
examples = [f"{slot} ({n})" for slot, n in edges.incomplete.most_common(example_limit)]
|
|
272
300
|
violations.append(StudyViolation("incomplete-edges", "edges", sum(edges.incomplete.values()), examples))
|
|
301
|
+
if edges.self_loops:
|
|
302
|
+
examples = [f"{curie} ({n})" for curie, n in edges.self_loops.most_common(example_limit)]
|
|
303
|
+
violations.append(StudyViolation("self-loop-edges", "edges", sum(edges.self_loops.values()), examples))
|
|
273
304
|
if category_override_declared and edges.demoted:
|
|
274
305
|
examples = [f"{predicate} ({n})" for predicate, n in edges.demoted.most_common(example_limit)]
|
|
275
306
|
violations.append(StudyViolation("demoted-edges", "edges", sum(edges.demoted.values()), examples))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|