tablassert 18.0.0__tar.gz → 18.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-18.0.0 → tablassert-18.1.0}/PKG-INFO +1 -1
- {tablassert-18.0.0 → tablassert-18.1.0}/pyproject.toml +2 -1
- {tablassert-18.0.0 → tablassert-18.1.0}/rust/src/fullmap.rs +102 -4
- {tablassert-18.0.0 → tablassert-18.1.0}/rust/tests/build_golden.rs +74 -6
- {tablassert-18.0.0 → tablassert-18.1.0}/rust/tests/common/mod.rs +9 -2
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/biolink.py +41 -3
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/cli.py +78 -12
- tablassert-18.1.0/src/tablassert/data/experimental_taxa.yaml +511 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/fullmap.py +30 -8
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/rs.pyi +5 -1
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/study.py +38 -3
- {tablassert-18.0.0 → tablassert-18.1.0}/LICENSE +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/README.md +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/rust/Cargo.lock +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/rust/Cargo.toml +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/rust/src/json.rs +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/rust/src/lib.rs +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/rust/src/ndjson.rs +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/rust/src/uuid.rs +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/__init__.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/agent.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/coerce.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/distill.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/enums.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/errors.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/extras.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/ingests.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/lib.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/log.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/models.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/nlp.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/progress.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/qc.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/rig.py +0 -0
- {tablassert-18.0.0 → tablassert-18.1.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "18.
|
|
3
|
+
version = "18.1.0"
|
|
4
4
|
description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -72,6 +72,7 @@ module-name = "tablassert.rs"
|
|
|
72
72
|
bindings = "pyo3"
|
|
73
73
|
manifest-path = "rust/Cargo.toml"
|
|
74
74
|
features = ["extension-module"]
|
|
75
|
+
include = ["src/tablassert/data/experimental_taxa.yaml"]
|
|
75
76
|
|
|
76
77
|
[project.scripts]
|
|
77
78
|
tablassert = "tablassert.cli:APP"
|
|
@@ -294,7 +294,6 @@ fn token_qc(value: &str) -> bool {
|
|
|
294
294
|
&& !value.contains('\t')
|
|
295
295
|
&& !value.contains('\n')
|
|
296
296
|
&& !value.contains('\r')
|
|
297
|
-
&& !value.contains("inchikey")
|
|
298
297
|
&& !value.contains("uncharacterized")
|
|
299
298
|
&& !value.contains("hypothetical")
|
|
300
299
|
}
|
|
@@ -459,6 +458,15 @@ fn first_category(row: &SynonymRow<'_>) -> String {
|
|
|
459
458
|
.to_string()
|
|
460
459
|
}
|
|
461
460
|
|
|
461
|
+
fn is_organism_taxon(row: &SynonymRow<'_>) -> bool {
|
|
462
|
+
row.types.iter().any(|category| {
|
|
463
|
+
category
|
|
464
|
+
.trim()
|
|
465
|
+
.trim_start_matches("biolink:")
|
|
466
|
+
.eq_ignore_ascii_case("OrganismTaxon")
|
|
467
|
+
})
|
|
468
|
+
}
|
|
469
|
+
|
|
462
470
|
fn first_taxon(row: &SynonymRow<'_>) -> i32 {
|
|
463
471
|
row.taxa
|
|
464
472
|
.first()
|
|
@@ -469,6 +477,56 @@ fn first_taxon(row: &SynonymRow<'_>) -> i32 {
|
|
|
469
477
|
.unwrap_or(0)
|
|
470
478
|
}
|
|
471
479
|
|
|
480
|
+
fn parsed_taxa(row: &SynonymRow<'_>) -> Vec<i32> {
|
|
481
|
+
row.taxa
|
|
482
|
+
.iter()
|
|
483
|
+
.filter_map(|taxon| {
|
|
484
|
+
let taxon = taxon.trim();
|
|
485
|
+
let local = taxon
|
|
486
|
+
.split_once(':')
|
|
487
|
+
.filter(|(prefix, _)| prefix.eq_ignore_ascii_case("NCBITaxon"))
|
|
488
|
+
.map(|(_, local)| local)
|
|
489
|
+
.unwrap_or(taxon);
|
|
490
|
+
local.parse::<i32>().ok().filter(|id| *id > 0)
|
|
491
|
+
})
|
|
492
|
+
.collect()
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
fn allowlist_identity(taxa: &HashSet<i32>) -> String {
|
|
496
|
+
let mut ids: Vec<i32> = taxa.iter().copied().collect();
|
|
497
|
+
ids.sort_unstable();
|
|
498
|
+
let encoded = ids
|
|
499
|
+
.iter()
|
|
500
|
+
.map(i32::to_string)
|
|
501
|
+
.collect::<Vec<String>>()
|
|
502
|
+
.join(",");
|
|
503
|
+
format!(
|
|
504
|
+
"count={};xxh64={:016x}",
|
|
505
|
+
ids.len(),
|
|
506
|
+
xxh64(encoded.as_bytes(), 0)
|
|
507
|
+
)
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
fn retain_for_taxon_allowlist(row: &SynonymRow<'_>, allowlist: Option<&HashSet<i32>>) -> bool {
|
|
511
|
+
let Some(allowlist) = allowlist else {
|
|
512
|
+
return true;
|
|
513
|
+
};
|
|
514
|
+
if is_organism_taxon(row) {
|
|
515
|
+
return true;
|
|
516
|
+
}
|
|
517
|
+
let mut has_valid_taxon = false;
|
|
518
|
+
for taxon in parsed_taxa(row) {
|
|
519
|
+
has_valid_taxon = true;
|
|
520
|
+
if allowlist.contains(&taxon) {
|
|
521
|
+
return true;
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
// Taxonless rows remain available for categories whose BABEL synonym rows
|
|
525
|
+
// do not carry a positive taxon ID. This includes the conventional
|
|
526
|
+
// `NCBITaxon:0` sentinel and malformed/empty taxon values.
|
|
527
|
+
!has_valid_taxon
|
|
528
|
+
}
|
|
529
|
+
|
|
472
530
|
fn split_curie(curie: &str) -> Option<(&str, &str)> {
|
|
473
531
|
curie
|
|
474
532
|
.split_once(':')
|
|
@@ -1358,6 +1416,7 @@ struct SynonymShared<'a> {
|
|
|
1358
1416
|
curie_counter: &'a AtomicU32,
|
|
1359
1417
|
equivalents: &'a EquivIndex,
|
|
1360
1418
|
exclude_prefixes: &'a HashSet<String>,
|
|
1419
|
+
taxon_allowlist: Option<&'a HashSet<i32>>,
|
|
1361
1420
|
spill_dir: &'a Path,
|
|
1362
1421
|
local_spill: usize,
|
|
1363
1422
|
curie_spill: usize,
|
|
@@ -1393,6 +1452,9 @@ fn process_row(
|
|
|
1393
1452
|
let Some((prefix, local_id)) = split_curie(curie) else {
|
|
1394
1453
|
return Ok(());
|
|
1395
1454
|
};
|
|
1455
|
+
if !retain_for_taxon_allowlist(row, sh.taxon_allowlist) {
|
|
1456
|
+
return Ok(());
|
|
1457
|
+
}
|
|
1396
1458
|
// Skip CURIEs whose prefix the caller excludes (opt-in via
|
|
1397
1459
|
// TABLASSERT_FULLMAP_EXCLUDE_PREFIXES) — filtered out downstream anyway.
|
|
1398
1460
|
if sh.exclude_prefixes.contains(prefix) {
|
|
@@ -1623,6 +1685,7 @@ fn process_synonyms(
|
|
|
1623
1685
|
local_spill: usize,
|
|
1624
1686
|
curie_spill: usize,
|
|
1625
1687
|
exclude_prefixes: &HashSet<String>,
|
|
1688
|
+
taxon_allowlist: Option<&HashSet<i32>>,
|
|
1626
1689
|
worker_count: usize,
|
|
1627
1690
|
shard_count: usize,
|
|
1628
1691
|
chunk_bytes: usize,
|
|
@@ -1671,6 +1734,7 @@ fn process_synonyms(
|
|
|
1671
1734
|
curie_counter: &curie_counter,
|
|
1672
1735
|
equivalents,
|
|
1673
1736
|
exclude_prefixes,
|
|
1737
|
+
taxon_allowlist,
|
|
1674
1738
|
spill_dir,
|
|
1675
1739
|
local_spill,
|
|
1676
1740
|
curie_spill,
|
|
@@ -1799,6 +1863,7 @@ fn write_final_database(
|
|
|
1799
1863
|
insert_batch: usize,
|
|
1800
1864
|
shard_count: usize,
|
|
1801
1865
|
progress: Option<&Arc<Progress>>,
|
|
1866
|
+
taxon_allowlist_identity: Option<&str>,
|
|
1802
1867
|
) -> PyResult<()> {
|
|
1803
1868
|
let build_id = new_build_id();
|
|
1804
1869
|
let build_id_str = build_id.to_string();
|
|
@@ -1855,6 +1920,9 @@ fn write_final_database(
|
|
|
1855
1920
|
let shard_count_str = shard_count.to_string();
|
|
1856
1921
|
meta.insert("shards", shard_count_str.as_str())
|
|
1857
1922
|
.map_err(py_err)?;
|
|
1923
|
+
if let Some(identity) = taxon_allowlist_identity {
|
|
1924
|
+
meta.insert("taxon_allowlist", identity).map_err(py_err)?;
|
|
1925
|
+
}
|
|
1858
1926
|
}
|
|
1859
1927
|
write.commit().map_err(py_err)?;
|
|
1860
1928
|
|
|
@@ -2229,6 +2297,8 @@ fn build_fullmap_inner(
|
|
|
2229
2297
|
local_spill: usize,
|
|
2230
2298
|
curie_spill: usize,
|
|
2231
2299
|
exclude_prefixes: HashSet<String>,
|
|
2300
|
+
taxon_allowlist: Option<HashSet<i32>>,
|
|
2301
|
+
taxon_allowlist_identity: Option<String>,
|
|
2232
2302
|
chunk_bytes: usize,
|
|
2233
2303
|
producers: usize,
|
|
2234
2304
|
cache_bytes: usize,
|
|
@@ -2258,6 +2328,7 @@ fn build_fullmap_inner(
|
|
|
2258
2328
|
local_spill,
|
|
2259
2329
|
curie_spill,
|
|
2260
2330
|
&exclude_prefixes,
|
|
2331
|
+
taxon_allowlist.as_ref(),
|
|
2261
2332
|
worker_count,
|
|
2262
2333
|
shard_count,
|
|
2263
2334
|
chunk_bytes,
|
|
@@ -2288,6 +2359,7 @@ fn build_fullmap_inner(
|
|
|
2288
2359
|
insert_batch,
|
|
2289
2360
|
shard_count,
|
|
2290
2361
|
progress.as_ref(),
|
|
2362
|
+
taxon_allowlist_identity.as_deref(),
|
|
2291
2363
|
)?;
|
|
2292
2364
|
|
|
2293
2365
|
// Clean up spill runs on success (left in place on error for inspection).
|
|
@@ -2297,13 +2369,14 @@ fn build_fullmap_inner(
|
|
|
2297
2369
|
}
|
|
2298
2370
|
|
|
2299
2371
|
#[pyfunction]
|
|
2300
|
-
#[pyo3(signature = (output, classes, synonyms, progress=None))]
|
|
2372
|
+
#[pyo3(signature = (output, classes, synonyms, progress=None, taxon_allowlist=None))]
|
|
2301
2373
|
pub fn build_fullmap_db(
|
|
2302
2374
|
py: Python<'_>,
|
|
2303
2375
|
output: PathBuf,
|
|
2304
2376
|
classes: Vec<PathBuf>,
|
|
2305
2377
|
synonyms: Vec<PathBuf>,
|
|
2306
2378
|
progress: Option<Py<PyAny>>,
|
|
2379
|
+
taxon_allowlist: Option<Vec<i32>>,
|
|
2307
2380
|
) -> PyResult<()> {
|
|
2308
2381
|
if synonyms.is_empty() {
|
|
2309
2382
|
return Err(PyValueError::new_err(
|
|
@@ -2407,6 +2480,9 @@ pub fn build_fullmap_db(
|
|
|
2407
2480
|
PathBuf::from(p)
|
|
2408
2481
|
});
|
|
2409
2482
|
|
|
2483
|
+
let taxon_allowlist: Option<HashSet<i32>> =
|
|
2484
|
+
taxon_allowlist.map(|ids| ids.into_iter().filter(|id| *id > 0).collect());
|
|
2485
|
+
let taxon_allowlist_identity: Option<String> = taxon_allowlist.as_ref().map(allowlist_identity);
|
|
2410
2486
|
let progress = progress.map(|cb| Arc::new(Progress { cb }));
|
|
2411
2487
|
|
|
2412
2488
|
// Release the GIL for the whole build so rich's Live display thread can
|
|
@@ -2422,6 +2498,8 @@ pub fn build_fullmap_db(
|
|
|
2422
2498
|
local_spill,
|
|
2423
2499
|
curie_spill,
|
|
2424
2500
|
exclude_prefixes,
|
|
2501
|
+
taxon_allowlist,
|
|
2502
|
+
taxon_allowlist_identity,
|
|
2425
2503
|
chunk_bytes,
|
|
2426
2504
|
producers,
|
|
2427
2505
|
cache_bytes,
|
|
@@ -3650,6 +3728,8 @@ mod tests {
|
|
|
3650
3728
|
local_spill,
|
|
3651
3729
|
1_000_000,
|
|
3652
3730
|
HashSet::new(),
|
|
3731
|
+
None,
|
|
3732
|
+
None,
|
|
3653
3733
|
DEFAULT_CHUNK_BYTES,
|
|
3654
3734
|
2,
|
|
3655
3735
|
64 * 1024 * 1024,
|
|
@@ -3678,10 +3758,12 @@ mod tests {
|
|
|
3678
3758
|
}
|
|
3679
3759
|
|
|
3680
3760
|
#[test]
|
|
3681
|
-
fn
|
|
3761
|
+
fn token_qc_rejects_banned_tokens_but_accepts_inchikey() {
|
|
3682
3762
|
assert!(token_qc("brca1"));
|
|
3763
|
+
assert!(token_qc("inchikey=abc"));
|
|
3683
3764
|
assert!(!token_qc("hypothetical protein"));
|
|
3684
3765
|
assert!(!token_qc("line\nbreak"));
|
|
3766
|
+
assert!(!token_qc("tab\tvalue"));
|
|
3685
3767
|
}
|
|
3686
3768
|
|
|
3687
3769
|
#[test]
|
|
@@ -4028,6 +4110,7 @@ mod tests {
|
|
|
4028
4110
|
30,
|
|
4029
4111
|
1_000_000,
|
|
4030
4112
|
&HashSet::new(),
|
|
4113
|
+
None,
|
|
4031
4114
|
4,
|
|
4032
4115
|
shard_count,
|
|
4033
4116
|
DEFAULT_CHUNK_BYTES,
|
|
@@ -4085,6 +4168,8 @@ mod tests {
|
|
|
4085
4168
|
100,
|
|
4086
4169
|
1_000_000,
|
|
4087
4170
|
HashSet::new(),
|
|
4171
|
+
None,
|
|
4172
|
+
None,
|
|
4088
4173
|
DEFAULT_CHUNK_BYTES,
|
|
4089
4174
|
2,
|
|
4090
4175
|
64 * 1024 * 1024,
|
|
@@ -4102,6 +4187,8 @@ mod tests {
|
|
|
4102
4187
|
100,
|
|
4103
4188
|
1_000_000,
|
|
4104
4189
|
HashSet::new(),
|
|
4190
|
+
None,
|
|
4191
|
+
None,
|
|
4105
4192
|
DEFAULT_CHUNK_BYTES,
|
|
4106
4193
|
2,
|
|
4107
4194
|
64 * 1024 * 1024,
|
|
@@ -4519,6 +4606,8 @@ mod tests {
|
|
|
4519
4606
|
4_000_000,
|
|
4520
4607
|
1_000_000,
|
|
4521
4608
|
HashSet::new(),
|
|
4609
|
+
None,
|
|
4610
|
+
None,
|
|
4522
4611
|
DEFAULT_CHUNK_BYTES,
|
|
4523
4612
|
2,
|
|
4524
4613
|
64 * 1024 * 1024,
|
|
@@ -4588,7 +4677,7 @@ mod tests {
|
|
|
4588
4677
|
|
|
4589
4678
|
std::env::set_var("TABLASSERT_FULLMAP_SHARDS", "2");
|
|
4590
4679
|
let built = Python::attach(|py| {
|
|
4591
|
-
build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None)
|
|
4680
|
+
build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None, None)
|
|
4592
4681
|
});
|
|
4593
4682
|
std::env::remove_var("TABLASSERT_FULLMAP_SHARDS");
|
|
4594
4683
|
built.unwrap();
|
|
@@ -4649,6 +4738,8 @@ mod tests {
|
|
|
4649
4738
|
4_000_000,
|
|
4650
4739
|
1_000_000,
|
|
4651
4740
|
HashSet::new(),
|
|
4741
|
+
None,
|
|
4742
|
+
None,
|
|
4652
4743
|
DEFAULT_CHUNK_BYTES,
|
|
4653
4744
|
2,
|
|
4654
4745
|
64 * 1024 * 1024,
|
|
@@ -5356,6 +5447,8 @@ mod tests {
|
|
|
5356
5447
|
4_000_000,
|
|
5357
5448
|
2,
|
|
5358
5449
|
HashSet::new(),
|
|
5450
|
+
None,
|
|
5451
|
+
None,
|
|
5359
5452
|
DEFAULT_CHUNK_BYTES,
|
|
5360
5453
|
2,
|
|
5361
5454
|
64 * 1024 * 1024,
|
|
@@ -5446,6 +5539,8 @@ mod tests {
|
|
|
5446
5539
|
4_000_000,
|
|
5447
5540
|
1_000_000,
|
|
5448
5541
|
exclude,
|
|
5542
|
+
None,
|
|
5543
|
+
None,
|
|
5449
5544
|
DEFAULT_CHUNK_BYTES,
|
|
5450
5545
|
2,
|
|
5451
5546
|
64 * 1024 * 1024,
|
|
@@ -5505,6 +5600,8 @@ mod tests {
|
|
|
5505
5600
|
4_000_000,
|
|
5506
5601
|
1_000_000,
|
|
5507
5602
|
HashSet::new(),
|
|
5603
|
+
None,
|
|
5604
|
+
None,
|
|
5508
5605
|
8192,
|
|
5509
5606
|
2,
|
|
5510
5607
|
64 * 1024 * 1024,
|
|
@@ -5546,6 +5643,7 @@ mod tests {
|
|
|
5546
5643
|
Vec::new(),
|
|
5547
5644
|
Vec::new(),
|
|
5548
5645
|
None,
|
|
5646
|
+
None,
|
|
5549
5647
|
)
|
|
5550
5648
|
.expect_err("empty synonyms should fail");
|
|
5551
5649
|
assert!(err
|
|
@@ -285,6 +285,57 @@ fn schema_and_shard_count_are_pinned() {
|
|
|
285
285
|
);
|
|
286
286
|
}
|
|
287
287
|
|
|
288
|
+
#[test]
|
|
289
|
+
fn taxon_allowlist_metadata_is_stable_and_omitted_for_default_builds() {
|
|
290
|
+
let dir = tempfile::tempdir().unwrap();
|
|
291
|
+
let synonyms = dir.path().join("SRC.ndjson");
|
|
292
|
+
write_jsonl(
|
|
293
|
+
&synonyms,
|
|
294
|
+
&[
|
|
295
|
+
r#"{"curie":"HGNC:1","preferred_name":"Gene","names":["gene"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#,
|
|
296
|
+
],
|
|
297
|
+
);
|
|
298
|
+
let filtered_a = dir.path().join("filtered_a.redb");
|
|
299
|
+
let filtered_b = dir.path().join("filtered_b.redb");
|
|
300
|
+
let unfiltered = dir.path().join("unfiltered.redb");
|
|
301
|
+
pyo3::Python::initialize();
|
|
302
|
+
pyo3::Python::attach(|py| {
|
|
303
|
+
tablassert_rs::build_fullmap_db(
|
|
304
|
+
py,
|
|
305
|
+
filtered_a.clone(),
|
|
306
|
+
vec![],
|
|
307
|
+
vec![synonyms.clone()],
|
|
308
|
+
None,
|
|
309
|
+
Some(vec![10090, 9606]),
|
|
310
|
+
)
|
|
311
|
+
.unwrap();
|
|
312
|
+
tablassert_rs::build_fullmap_db(
|
|
313
|
+
py,
|
|
314
|
+
filtered_b.clone(),
|
|
315
|
+
vec![],
|
|
316
|
+
vec![synonyms.clone()],
|
|
317
|
+
None,
|
|
318
|
+
Some(vec![9606, 10090]),
|
|
319
|
+
)
|
|
320
|
+
.unwrap();
|
|
321
|
+
tablassert_rs::build_fullmap_db(py, unfiltered.clone(), vec![], vec![synonyms], None, None)
|
|
322
|
+
.unwrap();
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
let read_allowlist = |path: &std::path::Path| -> Option<String> {
|
|
326
|
+
let db = open_primary_copy(path);
|
|
327
|
+
let read = db.begin_read().unwrap();
|
|
328
|
+
let meta = read.open_table(META).unwrap();
|
|
329
|
+
meta.get("taxon_allowlist")
|
|
330
|
+
.unwrap()
|
|
331
|
+
.map(|value| value.value().to_string())
|
|
332
|
+
};
|
|
333
|
+
let identity = read_allowlist(&filtered_a).unwrap();
|
|
334
|
+
assert_eq!(Some(identity.clone()), read_allowlist(&filtered_b));
|
|
335
|
+
assert_eq!(None, read_allowlist(&unfiltered));
|
|
336
|
+
assert!(identity.starts_with("count=2;xxh64="));
|
|
337
|
+
}
|
|
338
|
+
|
|
288
339
|
// ---------------------------------------------------------------------------
|
|
289
340
|
// (f) GZ INPUT — a gzipped synonym file yields identical results to plain.
|
|
290
341
|
// ---------------------------------------------------------------------------
|
|
@@ -324,6 +375,7 @@ fn gz_input_matches_plain_input() {
|
|
|
324
375
|
vec![classes],
|
|
325
376
|
vec![synonyms_gz],
|
|
326
377
|
None,
|
|
378
|
+
None,
|
|
327
379
|
)
|
|
328
380
|
.unwrap();
|
|
329
381
|
});
|
|
@@ -351,8 +403,15 @@ fn empty_synonym_file_builds_empty_db() {
|
|
|
351
403
|
write_jsonl(&synonyms, &[]); // 0 rows
|
|
352
404
|
let output = dir.path().join("fullmap.redb");
|
|
353
405
|
pyo3::Python::attach(|py| {
|
|
354
|
-
tablassert_rs::build_fullmap_db(
|
|
355
|
-
|
|
406
|
+
tablassert_rs::build_fullmap_db(
|
|
407
|
+
py,
|
|
408
|
+
output.clone(),
|
|
409
|
+
vec![classes],
|
|
410
|
+
vec![synonyms],
|
|
411
|
+
None,
|
|
412
|
+
None,
|
|
413
|
+
)
|
|
414
|
+
.unwrap();
|
|
356
415
|
});
|
|
357
416
|
|
|
358
417
|
assert!(
|
|
@@ -383,7 +442,8 @@ fn synonym_row_with_no_names_indexes_only_curie() {
|
|
|
383
442
|
);
|
|
384
443
|
let output = dir.path().join("fullmap.redb");
|
|
385
444
|
pyo3::Python::attach(|py| {
|
|
386
|
-
tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], None)
|
|
445
|
+
tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], None, None)
|
|
446
|
+
.unwrap();
|
|
387
447
|
});
|
|
388
448
|
|
|
389
449
|
let map = term_curie_map(&output);
|
|
@@ -409,7 +469,8 @@ fn null_preferred_name_falls_back_to_curie() {
|
|
|
409
469
|
);
|
|
410
470
|
let output = dir.path().join("fullmap.redb");
|
|
411
471
|
pyo3::Python::attach(|py| {
|
|
412
|
-
tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], None)
|
|
472
|
+
tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], None, None)
|
|
473
|
+
.unwrap();
|
|
413
474
|
});
|
|
414
475
|
|
|
415
476
|
let db = open_primary_copy(&output);
|
|
@@ -444,8 +505,15 @@ fn class_row_without_equivalents_builds() {
|
|
|
444
505
|
);
|
|
445
506
|
let output = dir.path().join("fullmap.redb");
|
|
446
507
|
pyo3::Python::attach(|py| {
|
|
447
|
-
tablassert_rs::build_fullmap_db(
|
|
448
|
-
|
|
508
|
+
tablassert_rs::build_fullmap_db(
|
|
509
|
+
py,
|
|
510
|
+
output.clone(),
|
|
511
|
+
vec![classes],
|
|
512
|
+
vec![synonyms],
|
|
513
|
+
None,
|
|
514
|
+
None,
|
|
515
|
+
)
|
|
516
|
+
.unwrap();
|
|
449
517
|
});
|
|
450
518
|
|
|
451
519
|
let map = term_curie_map(&output);
|
|
@@ -117,8 +117,15 @@ pub fn build_fixture(dir: &Path) -> PathBuf {
|
|
|
117
117
|
write_jsonl(&synonyms, SYNONYM_LINES);
|
|
118
118
|
let output = dir.join("fullmap.redb");
|
|
119
119
|
pyo3::Python::attach(|py| {
|
|
120
|
-
tablassert_rs::build_fullmap_db(
|
|
121
|
-
|
|
120
|
+
tablassert_rs::build_fullmap_db(
|
|
121
|
+
py,
|
|
122
|
+
output.clone(),
|
|
123
|
+
vec![classes],
|
|
124
|
+
vec![synonyms],
|
|
125
|
+
None,
|
|
126
|
+
None,
|
|
127
|
+
)
|
|
128
|
+
.unwrap();
|
|
122
129
|
});
|
|
123
130
|
output
|
|
124
131
|
}
|
|
@@ -59,6 +59,7 @@ if TYPE_CHECKING:
|
|
|
59
59
|
__all__ = [
|
|
60
60
|
"ALLOWED_EDGE_FIELDS",
|
|
61
61
|
"BIOLINK_VERSION",
|
|
62
|
+
"CATEGORY_OVERRIDES",
|
|
62
63
|
"CLASS_FIELD_OVERRIDES",
|
|
63
64
|
"DISABLED_EDGE_FIELDS",
|
|
64
65
|
"EFFECT_TYPE_VALUES",
|
|
@@ -153,18 +154,54 @@ def _category_name(cls: Any) -> str:
|
|
|
153
154
|
return str(cls.__name__)
|
|
154
155
|
|
|
155
156
|
|
|
157
|
+
CATEGORY_OVERRIDES: frozenset[str] = frozenset({"GenomicEntity"})
|
|
158
|
+
"""Biolink category names the ``Categories`` enum must carry ahead of the pinned model.
|
|
159
|
+
|
|
160
|
+
The enum is built from the ``pydanticmodel_v2`` classes that subclass ``Entity``, but
|
|
161
|
+
Biolink declares some node categories as MIXINS -- and a mixin is generated as a plain
|
|
162
|
+
``ConfiguredBaseModel`` that is NOT an ``Entity`` subclass (``GenomicEntity``'s MRO is
|
|
163
|
+
``GenomicEntity -> ConfiguredBaseModel -> BaseModel -> object``). The scan therefore
|
|
164
|
+
cannot see them, while the fullmap -- whose ``CATEGORY_NAME`` vocabulary comes from
|
|
165
|
+
BABEL, not from the Pydantic class tree -- assigns them to real terms (UMLS
|
|
166
|
+
"gene mutation" -> ``UMLS:C0678941``, the ``* wt Allele`` CUIs).
|
|
167
|
+
|
|
168
|
+
An unnameable category is a hole in every category-keyed config surface:
|
|
169
|
+
``NodeEncoding.avoid`` / ``prioritize`` and ``Statement.category_override`` keys are
|
|
170
|
+
validated against the enum, so an ``avoid``-list allow-list can never exclude a
|
|
171
|
+
category the resolver actually emits, and the row falls through to whatever the
|
|
172
|
+
(subject role, object role) pair derives -- for a mixin object with no
|
|
173
|
+
``CATEGORY_PARENT`` rollup that is bare ``biolink:Association``, whose
|
|
174
|
+
``prune_to_class`` then nulls every class-specific slot into the inlined
|
|
175
|
+
``has_supporting_studies`` junk drawer. (DAKP shipped 35 such edges before this
|
|
176
|
+
override existed.) ``fullmap.filter_and_rank`` additionally drops unnameable
|
|
177
|
+
categories when ``avoid`` is set, so a future vocabulary drift degrades to lost rows
|
|
178
|
+
rather than mis-classed edges.
|
|
179
|
+
|
|
180
|
+
Names here are unioned into :func:`_entity_category_names`, so each is a first-class
|
|
181
|
+
``Categories`` member: valid in ``avoid``/``prioritize``, valid as a
|
|
182
|
+
``category_override`` key, and included in generated complements. Node emission is
|
|
183
|
+
unaffected -- ``resolve_node_class`` walks the MRO independently of this enum.
|
|
184
|
+
|
|
185
|
+
A tripwire test asserts every override is still invisible to the ``Entity`` scan: the
|
|
186
|
+
moment a biolink-model release promotes a mixin to a real ``Entity`` subclass (or the
|
|
187
|
+
scan otherwise picks it up), the suite fails and the stale name is removed from
|
|
188
|
+
``CATEGORY_OVERRIDES`` -- the same philosophy as ``CLASS_FIELD_OVERRIDES``.
|
|
189
|
+
"""
|
|
190
|
+
|
|
191
|
+
|
|
156
192
|
def _entity_category_names() -> list[str]:
|
|
157
193
|
"""Collect all Biolink entity category names from the Pydantic model.
|
|
158
194
|
|
|
159
195
|
A category is any ``pydanticmodel_v2`` class that subclasses ``Entity`` but is
|
|
160
196
|
not an ``Association`` subclass (associations are edge categories, handled
|
|
161
|
-
separately)
|
|
162
|
-
|
|
197
|
+
separately), plus the curated :data:`CATEGORY_OVERRIDES` mixin names the
|
|
198
|
+
``Entity`` scan cannot see. This mirrors the Biolink notion of a node category
|
|
199
|
+
while also including the upper-level ``Entity`` / ``NamedThing`` classes.
|
|
163
200
|
|
|
164
201
|
Returns:
|
|
165
202
|
Sorted unique category names.
|
|
166
203
|
"""
|
|
167
|
-
names: set[str] = set()
|
|
204
|
+
names: set[str] = set(CATEGORY_OVERRIDES)
|
|
168
205
|
for cls in vars(_bm).values():
|
|
169
206
|
if inspect.isclass(cls) and cls.__module__ == _bm.__name__ and issubclass(cls, _bm.Entity) and not issubclass(cls, _bm.Association):
|
|
170
207
|
names.add(_category_name(cls))
|
|
@@ -431,6 +468,7 @@ if TYPE_CHECKING:
|
|
|
431
468
|
class Categories(str, Enum):
|
|
432
469
|
DISEASE: Categories
|
|
433
470
|
GENE: Categories
|
|
471
|
+
GENOMIC_ENTITY: Categories
|
|
434
472
|
NAMED_THING: Categories
|
|
435
473
|
PHENOTYPIC_FEATURE: Categories
|
|
436
474
|
PROTEIN: Categories
|