tablassert 16.7.0__tar.gz → 17.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-16.7.0 → tablassert-17.0.0}/PKG-INFO +1 -1
- {tablassert-16.7.0 → tablassert-17.0.0}/pyproject.toml +1 -1
- {tablassert-16.7.0 → tablassert-17.0.0}/rust/src/fullmap.rs +118 -137
- tablassert-17.0.0/rust/src/json.rs +454 -0
- tablassert-17.0.0/rust/src/ndjson.rs +2342 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/rust/tests/build_golden.rs +27 -40
- {tablassert-16.7.0 → tablassert-17.0.0}/rust/tests/common/mod.rs +6 -13
- {tablassert-16.7.0 → tablassert-17.0.0}/rust/tests/extract_prebuilt.rs +9 -9
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/agent.py +1 -5
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/cli.py +6 -39
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/fullmap.py +5 -11
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/lib.py +1 -5
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/rs.pyi +2 -2
- tablassert-16.7.0/rust/src/json.rs +0 -156
- tablassert-16.7.0/rust/src/ndjson.rs +0 -1227
- {tablassert-16.7.0 → tablassert-17.0.0}/LICENSE +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/README.md +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/rust/Cargo.lock +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/rust/Cargo.toml +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/rust/src/lib.rs +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/rust/src/uuid.rs +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/__init__.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/biolink.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/coerce.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/distill.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/enums.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/errors.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/extras.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/ingests.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/log.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/models.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/nlp.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/progress.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/qc.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/rig.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/study.py +0 -0
- {tablassert-16.7.0 → tablassert-17.0.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "
|
|
3
|
+
version = "17.0.0"
|
|
4
4
|
description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -2297,13 +2297,12 @@ fn build_fullmap_inner(
|
|
|
2297
2297
|
}
|
|
2298
2298
|
|
|
2299
2299
|
#[pyfunction]
|
|
2300
|
-
#[pyo3(signature = (output, classes, synonyms,
|
|
2300
|
+
#[pyo3(signature = (output, classes, synonyms, progress=None))]
|
|
2301
2301
|
pub fn build_fullmap_db(
|
|
2302
2302
|
py: Python<'_>,
|
|
2303
2303
|
output: PathBuf,
|
|
2304
2304
|
classes: Vec<PathBuf>,
|
|
2305
2305
|
synonyms: Vec<PathBuf>,
|
|
2306
|
-
threads: Option<usize>,
|
|
2307
2306
|
progress: Option<Py<PyAny>>,
|
|
2308
2307
|
) -> PyResult<()> {
|
|
2309
2308
|
if synonyms.is_empty() {
|
|
@@ -2319,35 +2318,34 @@ pub fn build_fullmap_db(
|
|
|
2319
2318
|
// the build if this doesn't succeed.
|
|
2320
2319
|
let _ = rlimit::increase_nofile_limit(u64::MAX);
|
|
2321
2320
|
|
|
2322
|
-
let worker_count =
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
.
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
.
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
.max(1);
|
|
2321
|
+
let worker_count = {
|
|
2322
|
+
let cpus = std::thread::available_parallelism()
|
|
2323
|
+
.map(std::num::NonZero::get)
|
|
2324
|
+
.unwrap_or(1);
|
|
2325
|
+
// Cap at available_memory_gb / 2 to prevent swap on memory-constrained
|
|
2326
|
+
// machines. Each thread uses ~400 MB of local buffers; the cap is
|
|
2327
|
+
// generous (2 GB/thread) to avoid limiting CPU-bound throughput.
|
|
2328
|
+
let avail_kb = std::fs::read_to_string("/proc/meminfo")
|
|
2329
|
+
.ok()
|
|
2330
|
+
.and_then(|s| {
|
|
2331
|
+
s.lines()
|
|
2332
|
+
.find(|l| l.starts_with("MemAvailable:"))
|
|
2333
|
+
.and_then(|l| {
|
|
2334
|
+
l.split_whitespace()
|
|
2335
|
+
.nth(1)
|
|
2336
|
+
.and_then(|v| v.parse::<usize>().ok())
|
|
2337
|
+
})
|
|
2338
|
+
})
|
|
2339
|
+
.unwrap_or(0);
|
|
2340
|
+
if avail_kb > 0 {
|
|
2341
|
+
let avail_gb = avail_kb / (1024 * 1024);
|
|
2342
|
+
let mem_cap = (avail_gb / 2).max(1);
|
|
2343
|
+
cpus.min(mem_cap)
|
|
2344
|
+
} else {
|
|
2345
|
+
cpus * 9 / 10
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
.max(1);
|
|
2351
2349
|
|
|
2352
2350
|
if let Some(parent) = output.parent() {
|
|
2353
2351
|
std::fs::create_dir_all(parent).map_err(py_err)?;
|
|
@@ -3382,8 +3380,8 @@ fn lookup_pair_terms_db(
|
|
|
3382
3380
|
Ok(tagged.into_iter().map(|(_, pairs)| pairs).collect())
|
|
3383
3381
|
}
|
|
3384
3382
|
|
|
3385
|
-
/// Smallest batch that
|
|
3386
|
-
///
|
|
3383
|
+
/// Smallest batch that fans out across the shards in parallel. Below this,
|
|
3384
|
+
/// lookups stay single-threaded: a point/small lookup
|
|
3387
3385
|
/// (and any cache-warm path) finishes faster serially than the cost of spawning
|
|
3388
3386
|
/// shard-reader threads. At/above it, the per-shard fan-out in
|
|
3389
3387
|
/// `lookup_pair_terms_db` wins. 1024 terms ~= a few ms of serial redb point
|
|
@@ -3392,9 +3390,9 @@ fn lookup_pair_terms_db(
|
|
|
3392
3390
|
/// while never penalizing small lookups.
|
|
3393
3391
|
const LOOKUP_PARALLEL_MIN: usize = 1024;
|
|
3394
3392
|
|
|
3395
|
-
///
|
|
3393
|
+
/// Worker count for lookups, selected automatically from the batch size.
|
|
3396
3394
|
/// The production build-kg resolve sends ONE batch of all distinct node-column
|
|
3397
|
-
/// terms (often huge)
|
|
3395
|
+
/// terms (often huge); parallelizing that across the RECORDS
|
|
3398
3396
|
/// shards is the win, so large batches default to `available_parallelism`. Small
|
|
3399
3397
|
/// batches (< `LOOKUP_PARALLEL_MIN`) stay single-threaded to avoid spawn overhead.
|
|
3400
3398
|
/// The fan-out is already capped by the non-empty shard count inside
|
|
@@ -3409,13 +3407,8 @@ fn default_lookup_workers(terms_len: usize) -> usize {
|
|
|
3409
3407
|
.unwrap_or(1)
|
|
3410
3408
|
}
|
|
3411
3409
|
|
|
3412
|
-
fn lookup_pair_terms(
|
|
3413
|
-
|
|
3414
|
-
terms: Vec<String>,
|
|
3415
|
-
threads: Option<usize>,
|
|
3416
|
-
) -> PyResult<PairRecords> {
|
|
3417
|
-
let workers = threads
|
|
3418
|
-
.unwrap_or_else(|| default_lookup_workers(terms.len()))
|
|
3410
|
+
fn lookup_pair_terms(db: PathBuf, terms: Vec<String>) -> PyResult<PairRecords> {
|
|
3411
|
+
let workers = default_lookup_workers(terms.len())
|
|
3419
3412
|
.max(1)
|
|
3420
3413
|
.min(terms.len().max(1));
|
|
3421
3414
|
// Open (and schema-validate) the primary, then route pair lookups to shards.
|
|
@@ -3463,11 +3456,7 @@ fn hydrate_curie_rows(database: &ReadOnlyDatabase, curie_ids: &[u32]) -> PyResul
|
|
|
3463
3456
|
Ok(out)
|
|
3464
3457
|
}
|
|
3465
3458
|
|
|
3466
|
-
fn lookup_terms(
|
|
3467
|
-
db: PathBuf,
|
|
3468
|
-
terms: Vec<String>,
|
|
3469
|
-
threads: Option<usize>,
|
|
3470
|
-
) -> PyResult<Vec<(String, Vec<FullmapRecord>)>> {
|
|
3459
|
+
fn lookup_terms(db: PathBuf, terms: Vec<String>) -> PyResult<Vec<(String, Vec<FullmapRecord>)>> {
|
|
3471
3460
|
// Open the primary ONCE (the cached shared-lock handle) for dims/CURIES
|
|
3472
3461
|
// hydration; pair lookups route to the shard files. One handle per file is
|
|
3473
3462
|
// a cache choice, not a lock constraint — read-only opens coexist.
|
|
@@ -3476,8 +3465,7 @@ fn lookup_terms(
|
|
|
3476
3465
|
let category_map = load_string_table(&database, CATEGORIES)?;
|
|
3477
3466
|
let source_map = load_sources(&database)?;
|
|
3478
3467
|
let shards = open_cached_shards(&db)?;
|
|
3479
|
-
let workers =
|
|
3480
|
-
.unwrap_or_else(|| default_lookup_workers(terms.len()))
|
|
3468
|
+
let workers = default_lookup_workers(terms.len())
|
|
3481
3469
|
.max(1)
|
|
3482
3470
|
.min(terms.len().max(1));
|
|
3483
3471
|
let pair_rows = lookup_pair_terms_db(&shards, &terms, workers)?;
|
|
@@ -3519,26 +3507,23 @@ fn lookup_terms(
|
|
|
3519
3507
|
Ok(out)
|
|
3520
3508
|
}
|
|
3521
3509
|
|
|
3522
|
-
/// Look up fullmap records for `terms`.
|
|
3523
|
-
///
|
|
3524
|
-
///
|
|
3525
|
-
///
|
|
3526
|
-
/// explicit `threads=Some(1)` always forces the serial path. The GIL is released
|
|
3527
|
-
/// for the whole lookup.
|
|
3510
|
+
/// Look up fullmap records for `terms`. The worker count is selected
|
|
3511
|
+
/// automatically via `default_lookup_workers`: batches >= `LOOKUP_PARALLEL_MIN`
|
|
3512
|
+
/// fan out across the RECORDS shards in parallel (up to 16 by default), smaller
|
|
3513
|
+
/// batches stay single-threaded. The GIL is released for the whole lookup.
|
|
3528
3514
|
#[pyfunction]
|
|
3529
|
-
#[pyo3(signature = (db, terms,
|
|
3515
|
+
#[pyo3(signature = (db, terms, return_format="rows"))]
|
|
3530
3516
|
pub fn lookup_fullmap_terms<'py>(
|
|
3531
3517
|
py: Python<'py>,
|
|
3532
3518
|
db: PathBuf,
|
|
3533
3519
|
terms: Vec<String>,
|
|
3534
|
-
threads: Option<usize>,
|
|
3535
3520
|
return_format: &str,
|
|
3536
3521
|
) -> PyResult<Bound<'py, PyList>> {
|
|
3537
3522
|
if return_format == "pairs" {
|
|
3538
3523
|
// Release the GIL for the whole lookup (pure-Rust shard reads); the
|
|
3539
3524
|
// PyList is built only after re-acquiring it so rich's Live display
|
|
3540
3525
|
// thread can repaint and Ctrl-C works mid-lookup.
|
|
3541
|
-
let pair_rows = py.detach(move || lookup_pair_terms(db, terms
|
|
3526
|
+
let pair_rows = py.detach(move || lookup_pair_terms(db, terms))?;
|
|
3542
3527
|
let list = PyList::empty(py);
|
|
3543
3528
|
for (term, pairs) in pair_rows {
|
|
3544
3529
|
let row = PyDict::new(py);
|
|
@@ -3556,7 +3541,7 @@ pub fn lookup_fullmap_terms<'py>(
|
|
|
3556
3541
|
// Release the GIL for the whole lookup (pure-Rust shard reads + CURIE/dim
|
|
3557
3542
|
// hydration against the primary); the PyList is built only after
|
|
3558
3543
|
// re-acquiring the GIL.
|
|
3559
|
-
let rows = py.detach(move || lookup_terms(db, terms
|
|
3544
|
+
let rows = py.detach(move || lookup_terms(db, terms))?;
|
|
3560
3545
|
let list = PyList::empty(py);
|
|
3561
3546
|
for (term, records) in rows {
|
|
3562
3547
|
for record in records {
|
|
@@ -3768,12 +3753,8 @@ mod tests {
|
|
|
3768
3753
|
.unwrap();
|
|
3769
3754
|
|
|
3770
3755
|
build_test(output.clone(), vec![classes], vec![synonyms], 1, 4_000_000).unwrap();
|
|
3771
|
-
let rows =
|
|
3772
|
-
output,
|
|
3773
|
-
vec!["brca1".to_string(), "ncbigene672".to_string()],
|
|
3774
|
-
Some(1),
|
|
3775
|
-
)
|
|
3776
|
-
.unwrap();
|
|
3756
|
+
let rows =
|
|
3757
|
+
lookup_terms(output, vec!["brca1".to_string(), "ncbigene672".to_string()]).unwrap();
|
|
3777
3758
|
|
|
3778
3759
|
assert_eq!(rows.len(), 2);
|
|
3779
3760
|
assert_eq!(rows[0].1[0].curie, "HGNC:1100");
|
|
@@ -3836,7 +3817,7 @@ mod tests {
|
|
|
3836
3817
|
}
|
|
3837
3818
|
|
|
3838
3819
|
// The single indexed term still resolves (routed through its shard).
|
|
3839
|
-
let rows = lookup_terms(output, vec!["brca1".to_string()]
|
|
3820
|
+
let rows = lookup_terms(output, vec!["brca1".to_string()]).unwrap();
|
|
3840
3821
|
assert_eq!(rows.len(), 1);
|
|
3841
3822
|
assert_eq!(rows[0].1[0].curie, "HGNC:1100");
|
|
3842
3823
|
}
|
|
@@ -3979,7 +3960,7 @@ mod tests {
|
|
|
3979
3960
|
.flat_map(|i| [format!("gene{i}"), format!("alias{i}")])
|
|
3980
3961
|
.collect();
|
|
3981
3962
|
let norm = |db: PathBuf| -> Vec<(String, Vec<String>)> {
|
|
3982
|
-
let rows = lookup_terms(db, probes.clone()
|
|
3963
|
+
let rows = lookup_terms(db, probes.clone()).unwrap();
|
|
3983
3964
|
let mut out: Vec<(String, Vec<String>)> = rows
|
|
3984
3965
|
.into_iter()
|
|
3985
3966
|
.map(|(t, recs)| {
|
|
@@ -4133,7 +4114,7 @@ mod tests {
|
|
|
4133
4114
|
.flat_map(|i| [format!("gene{i}"), format!("alias{i}")])
|
|
4134
4115
|
.collect();
|
|
4135
4116
|
let norm = |db: PathBuf| -> Vec<(String, Vec<String>)> {
|
|
4136
|
-
let rows = lookup_terms(db, probes.clone()
|
|
4117
|
+
let rows = lookup_terms(db, probes.clone()).unwrap();
|
|
4137
4118
|
let mut out: Vec<(String, Vec<String>)> = rows
|
|
4138
4119
|
.into_iter()
|
|
4139
4120
|
.map(|(t, recs)| {
|
|
@@ -4215,12 +4196,19 @@ mod tests {
|
|
|
4215
4196
|
let got_order: Vec<String> = parallel.iter().map(|(t, _)| t.clone()).collect();
|
|
4216
4197
|
assert_eq!(got_order, expected_order, "merge broke input order");
|
|
4217
4198
|
|
|
4218
|
-
// End-to-end hydration (through the primary)
|
|
4219
|
-
//
|
|
4220
|
-
let
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4199
|
+
// End-to-end hydration (through the primary) agrees with the shard-level
|
|
4200
|
+
// pairs and yields one row group per hit term.
|
|
4201
|
+
let rows = lookup_terms(output, probes.clone()).unwrap();
|
|
4202
|
+
assert_eq!(rows.len(), expected_order.len());
|
|
4203
|
+
let pair_counts: Vec<(String, usize)> = parallel
|
|
4204
|
+
.iter()
|
|
4205
|
+
.map(|(t, pairs)| (t.clone(), pairs.len()))
|
|
4206
|
+
.collect();
|
|
4207
|
+
let row_counts: Vec<(String, usize)> = rows
|
|
4208
|
+
.iter()
|
|
4209
|
+
.map(|(t, recs)| (t.clone(), recs.len()))
|
|
4210
|
+
.collect();
|
|
4211
|
+
assert_eq!(pair_counts, row_counts);
|
|
4224
4212
|
}
|
|
4225
4213
|
|
|
4226
4214
|
/// `split_counts` hands surplus workers to the busiest bucket (most
|
|
@@ -4324,29 +4312,37 @@ mod tests {
|
|
|
4324
4312
|
"bucket splitting broke input order"
|
|
4325
4313
|
);
|
|
4326
4314
|
|
|
4327
|
-
// End-to-end (through `lookup_pair_terms`, which
|
|
4328
|
-
// term count)
|
|
4329
|
-
|
|
4330
|
-
let
|
|
4331
|
-
assert_eq!(
|
|
4332
|
-
let
|
|
4333
|
-
let
|
|
4334
|
-
|
|
4315
|
+
// End-to-end (through `lookup_pair_terms` / `lookup_terms`, which clamp
|
|
4316
|
+
// the auto-selected workers to the term count) matches the shard-level
|
|
4317
|
+
// serial result, through full hydration alike.
|
|
4318
|
+
let pairs_auto = lookup_pair_terms(output.clone(), probes.clone()).unwrap();
|
|
4319
|
+
assert_eq!(pairs_auto, via_serial);
|
|
4320
|
+
let rows_auto = lookup_terms(output, probes.clone()).unwrap();
|
|
4321
|
+
let row_counts: Vec<(String, usize)> = rows_auto
|
|
4322
|
+
.iter()
|
|
4323
|
+
.map(|(t, recs)| (t.clone(), recs.len()))
|
|
4324
|
+
.collect();
|
|
4325
|
+
let pair_counts: Vec<(String, usize)> = via_serial
|
|
4326
|
+
.iter()
|
|
4327
|
+
.map(|(t, pairs)| (t.clone(), pairs.len()))
|
|
4328
|
+
.collect();
|
|
4329
|
+
assert_eq!(row_counts, pair_counts);
|
|
4335
4330
|
}
|
|
4336
4331
|
|
|
4337
|
-
/// The production build-kg resolve calls `lookup_fullmap_terms` with
|
|
4338
|
-
///
|
|
4339
|
-
/// DEFAULT alone
|
|
4332
|
+
/// The production build-kg resolve calls `lookup_fullmap_terms` with no
|
|
4333
|
+
/// thread tuning, so the parallel shard fan-out must kick in from the Rust
|
|
4334
|
+
/// DEFAULT alone. This builds a
|
|
4340
4335
|
/// large fixture and probes it with a batch that crosses `LOOKUP_PARALLEL_MIN`
|
|
4341
4336
|
/// and spans every default shard, then asserts: (a) the default worker count
|
|
4342
4337
|
/// is >1 on any multi-core host, and reaches the 16-shard fan-out cap on a
|
|
4343
4338
|
/// host with at least 16 CPUs (so `lookup_pair_terms_db` can spawn one reader
|
|
4344
|
-
/// per non-empty shard), and (b)
|
|
4345
|
-
/// (content + order) to
|
|
4339
|
+
/// per non-empty shard), and (b) the auto-selected parallel lookup returns
|
|
4340
|
+
/// results IDENTICAL (content + order) to a forced-serial
|
|
4341
|
+
/// `lookup_pair_terms_db(..., 1)`, with misses
|
|
4346
4342
|
/// dropped. On smaller hosts the maximum-fanout assertion is skipped but
|
|
4347
4343
|
/// equivalence still holds.
|
|
4348
4344
|
#[test]
|
|
4349
|
-
fn
|
|
4345
|
+
fn large_batch_lookup_defaults_to_parallel() {
|
|
4350
4346
|
pyo3::Python::initialize();
|
|
4351
4347
|
let dir = tempfile::tempdir().unwrap();
|
|
4352
4348
|
let synonyms = dir.path().join("large.ndjson");
|
|
@@ -4413,7 +4409,7 @@ mod tests {
|
|
|
4413
4409
|
"large batch should be able to fan out across all default shards"
|
|
4414
4410
|
);
|
|
4415
4411
|
}
|
|
4416
|
-
//
|
|
4412
|
+
// Sub-threshold batches stay serial.
|
|
4417
4413
|
assert_eq!(default_lookup_workers(0), 1, "empty batch stays serial");
|
|
4418
4414
|
assert_eq!(
|
|
4419
4415
|
default_lookup_workers(LOOKUP_PARALLEL_MIN - 1),
|
|
@@ -4421,13 +4417,14 @@ mod tests {
|
|
|
4421
4417
|
"sub-threshold batch stays serial"
|
|
4422
4418
|
);
|
|
4423
4419
|
|
|
4424
|
-
//
|
|
4425
|
-
// content AND order, misses dropped.
|
|
4426
|
-
let via_default = lookup_pair_terms(output.clone(), probes.clone()
|
|
4427
|
-
let
|
|
4420
|
+
// The auto-selected lookup (production path) == forced-serial shard read:
|
|
4421
|
+
// identical content AND order, misses dropped.
|
|
4422
|
+
let via_default = lookup_pair_terms(output.clone(), probes.clone()).unwrap();
|
|
4423
|
+
let shards = open_cached_shards(&output).unwrap();
|
|
4424
|
+
let via_serial = lookup_pair_terms_db(&shards, &probes, 1).unwrap();
|
|
4428
4425
|
assert_eq!(
|
|
4429
4426
|
via_default, via_serial,
|
|
4430
|
-
"
|
|
4427
|
+
"auto-selected workers diverged from forced-serial"
|
|
4431
4428
|
);
|
|
4432
4429
|
let expected_order: Vec<String> = probes
|
|
4433
4430
|
.iter()
|
|
@@ -4461,9 +4458,9 @@ mod tests {
|
|
|
4461
4458
|
let body = &tail[..end];
|
|
4462
4459
|
|
|
4463
4460
|
// Rows fetch: exactly one call, and it is the detached one.
|
|
4464
|
-
let rows_calls = body.matches("lookup_terms(db, terms
|
|
4461
|
+
let rows_calls = body.matches("lookup_terms(db, terms)").count();
|
|
4465
4462
|
let rows_detached = body
|
|
4466
|
-
.matches("py.detach(move || lookup_terms(db, terms
|
|
4463
|
+
.matches("py.detach(move || lookup_terms(db, terms)")
|
|
4467
4464
|
.count();
|
|
4468
4465
|
assert_eq!(rows_calls, 1, "rows fetch must be called exactly once");
|
|
4469
4466
|
assert_eq!(
|
|
@@ -4472,11 +4469,9 @@ mod tests {
|
|
|
4472
4469
|
);
|
|
4473
4470
|
|
|
4474
4471
|
// Pairs fetch: exactly one call, and it is the detached one.
|
|
4475
|
-
let pair_calls = body
|
|
4476
|
-
.matches("lookup_pair_terms(db, terms, threads)")
|
|
4477
|
-
.count();
|
|
4472
|
+
let pair_calls = body.matches("lookup_pair_terms(db, terms)").count();
|
|
4478
4473
|
let pair_detached = body
|
|
4479
|
-
.matches("py.detach(move || lookup_pair_terms(db, terms
|
|
4474
|
+
.matches("py.detach(move || lookup_pair_terms(db, terms)")
|
|
4480
4475
|
.count();
|
|
4481
4476
|
assert_eq!(pair_calls, 1, "pairs fetch must be called exactly once");
|
|
4482
4477
|
assert_eq!(
|
|
@@ -4562,7 +4557,7 @@ mod tests {
|
|
|
4562
4557
|
let shards = open_cached_shards(&output).unwrap();
|
|
4563
4558
|
assert_eq!(shards.len(), 2);
|
|
4564
4559
|
let terms: Vec<String> = (0..50).map(|i| format!("gene{i}")).collect();
|
|
4565
|
-
let rows = lookup_terms(output, terms
|
|
4560
|
+
let rows = lookup_terms(output, terms).unwrap();
|
|
4566
4561
|
assert_eq!(rows.len(), 50);
|
|
4567
4562
|
}
|
|
4568
4563
|
|
|
@@ -4593,14 +4588,7 @@ mod tests {
|
|
|
4593
4588
|
|
|
4594
4589
|
std::env::set_var("TABLASSERT_FULLMAP_SHARDS", "2");
|
|
4595
4590
|
let built = Python::attach(|py| {
|
|
4596
|
-
build_fullmap_db(
|
|
4597
|
-
py,
|
|
4598
|
-
output.clone(),
|
|
4599
|
-
Vec::new(),
|
|
4600
|
-
vec![synonyms],
|
|
4601
|
-
Some(2),
|
|
4602
|
-
None,
|
|
4603
|
-
)
|
|
4591
|
+
build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None)
|
|
4604
4592
|
});
|
|
4605
4593
|
std::env::remove_var("TABLASSERT_FULLMAP_SHARDS");
|
|
4606
4594
|
built.unwrap();
|
|
@@ -4623,7 +4611,7 @@ mod tests {
|
|
|
4623
4611
|
|
|
4624
4612
|
// Lookups still resolve across all 16 shards.
|
|
4625
4613
|
let terms: Vec<String> = (0..50).map(|i| format!("gene{i}")).collect();
|
|
4626
|
-
let rows = lookup_terms(output, terms
|
|
4614
|
+
let rows = lookup_terms(output, terms).unwrap();
|
|
4627
4615
|
assert_eq!(rows.len(), 50);
|
|
4628
4616
|
}
|
|
4629
4617
|
|
|
@@ -4692,7 +4680,7 @@ mod tests {
|
|
|
4692
4680
|
}
|
|
4693
4681
|
|
|
4694
4682
|
let terms: Vec<String> = (0..80).map(|i| format!("gene{i}")).collect();
|
|
4695
|
-
let rows = lookup_terms(output, terms
|
|
4683
|
+
let rows = lookup_terms(output, terms).unwrap();
|
|
4696
4684
|
assert_eq!(rows.len(), 80);
|
|
4697
4685
|
}
|
|
4698
4686
|
|
|
@@ -4710,7 +4698,7 @@ mod tests {
|
|
|
4710
4698
|
write.commit().unwrap();
|
|
4711
4699
|
drop(database);
|
|
4712
4700
|
|
|
4713
|
-
let err = lookup_terms(output, vec!["brca1".to_string()]
|
|
4701
|
+
let err = lookup_terms(output, vec!["brca1".to_string()]).unwrap_err();
|
|
4714
4702
|
assert!(err
|
|
4715
4703
|
.to_string()
|
|
4716
4704
|
.contains("fullmap DB is outdated; rebuild with 'tablassert build-fullmap'"));
|
|
@@ -4733,7 +4721,7 @@ mod tests {
|
|
|
4733
4721
|
write.commit().unwrap();
|
|
4734
4722
|
drop(database);
|
|
4735
4723
|
|
|
4736
|
-
let err = lookup_terms(output, vec!["brca1".to_string()]
|
|
4724
|
+
let err = lookup_terms(output, vec!["brca1".to_string()]).unwrap_err();
|
|
4737
4725
|
assert!(err
|
|
4738
4726
|
.to_string()
|
|
4739
4727
|
.contains("fullmap DB is outdated; rebuild with 'tablassert build-fullmap'"));
|
|
@@ -5087,7 +5075,7 @@ mod tests {
|
|
|
5087
5075
|
)
|
|
5088
5076
|
.unwrap();
|
|
5089
5077
|
build_test(output.clone(), Vec::new(), vec![synonyms1], 1, 4_000_000).unwrap();
|
|
5090
|
-
let rows = lookup_terms(output.clone(), vec!["brca1".to_string()]
|
|
5078
|
+
let rows = lookup_terms(output.clone(), vec!["brca1".to_string()]).unwrap();
|
|
5091
5079
|
assert_eq!(rows.len(), 1);
|
|
5092
5080
|
assert_eq!(rows[0].1[0].curie, "HGNC:1100");
|
|
5093
5081
|
|
|
@@ -5105,14 +5093,14 @@ mod tests {
|
|
|
5105
5093
|
swap_build_over(&built, &output);
|
|
5106
5094
|
|
|
5107
5095
|
// Old-generation term must be gone (stale shards would resurrect it).
|
|
5108
|
-
let rows = lookup_terms(output.clone(), vec!["brca1".to_string()]
|
|
5096
|
+
let rows = lookup_terms(output.clone(), vec!["brca1".to_string()]).unwrap();
|
|
5109
5097
|
assert!(
|
|
5110
5098
|
rows.is_empty(),
|
|
5111
5099
|
"stale shard generation resurrected an old term: {rows:?}"
|
|
5112
5100
|
);
|
|
5113
5101
|
// New term resolves against the NEW primary's dims/CURIES (a stale
|
|
5114
5102
|
// primary would hydrate the wrong curie/preferred_name).
|
|
5115
|
-
let rows = lookup_terms(output, vec!["tp53".to_string()]
|
|
5103
|
+
let rows = lookup_terms(output, vec!["tp53".to_string()]).unwrap();
|
|
5116
5104
|
assert_eq!(rows.len(), 1);
|
|
5117
5105
|
assert_eq!(rows[0].1[0].curie, "NCBIGene:7157");
|
|
5118
5106
|
assert_eq!(rows[0].1[0].preferred_name, "TP53");
|
|
@@ -5385,12 +5373,7 @@ mod tests {
|
|
|
5385
5373
|
drop(database);
|
|
5386
5374
|
|
|
5387
5375
|
// Every gene resolves through the streamed CURIES table.
|
|
5388
|
-
let rows = lookup_terms(
|
|
5389
|
-
output,
|
|
5390
|
-
(0..5).map(|i| format!("gene{i}")).collect(),
|
|
5391
|
-
Some(1),
|
|
5392
|
-
)
|
|
5393
|
-
.unwrap();
|
|
5376
|
+
let rows = lookup_terms(output, (0..5).map(|i| format!("gene{i}")).collect()).unwrap();
|
|
5394
5377
|
let mut got: Vec<String> = rows
|
|
5395
5378
|
.iter()
|
|
5396
5379
|
.flat_map(|(_, recs)| recs.iter().map(|r| r.curie.clone()))
|
|
@@ -5419,11 +5402,11 @@ mod tests {
|
|
|
5419
5402
|
|
|
5420
5403
|
build_test(output.clone(), Vec::new(), vec![synonyms], 1, 4_000_000).unwrap();
|
|
5421
5404
|
|
|
5422
|
-
let alive = lookup_terms(output.clone(), vec!["realname".to_string()]
|
|
5405
|
+
let alive = lookup_terms(output.clone(), vec!["realname".to_string()]).unwrap();
|
|
5423
5406
|
assert_eq!(alive.len(), 1);
|
|
5424
5407
|
assert_eq!(alive[0].1[0].curie, "HGNC:1100");
|
|
5425
5408
|
|
|
5426
|
-
let dead = lookup_terms(output, vec!["12345".to_string()]
|
|
5409
|
+
let dead = lookup_terms(output, vec!["12345".to_string()]).unwrap();
|
|
5427
5410
|
assert!(dead.is_empty() || dead[0].1.is_empty());
|
|
5428
5411
|
}
|
|
5429
5412
|
|
|
@@ -5480,11 +5463,11 @@ mod tests {
|
|
|
5480
5463
|
drop(read);
|
|
5481
5464
|
drop(database);
|
|
5482
5465
|
|
|
5483
|
-
let kept = lookup_terms(output.clone(), vec!["water".to_string()]
|
|
5466
|
+
let kept = lookup_terms(output.clone(), vec!["water".to_string()]).unwrap();
|
|
5484
5467
|
assert_eq!(kept.len(), 1);
|
|
5485
5468
|
assert_eq!(kept[0].1[0].curie, "CHEBI:2");
|
|
5486
5469
|
|
|
5487
|
-
let dropped = lookup_terms(output, vec!["genea".to_string()]
|
|
5470
|
+
let dropped = lookup_terms(output, vec!["genea".to_string()]).unwrap();
|
|
5488
5471
|
assert!(dropped.is_empty() || dropped[0].1.is_empty());
|
|
5489
5472
|
}
|
|
5490
5473
|
|
|
@@ -5543,7 +5526,7 @@ mod tests {
|
|
|
5543
5526
|
.into_iter()
|
|
5544
5527
|
.map(|i| format!("gene{i}"))
|
|
5545
5528
|
.collect();
|
|
5546
|
-
let rows = lookup_terms(output, terms
|
|
5529
|
+
let rows = lookup_terms(output, terms).unwrap();
|
|
5547
5530
|
let mut got: Vec<String> = rows
|
|
5548
5531
|
.iter()
|
|
5549
5532
|
.flat_map(|(_, recs)| recs.iter().map(|r| r.curie.clone()))
|
|
@@ -5562,7 +5545,6 @@ mod tests {
|
|
|
5562
5545
|
PathBuf::from("/tmp/should-not-exist.redb"),
|
|
5563
5546
|
Vec::new(),
|
|
5564
5547
|
Vec::new(),
|
|
5565
|
-
Some(1),
|
|
5566
5548
|
None,
|
|
5567
5549
|
)
|
|
5568
5550
|
.expect_err("empty synonyms should fail");
|
|
@@ -5587,7 +5569,7 @@ mod tests {
|
|
|
5587
5569
|
.unwrap();
|
|
5588
5570
|
|
|
5589
5571
|
build_test(output.clone(), Vec::new(), vec![synonyms], 1, 4_000_000).unwrap();
|
|
5590
|
-
let rows = lookup_terms(output, vec!["alias disease".to_string()]
|
|
5572
|
+
let rows = lookup_terms(output, vec!["alias disease".to_string()]).unwrap();
|
|
5591
5573
|
|
|
5592
5574
|
assert_eq!(rows.len(), 1);
|
|
5593
5575
|
assert_eq!(rows[0].1[0].curie, "MONDO:1");
|
|
@@ -5613,7 +5595,6 @@ mod tests {
|
|
|
5613
5595
|
let rows = lookup_terms(
|
|
5614
5596
|
output,
|
|
5615
5597
|
vec!["hypothetical protein".to_string(), "gene1".to_string()],
|
|
5616
|
-
Some(1),
|
|
5617
5598
|
)
|
|
5618
5599
|
.unwrap();
|
|
5619
5600
|
|
|
@@ -5636,7 +5617,7 @@ mod tests {
|
|
|
5636
5617
|
.unwrap();
|
|
5637
5618
|
|
|
5638
5619
|
build_test(output.clone(), Vec::new(), vec![synonyms], 1, 4_000_000).unwrap();
|
|
5639
|
-
let rows = lookup_terms(output, vec!["quoted gene".to_string()]
|
|
5620
|
+
let rows = lookup_terms(output, vec!["quoted gene".to_string()]).unwrap();
|
|
5640
5621
|
|
|
5641
5622
|
assert_eq!(rows.len(), 1);
|
|
5642
5623
|
assert_eq!(rows[0].1[0].preferred_name, "Quoted Gene");
|
|
@@ -5677,8 +5658,8 @@ mod tests {
|
|
|
5677
5658
|
"beta".to_string(),
|
|
5678
5659
|
"delta".to_string(),
|
|
5679
5660
|
];
|
|
5680
|
-
let big = lookup_terms(out_big, probes.clone()
|
|
5681
|
-
let tiny = lookup_terms(out_tiny, probes
|
|
5661
|
+
let big = lookup_terms(out_big, probes.clone()).unwrap();
|
|
5662
|
+
let tiny = lookup_terms(out_tiny, probes).unwrap();
|
|
5682
5663
|
|
|
5683
5664
|
// Same terms resolved, same hydrated records (order-independent compare).
|
|
5684
5665
|
let norm = |v: Vec<(String, Vec<FullmapRecord>)>| -> Vec<(String, Vec<String>)> {
|
|
@@ -5727,7 +5708,7 @@ mod tests {
|
|
|
5727
5708
|
|
|
5728
5709
|
// Several consecutive lookups (rows path) must all succeed.
|
|
5729
5710
|
for _ in 0..3 {
|
|
5730
|
-
let rows = lookup_terms(output.clone(), vec!["brca1".to_string()]
|
|
5711
|
+
let rows = lookup_terms(output.clone(), vec!["brca1".to_string()]).unwrap();
|
|
5731
5712
|
assert_eq!(rows.len(), 1);
|
|
5732
5713
|
}
|
|
5733
5714
|
// open_cached returns the same handle across calls.
|
|
@@ -6033,7 +6014,7 @@ mod tests {
|
|
|
6033
6014
|
}
|
|
6034
6015
|
write.commit().unwrap();
|
|
6035
6016
|
drop(database);
|
|
6036
|
-
let err = lookup_terms(v2, vec!["brca1".to_string()]
|
|
6017
|
+
let err = lookup_terms(v2, vec!["brca1".to_string()]).unwrap_err();
|
|
6037
6018
|
assert!(err
|
|
6038
6019
|
.to_string()
|
|
6039
6020
|
.contains("fullmap DB is outdated; rebuild with 'tablassert build-fullmap'"));
|
|
@@ -6049,7 +6030,7 @@ mod tests {
|
|
|
6049
6030
|
}
|
|
6050
6031
|
write.commit().unwrap();
|
|
6051
6032
|
drop(database);
|
|
6052
|
-
let err = lookup_terms(missing, vec!["brca1".to_string()]
|
|
6033
|
+
let err = lookup_terms(missing, vec!["brca1".to_string()]).unwrap_err();
|
|
6053
6034
|
assert!(err.to_string().contains("unsupported fullmap redb schema"));
|
|
6054
6035
|
|
|
6055
6036
|
// Garbage schema value => unsupported.
|
|
@@ -6062,7 +6043,7 @@ mod tests {
|
|
|
6062
6043
|
}
|
|
6063
6044
|
write.commit().unwrap();
|
|
6064
6045
|
drop(database);
|
|
6065
|
-
let err = lookup_terms(garbage, vec!["brca1".to_string()]
|
|
6046
|
+
let err = lookup_terms(garbage, vec!["brca1".to_string()]).unwrap_err();
|
|
6066
6047
|
assert!(err.to_string().contains("unsupported fullmap redb schema"));
|
|
6067
6048
|
}
|
|
6068
6049
|
|