tablassert 17.0.1__tar.gz → 18.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-17.0.1 → tablassert-18.1.0}/PKG-INFO +7 -3
- {tablassert-17.0.1 → tablassert-18.1.0}/README.md +6 -2
- {tablassert-17.0.1 → tablassert-18.1.0}/pyproject.toml +2 -1
- {tablassert-17.0.1 → tablassert-18.1.0}/rust/src/fullmap.rs +102 -4
- {tablassert-17.0.1 → tablassert-18.1.0}/rust/src/lib.rs +57 -2
- {tablassert-17.0.1 → tablassert-18.1.0}/rust/src/ndjson.rs +159 -2
- {tablassert-17.0.1 → tablassert-18.1.0}/rust/tests/build_golden.rs +74 -6
- {tablassert-17.0.1 → tablassert-18.1.0}/rust/tests/common/mod.rs +9 -2
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/biolink.py +62 -20
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/cli.py +124 -31
- tablassert-18.1.0/src/tablassert/data/experimental_taxa.yaml +511 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/errors.py +22 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/fullmap.py +30 -8
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/lib.py +5 -15
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/models.py +3 -4
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/rs.pyi +6 -1
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/study.py +38 -3
- tablassert-18.1.0/src/tablassert/utils.py +53 -0
- tablassert-17.0.1/src/tablassert/utils.py +0 -18
- {tablassert-17.0.1 → tablassert-18.1.0}/LICENSE +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/rust/Cargo.lock +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/rust/Cargo.toml +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/rust/src/json.rs +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/rust/src/uuid.rs +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/__init__.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/agent.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/coerce.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/distill.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/enums.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/extras.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/ingests.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/log.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/nlp.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/progress.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/qc.py +0 -0
- {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/rig.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version:
|
|
3
|
+
Version: 18.1.0
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -92,7 +92,7 @@ template:
|
|
|
92
92
|
source:
|
|
93
93
|
kind: text
|
|
94
94
|
local: ./gene-disease.csv
|
|
95
|
-
url: https://example.com/data.csv
|
|
95
|
+
url: [https://example.com/data.csv]
|
|
96
96
|
row_slice: [1, auto]
|
|
97
97
|
delimiter: ","
|
|
98
98
|
statement:
|
|
@@ -178,8 +178,9 @@ CSV/TSV/Excel sources; optional extras add runtime and pipeline capabilities:
|
|
|
178
178
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
179
179
|
| `aria2` | bundled aria2c downloader for `build-fullmap --aria2c` (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
180
180
|
| `qc` | four-stage QC audit (exact → fuzzy → abbreviation → SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
181
|
-
| `agent` | autonomous agent (smolagents, litellm,
|
|
181
|
+
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
182
182
|
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
183
|
+
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
183
184
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
184
185
|
|
|
185
186
|
QC is opt-in at build time (`build-kg --qc`). Reaching a feature whose extra is not installed never
|
|
@@ -215,9 +216,12 @@ no LazyFrame setup or NLP preprocessing required. See the
|
|
|
215
216
|
- **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
|
|
216
217
|
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
217
218
|
- **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
|
|
219
|
+
- **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
|
|
218
220
|
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
219
221
|
- **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
|
|
220
222
|
- **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
|
|
223
|
+
- **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
|
|
224
|
+
- **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
|
|
221
225
|
|
|
222
226
|
## Developing
|
|
223
227
|
|
|
@@ -33,7 +33,7 @@ template:
|
|
|
33
33
|
source:
|
|
34
34
|
kind: text
|
|
35
35
|
local: ./gene-disease.csv
|
|
36
|
-
url: https://example.com/data.csv
|
|
36
|
+
url: [https://example.com/data.csv]
|
|
37
37
|
row_slice: [1, auto]
|
|
38
38
|
delimiter: ","
|
|
39
39
|
statement:
|
|
@@ -119,8 +119,9 @@ CSV/TSV/Excel sources; optional extras add runtime and pipeline capabilities:
|
|
|
119
119
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
120
120
|
| `aria2` | bundled aria2c downloader for `build-fullmap --aria2c` (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
121
121
|
| `qc` | four-stage QC audit (exact → fuzzy → abbreviation → SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
122
|
-
| `agent` | autonomous agent (smolagents, litellm,
|
|
122
|
+
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
123
123
|
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
124
|
+
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
124
125
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
125
126
|
|
|
126
127
|
QC is opt-in at build time (`build-kg --qc`). Reaching a feature whose extra is not installed never
|
|
@@ -156,9 +157,12 @@ no LazyFrame setup or NLP preprocessing required. See the
|
|
|
156
157
|
- **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
|
|
157
158
|
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
158
159
|
- **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
|
|
160
|
+
- **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
|
|
159
161
|
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
160
162
|
- **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
|
|
161
163
|
- **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
|
|
164
|
+
- **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
|
|
165
|
+
- **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
|
|
162
166
|
|
|
163
167
|
## Developing
|
|
164
168
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "
|
|
3
|
+
version = "18.1.0"
|
|
4
4
|
description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -72,6 +72,7 @@ module-name = "tablassert.rs"
|
|
|
72
72
|
bindings = "pyo3"
|
|
73
73
|
manifest-path = "rust/Cargo.toml"
|
|
74
74
|
features = ["extension-module"]
|
|
75
|
+
include = ["src/tablassert/data/experimental_taxa.yaml"]
|
|
75
76
|
|
|
76
77
|
[project.scripts]
|
|
77
78
|
tablassert = "tablassert.cli:APP"
|
|
@@ -294,7 +294,6 @@ fn token_qc(value: &str) -> bool {
|
|
|
294
294
|
&& !value.contains('\t')
|
|
295
295
|
&& !value.contains('\n')
|
|
296
296
|
&& !value.contains('\r')
|
|
297
|
-
&& !value.contains("inchikey")
|
|
298
297
|
&& !value.contains("uncharacterized")
|
|
299
298
|
&& !value.contains("hypothetical")
|
|
300
299
|
}
|
|
@@ -459,6 +458,15 @@ fn first_category(row: &SynonymRow<'_>) -> String {
|
|
|
459
458
|
.to_string()
|
|
460
459
|
}
|
|
461
460
|
|
|
461
|
+
fn is_organism_taxon(row: &SynonymRow<'_>) -> bool {
|
|
462
|
+
row.types.iter().any(|category| {
|
|
463
|
+
category
|
|
464
|
+
.trim()
|
|
465
|
+
.trim_start_matches("biolink:")
|
|
466
|
+
.eq_ignore_ascii_case("OrganismTaxon")
|
|
467
|
+
})
|
|
468
|
+
}
|
|
469
|
+
|
|
462
470
|
fn first_taxon(row: &SynonymRow<'_>) -> i32 {
|
|
463
471
|
row.taxa
|
|
464
472
|
.first()
|
|
@@ -469,6 +477,56 @@ fn first_taxon(row: &SynonymRow<'_>) -> i32 {
|
|
|
469
477
|
.unwrap_or(0)
|
|
470
478
|
}
|
|
471
479
|
|
|
480
|
+
fn parsed_taxa(row: &SynonymRow<'_>) -> Vec<i32> {
|
|
481
|
+
row.taxa
|
|
482
|
+
.iter()
|
|
483
|
+
.filter_map(|taxon| {
|
|
484
|
+
let taxon = taxon.trim();
|
|
485
|
+
let local = taxon
|
|
486
|
+
.split_once(':')
|
|
487
|
+
.filter(|(prefix, _)| prefix.eq_ignore_ascii_case("NCBITaxon"))
|
|
488
|
+
.map(|(_, local)| local)
|
|
489
|
+
.unwrap_or(taxon);
|
|
490
|
+
local.parse::<i32>().ok().filter(|id| *id > 0)
|
|
491
|
+
})
|
|
492
|
+
.collect()
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
fn allowlist_identity(taxa: &HashSet<i32>) -> String {
|
|
496
|
+
let mut ids: Vec<i32> = taxa.iter().copied().collect();
|
|
497
|
+
ids.sort_unstable();
|
|
498
|
+
let encoded = ids
|
|
499
|
+
.iter()
|
|
500
|
+
.map(i32::to_string)
|
|
501
|
+
.collect::<Vec<String>>()
|
|
502
|
+
.join(",");
|
|
503
|
+
format!(
|
|
504
|
+
"count={};xxh64={:016x}",
|
|
505
|
+
ids.len(),
|
|
506
|
+
xxh64(encoded.as_bytes(), 0)
|
|
507
|
+
)
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
fn retain_for_taxon_allowlist(row: &SynonymRow<'_>, allowlist: Option<&HashSet<i32>>) -> bool {
|
|
511
|
+
let Some(allowlist) = allowlist else {
|
|
512
|
+
return true;
|
|
513
|
+
};
|
|
514
|
+
if is_organism_taxon(row) {
|
|
515
|
+
return true;
|
|
516
|
+
}
|
|
517
|
+
let mut has_valid_taxon = false;
|
|
518
|
+
for taxon in parsed_taxa(row) {
|
|
519
|
+
has_valid_taxon = true;
|
|
520
|
+
if allowlist.contains(&taxon) {
|
|
521
|
+
return true;
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
// Taxonless rows remain available for categories whose BABEL synonym rows
|
|
525
|
+
// do not carry a positive taxon ID. This includes the conventional
|
|
526
|
+
// `NCBITaxon:0` sentinel and malformed/empty taxon values.
|
|
527
|
+
!has_valid_taxon
|
|
528
|
+
}
|
|
529
|
+
|
|
472
530
|
fn split_curie(curie: &str) -> Option<(&str, &str)> {
|
|
473
531
|
curie
|
|
474
532
|
.split_once(':')
|
|
@@ -1358,6 +1416,7 @@ struct SynonymShared<'a> {
|
|
|
1358
1416
|
curie_counter: &'a AtomicU32,
|
|
1359
1417
|
equivalents: &'a EquivIndex,
|
|
1360
1418
|
exclude_prefixes: &'a HashSet<String>,
|
|
1419
|
+
taxon_allowlist: Option<&'a HashSet<i32>>,
|
|
1361
1420
|
spill_dir: &'a Path,
|
|
1362
1421
|
local_spill: usize,
|
|
1363
1422
|
curie_spill: usize,
|
|
@@ -1393,6 +1452,9 @@ fn process_row(
|
|
|
1393
1452
|
let Some((prefix, local_id)) = split_curie(curie) else {
|
|
1394
1453
|
return Ok(());
|
|
1395
1454
|
};
|
|
1455
|
+
if !retain_for_taxon_allowlist(row, sh.taxon_allowlist) {
|
|
1456
|
+
return Ok(());
|
|
1457
|
+
}
|
|
1396
1458
|
// Skip CURIEs whose prefix the caller excludes (opt-in via
|
|
1397
1459
|
// TABLASSERT_FULLMAP_EXCLUDE_PREFIXES) — filtered out downstream anyway.
|
|
1398
1460
|
if sh.exclude_prefixes.contains(prefix) {
|
|
@@ -1623,6 +1685,7 @@ fn process_synonyms(
|
|
|
1623
1685
|
local_spill: usize,
|
|
1624
1686
|
curie_spill: usize,
|
|
1625
1687
|
exclude_prefixes: &HashSet<String>,
|
|
1688
|
+
taxon_allowlist: Option<&HashSet<i32>>,
|
|
1626
1689
|
worker_count: usize,
|
|
1627
1690
|
shard_count: usize,
|
|
1628
1691
|
chunk_bytes: usize,
|
|
@@ -1671,6 +1734,7 @@ fn process_synonyms(
|
|
|
1671
1734
|
curie_counter: &curie_counter,
|
|
1672
1735
|
equivalents,
|
|
1673
1736
|
exclude_prefixes,
|
|
1737
|
+
taxon_allowlist,
|
|
1674
1738
|
spill_dir,
|
|
1675
1739
|
local_spill,
|
|
1676
1740
|
curie_spill,
|
|
@@ -1799,6 +1863,7 @@ fn write_final_database(
|
|
|
1799
1863
|
insert_batch: usize,
|
|
1800
1864
|
shard_count: usize,
|
|
1801
1865
|
progress: Option<&Arc<Progress>>,
|
|
1866
|
+
taxon_allowlist_identity: Option<&str>,
|
|
1802
1867
|
) -> PyResult<()> {
|
|
1803
1868
|
let build_id = new_build_id();
|
|
1804
1869
|
let build_id_str = build_id.to_string();
|
|
@@ -1855,6 +1920,9 @@ fn write_final_database(
|
|
|
1855
1920
|
let shard_count_str = shard_count.to_string();
|
|
1856
1921
|
meta.insert("shards", shard_count_str.as_str())
|
|
1857
1922
|
.map_err(py_err)?;
|
|
1923
|
+
if let Some(identity) = taxon_allowlist_identity {
|
|
1924
|
+
meta.insert("taxon_allowlist", identity).map_err(py_err)?;
|
|
1925
|
+
}
|
|
1858
1926
|
}
|
|
1859
1927
|
write.commit().map_err(py_err)?;
|
|
1860
1928
|
|
|
@@ -2229,6 +2297,8 @@ fn build_fullmap_inner(
|
|
|
2229
2297
|
local_spill: usize,
|
|
2230
2298
|
curie_spill: usize,
|
|
2231
2299
|
exclude_prefixes: HashSet<String>,
|
|
2300
|
+
taxon_allowlist: Option<HashSet<i32>>,
|
|
2301
|
+
taxon_allowlist_identity: Option<String>,
|
|
2232
2302
|
chunk_bytes: usize,
|
|
2233
2303
|
producers: usize,
|
|
2234
2304
|
cache_bytes: usize,
|
|
@@ -2258,6 +2328,7 @@ fn build_fullmap_inner(
|
|
|
2258
2328
|
local_spill,
|
|
2259
2329
|
curie_spill,
|
|
2260
2330
|
&exclude_prefixes,
|
|
2331
|
+
taxon_allowlist.as_ref(),
|
|
2261
2332
|
worker_count,
|
|
2262
2333
|
shard_count,
|
|
2263
2334
|
chunk_bytes,
|
|
@@ -2288,6 +2359,7 @@ fn build_fullmap_inner(
|
|
|
2288
2359
|
insert_batch,
|
|
2289
2360
|
shard_count,
|
|
2290
2361
|
progress.as_ref(),
|
|
2362
|
+
taxon_allowlist_identity.as_deref(),
|
|
2291
2363
|
)?;
|
|
2292
2364
|
|
|
2293
2365
|
// Clean up spill runs on success (left in place on error for inspection).
|
|
@@ -2297,13 +2369,14 @@ fn build_fullmap_inner(
|
|
|
2297
2369
|
}
|
|
2298
2370
|
|
|
2299
2371
|
#[pyfunction]
|
|
2300
|
-
#[pyo3(signature = (output, classes, synonyms, progress=None))]
|
|
2372
|
+
#[pyo3(signature = (output, classes, synonyms, progress=None, taxon_allowlist=None))]
|
|
2301
2373
|
pub fn build_fullmap_db(
|
|
2302
2374
|
py: Python<'_>,
|
|
2303
2375
|
output: PathBuf,
|
|
2304
2376
|
classes: Vec<PathBuf>,
|
|
2305
2377
|
synonyms: Vec<PathBuf>,
|
|
2306
2378
|
progress: Option<Py<PyAny>>,
|
|
2379
|
+
taxon_allowlist: Option<Vec<i32>>,
|
|
2307
2380
|
) -> PyResult<()> {
|
|
2308
2381
|
if synonyms.is_empty() {
|
|
2309
2382
|
return Err(PyValueError::new_err(
|
|
@@ -2407,6 +2480,9 @@ pub fn build_fullmap_db(
|
|
|
2407
2480
|
PathBuf::from(p)
|
|
2408
2481
|
});
|
|
2409
2482
|
|
|
2483
|
+
let taxon_allowlist: Option<HashSet<i32>> =
|
|
2484
|
+
taxon_allowlist.map(|ids| ids.into_iter().filter(|id| *id > 0).collect());
|
|
2485
|
+
let taxon_allowlist_identity: Option<String> = taxon_allowlist.as_ref().map(allowlist_identity);
|
|
2410
2486
|
let progress = progress.map(|cb| Arc::new(Progress { cb }));
|
|
2411
2487
|
|
|
2412
2488
|
// Release the GIL for the whole build so rich's Live display thread can
|
|
@@ -2422,6 +2498,8 @@ pub fn build_fullmap_db(
|
|
|
2422
2498
|
local_spill,
|
|
2423
2499
|
curie_spill,
|
|
2424
2500
|
exclude_prefixes,
|
|
2501
|
+
taxon_allowlist,
|
|
2502
|
+
taxon_allowlist_identity,
|
|
2425
2503
|
chunk_bytes,
|
|
2426
2504
|
producers,
|
|
2427
2505
|
cache_bytes,
|
|
@@ -3650,6 +3728,8 @@ mod tests {
|
|
|
3650
3728
|
local_spill,
|
|
3651
3729
|
1_000_000,
|
|
3652
3730
|
HashSet::new(),
|
|
3731
|
+
None,
|
|
3732
|
+
None,
|
|
3653
3733
|
DEFAULT_CHUNK_BYTES,
|
|
3654
3734
|
2,
|
|
3655
3735
|
64 * 1024 * 1024,
|
|
@@ -3678,10 +3758,12 @@ mod tests {
|
|
|
3678
3758
|
}
|
|
3679
3759
|
|
|
3680
3760
|
#[test]
|
|
3681
|
-
fn
|
|
3761
|
+
fn token_qc_rejects_banned_tokens_but_accepts_inchikey() {
|
|
3682
3762
|
assert!(token_qc("brca1"));
|
|
3763
|
+
assert!(token_qc("inchikey=abc"));
|
|
3683
3764
|
assert!(!token_qc("hypothetical protein"));
|
|
3684
3765
|
assert!(!token_qc("line\nbreak"));
|
|
3766
|
+
assert!(!token_qc("tab\tvalue"));
|
|
3685
3767
|
}
|
|
3686
3768
|
|
|
3687
3769
|
#[test]
|
|
@@ -4028,6 +4110,7 @@ mod tests {
|
|
|
4028
4110
|
30,
|
|
4029
4111
|
1_000_000,
|
|
4030
4112
|
&HashSet::new(),
|
|
4113
|
+
None,
|
|
4031
4114
|
4,
|
|
4032
4115
|
shard_count,
|
|
4033
4116
|
DEFAULT_CHUNK_BYTES,
|
|
@@ -4085,6 +4168,8 @@ mod tests {
|
|
|
4085
4168
|
100,
|
|
4086
4169
|
1_000_000,
|
|
4087
4170
|
HashSet::new(),
|
|
4171
|
+
None,
|
|
4172
|
+
None,
|
|
4088
4173
|
DEFAULT_CHUNK_BYTES,
|
|
4089
4174
|
2,
|
|
4090
4175
|
64 * 1024 * 1024,
|
|
@@ -4102,6 +4187,8 @@ mod tests {
|
|
|
4102
4187
|
100,
|
|
4103
4188
|
1_000_000,
|
|
4104
4189
|
HashSet::new(),
|
|
4190
|
+
None,
|
|
4191
|
+
None,
|
|
4105
4192
|
DEFAULT_CHUNK_BYTES,
|
|
4106
4193
|
2,
|
|
4107
4194
|
64 * 1024 * 1024,
|
|
@@ -4519,6 +4606,8 @@ mod tests {
|
|
|
4519
4606
|
4_000_000,
|
|
4520
4607
|
1_000_000,
|
|
4521
4608
|
HashSet::new(),
|
|
4609
|
+
None,
|
|
4610
|
+
None,
|
|
4522
4611
|
DEFAULT_CHUNK_BYTES,
|
|
4523
4612
|
2,
|
|
4524
4613
|
64 * 1024 * 1024,
|
|
@@ -4588,7 +4677,7 @@ mod tests {
|
|
|
4588
4677
|
|
|
4589
4678
|
std::env::set_var("TABLASSERT_FULLMAP_SHARDS", "2");
|
|
4590
4679
|
let built = Python::attach(|py| {
|
|
4591
|
-
build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None)
|
|
4680
|
+
build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None, None)
|
|
4592
4681
|
});
|
|
4593
4682
|
std::env::remove_var("TABLASSERT_FULLMAP_SHARDS");
|
|
4594
4683
|
built.unwrap();
|
|
@@ -4649,6 +4738,8 @@ mod tests {
|
|
|
4649
4738
|
4_000_000,
|
|
4650
4739
|
1_000_000,
|
|
4651
4740
|
HashSet::new(),
|
|
4741
|
+
None,
|
|
4742
|
+
None,
|
|
4652
4743
|
DEFAULT_CHUNK_BYTES,
|
|
4653
4744
|
2,
|
|
4654
4745
|
64 * 1024 * 1024,
|
|
@@ -5356,6 +5447,8 @@ mod tests {
|
|
|
5356
5447
|
4_000_000,
|
|
5357
5448
|
2,
|
|
5358
5449
|
HashSet::new(),
|
|
5450
|
+
None,
|
|
5451
|
+
None,
|
|
5359
5452
|
DEFAULT_CHUNK_BYTES,
|
|
5360
5453
|
2,
|
|
5361
5454
|
64 * 1024 * 1024,
|
|
@@ -5446,6 +5539,8 @@ mod tests {
|
|
|
5446
5539
|
4_000_000,
|
|
5447
5540
|
1_000_000,
|
|
5448
5541
|
exclude,
|
|
5542
|
+
None,
|
|
5543
|
+
None,
|
|
5449
5544
|
DEFAULT_CHUNK_BYTES,
|
|
5450
5545
|
2,
|
|
5451
5546
|
64 * 1024 * 1024,
|
|
@@ -5505,6 +5600,8 @@ mod tests {
|
|
|
5505
5600
|
4_000_000,
|
|
5506
5601
|
1_000_000,
|
|
5507
5602
|
HashSet::new(),
|
|
5603
|
+
None,
|
|
5604
|
+
None,
|
|
5508
5605
|
8192,
|
|
5509
5606
|
2,
|
|
5510
5607
|
64 * 1024 * 1024,
|
|
@@ -5546,6 +5643,7 @@ mod tests {
|
|
|
5546
5643
|
Vec::new(),
|
|
5547
5644
|
Vec::new(),
|
|
5548
5645
|
None,
|
|
5646
|
+
None,
|
|
5549
5647
|
)
|
|
5550
5648
|
.expect_err("empty synonyms should fail");
|
|
5551
5649
|
assert!(err
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
use pyo3::exceptions::PyOSError;
|
|
1
2
|
use pyo3::prelude::*;
|
|
2
3
|
|
|
3
4
|
// mimalloc returns memory to the OS far better than glibc malloc under heavy
|
|
@@ -5,7 +6,10 @@ use pyo3::prelude::*;
|
|
|
5
6
|
// making millions of small String/Vec allocations). Without it, per-thread
|
|
6
7
|
// malloc arenas retain freed memory and inflate peak RSS several-fold.
|
|
7
8
|
use mimalloc::MiMalloc;
|
|
8
|
-
use
|
|
9
|
+
use std::fs::File;
|
|
10
|
+
use std::io::Read;
|
|
11
|
+
use std::path::PathBuf;
|
|
12
|
+
use xxhash_rust::xxh64::{xxh64 as xxh64_digest, Xxh64};
|
|
9
13
|
|
|
10
14
|
#[global_allocator]
|
|
11
15
|
static GLOBAL: MiMalloc = MiMalloc;
|
|
@@ -26,6 +30,27 @@ fn xxh64(data: &str) -> String {
|
|
|
26
30
|
format!("{:016x}", xxh64_digest(data.as_bytes(), 0))
|
|
27
31
|
}
|
|
28
32
|
|
|
33
|
+
/// XXH64 hex digest (seed 0) of a file's raw bytes, streamed in fixed-size chunks.
|
|
34
|
+
#[pyfunction]
|
|
35
|
+
fn xxh64_file(path: PathBuf) -> PyResult<String> {
|
|
36
|
+
const CHUNK_SIZE: usize = 8 * 1024 * 1024;
|
|
37
|
+
|
|
38
|
+
let mut file = File::open(&path)
|
|
39
|
+
.map_err(|error| PyOSError::new_err(format!("{}: {}", path.display(), error)))?;
|
|
40
|
+
let mut hasher = Xxh64::new(0);
|
|
41
|
+
let mut chunk = vec![0; CHUNK_SIZE];
|
|
42
|
+
loop {
|
|
43
|
+
let bytes_read = file
|
|
44
|
+
.read(&mut chunk)
|
|
45
|
+
.map_err(|error| PyOSError::new_err(format!("{}: {}", path.display(), error)))?;
|
|
46
|
+
if bytes_read == 0 {
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
hasher.update(&chunk[..bytes_read]);
|
|
50
|
+
}
|
|
51
|
+
Ok(format!("{:016x}", hasher.digest()))
|
|
52
|
+
}
|
|
53
|
+
|
|
29
54
|
// Public re-exports of the fullmap `#[pyfunction]`s so Rust integration tests
|
|
30
55
|
// (`rust/tests/`) and other non-Python embedders can drive the exact production
|
|
31
56
|
// build/read path. The `fullmap` module itself stays private; only these
|
|
@@ -48,12 +73,13 @@ fn rs(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
|
48
73
|
module.add_function(wrap_pyfunction!(ndjson::dedup_ndjson, module)?)?;
|
|
49
74
|
module.add_function(wrap_pyfunction!(uuid::namespace_uuid, module)?)?;
|
|
50
75
|
module.add_function(wrap_pyfunction!(xxh64, module)?)?;
|
|
76
|
+
module.add_function(wrap_pyfunction!(xxh64_file, module)?)?;
|
|
51
77
|
Ok(())
|
|
52
78
|
}
|
|
53
79
|
|
|
54
80
|
#[cfg(test)]
|
|
55
81
|
mod tests {
|
|
56
|
-
use super::xxh64;
|
|
82
|
+
use super::{xxh64, xxh64_file};
|
|
57
83
|
|
|
58
84
|
#[test]
|
|
59
85
|
fn xxh64_matches_known_digests() {
|
|
@@ -63,4 +89,33 @@ mod tests {
|
|
|
63
89
|
assert_eq!(xxh64("hello"), "26c7827d889f6da3");
|
|
64
90
|
assert_eq!(xxh64(""), "ef46db3751d8e999");
|
|
65
91
|
}
|
|
92
|
+
|
|
93
|
+
#[test]
|
|
94
|
+
fn xxh64_file_matches_known_digests() {
|
|
95
|
+
// WHY: pin file hashing to the same seed-0 digest as the string primitive,
|
|
96
|
+
// including the empty-file identity value and UTF-8 bytes.
|
|
97
|
+
let dir = tempfile::tempdir().unwrap();
|
|
98
|
+
let path = dir.path().join("hello");
|
|
99
|
+
std::fs::write(&path, b"hello").unwrap();
|
|
100
|
+
assert_eq!(xxh64_file(path).unwrap(), "26c7827d889f6da3");
|
|
101
|
+
|
|
102
|
+
let empty = dir.path().join("empty");
|
|
103
|
+
std::fs::write(&empty, b"").unwrap();
|
|
104
|
+
assert_eq!(xxh64_file(empty).unwrap(), "ef46db3751d8e999");
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
#[test]
|
|
108
|
+
fn xxh64_file_reports_io_failures_as_os_error() {
|
|
109
|
+
// WHY: callers must distinguish I/O failures from valid digests and see
|
|
110
|
+
// the offending path, including for missing files and directories.
|
|
111
|
+
let dir = tempfile::tempdir().unwrap();
|
|
112
|
+
let missing = dir.path().join("missing");
|
|
113
|
+
let error = xxh64_file(missing.clone()).unwrap_err();
|
|
114
|
+
assert!(error.to_string().contains(&missing.display().to_string()));
|
|
115
|
+
|
|
116
|
+
let error = xxh64_file(dir.path().to_path_buf()).unwrap_err();
|
|
117
|
+
assert!(error
|
|
118
|
+
.to_string()
|
|
119
|
+
.contains(&dir.path().display().to_string()));
|
|
120
|
+
}
|
|
66
121
|
}
|