tablassert 17.0.1__tar.gz → 18.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. {tablassert-17.0.1 → tablassert-18.1.0}/PKG-INFO +7 -3
  2. {tablassert-17.0.1 → tablassert-18.1.0}/README.md +6 -2
  3. {tablassert-17.0.1 → tablassert-18.1.0}/pyproject.toml +2 -1
  4. {tablassert-17.0.1 → tablassert-18.1.0}/rust/src/fullmap.rs +102 -4
  5. {tablassert-17.0.1 → tablassert-18.1.0}/rust/src/lib.rs +57 -2
  6. {tablassert-17.0.1 → tablassert-18.1.0}/rust/src/ndjson.rs +159 -2
  7. {tablassert-17.0.1 → tablassert-18.1.0}/rust/tests/build_golden.rs +74 -6
  8. {tablassert-17.0.1 → tablassert-18.1.0}/rust/tests/common/mod.rs +9 -2
  9. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/biolink.py +62 -20
  10. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/cli.py +124 -31
  11. tablassert-18.1.0/src/tablassert/data/experimental_taxa.yaml +511 -0
  12. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/errors.py +22 -0
  13. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/fullmap.py +30 -8
  14. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/lib.py +5 -15
  15. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/models.py +3 -4
  16. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/rs.pyi +6 -1
  17. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/study.py +38 -3
  18. tablassert-18.1.0/src/tablassert/utils.py +53 -0
  19. tablassert-17.0.1/src/tablassert/utils.py +0 -18
  20. {tablassert-17.0.1 → tablassert-18.1.0}/LICENSE +0 -0
  21. {tablassert-17.0.1 → tablassert-18.1.0}/rust/Cargo.lock +0 -0
  22. {tablassert-17.0.1 → tablassert-18.1.0}/rust/Cargo.toml +0 -0
  23. {tablassert-17.0.1 → tablassert-18.1.0}/rust/examples/count_tables.rs +0 -0
  24. {tablassert-17.0.1 → tablassert-18.1.0}/rust/src/json.rs +0 -0
  25. {tablassert-17.0.1 → tablassert-18.1.0}/rust/src/uuid.rs +0 -0
  26. {tablassert-17.0.1 → tablassert-18.1.0}/rust/tests/extract_prebuilt.rs +0 -0
  27. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/__init__.py +0 -0
  28. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/_lazy.py +0 -0
  29. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/agent.py +0 -0
  30. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/coerce.py +0 -0
  31. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/distill.py +0 -0
  32. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/enums.py +0 -0
  33. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/extras.py +0 -0
  34. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/graph_target.py +0 -0
  35. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/ingests.py +0 -0
  36. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/log.py +0 -0
  37. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/nlp.py +0 -0
  38. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/progress.py +0 -0
  39. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/qc.py +0 -0
  40. {tablassert-17.0.1 → tablassert-18.1.0}/src/tablassert/rig.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 17.0.1
3
+ Version: 18.1.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -92,7 +92,7 @@ template:
92
92
  source:
93
93
  kind: text
94
94
  local: ./gene-disease.csv
95
- url: https://example.com/data.csv
95
+ url: [https://example.com/data.csv]
96
96
  row_slice: [1, auto]
97
97
  delimiter: ","
98
98
  statement:
@@ -178,8 +178,9 @@ CSV/TSV/Excel sources; optional extras add runtime and pipeline capabilities:
178
178
  | `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
179
179
  | `aria2` | bundled aria2c downloader for `build-fullmap --aria2c` (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
180
180
  | `qc` | four-stage QC audit (exact → fuzzy → abbreviation → SapBERT embeddings) | `pip install "tablassert[qc]"` |
181
- | `agent` | autonomous agent (smolagents, litellm, PDF context) | `pip install "tablassert[agent]"` |
181
+ | `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
182
182
  | `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
183
+ | `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
183
184
  | `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
184
185
 
185
186
  QC is opt-in at build time (`build-kg --qc`). Reaching a feature whose extra is not installed never
@@ -215,9 +216,12 @@ no LazyFrame setup or NLP preprocessing required. See the
215
216
  - **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
216
217
  - **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
217
218
  - **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
219
+ - **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
218
220
  - **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
219
221
  - **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
220
222
  - **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
223
+ - **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
224
+ - **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
221
225
 
222
226
  ## Developing
223
227
 
@@ -33,7 +33,7 @@ template:
33
33
  source:
34
34
  kind: text
35
35
  local: ./gene-disease.csv
36
- url: https://example.com/data.csv
36
+ url: [https://example.com/data.csv]
37
37
  row_slice: [1, auto]
38
38
  delimiter: ","
39
39
  statement:
@@ -119,8 +119,9 @@ CSV/TSV/Excel sources; optional extras add runtime and pipeline capabilities:
119
119
  | `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
120
120
  | `aria2` | bundled aria2c downloader for `build-fullmap --aria2c` (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
121
121
  | `qc` | four-stage QC audit (exact → fuzzy → abbreviation → SapBERT embeddings) | `pip install "tablassert[qc]"` |
122
- | `agent` | autonomous agent (smolagents, litellm, PDF context) | `pip install "tablassert[agent]"` |
122
+ | `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
123
123
  | `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
124
+ | `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
124
125
  | `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
125
126
 
126
127
  QC is opt-in at build time (`build-kg --qc`). Reaching a feature whose extra is not installed never
@@ -156,9 +157,12 @@ no LazyFrame setup or NLP preprocessing required. See the
156
157
  - **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
157
158
  - **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
158
159
  - **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
160
+ - **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
159
161
  - **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
160
162
  - **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
161
163
  - **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
164
+ - **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
165
+ - **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
162
166
 
163
167
  ## Developing
164
168
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "17.0.1"
3
+ version = "18.1.0"
4
4
  description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -72,6 +72,7 @@ module-name = "tablassert.rs"
72
72
  bindings = "pyo3"
73
73
  manifest-path = "rust/Cargo.toml"
74
74
  features = ["extension-module"]
75
+ include = ["src/tablassert/data/experimental_taxa.yaml"]
75
76
 
76
77
  [project.scripts]
77
78
  tablassert = "tablassert.cli:APP"
@@ -294,7 +294,6 @@ fn token_qc(value: &str) -> bool {
294
294
  && !value.contains('\t')
295
295
  && !value.contains('\n')
296
296
  && !value.contains('\r')
297
- && !value.contains("inchikey")
298
297
  && !value.contains("uncharacterized")
299
298
  && !value.contains("hypothetical")
300
299
  }
@@ -459,6 +458,15 @@ fn first_category(row: &SynonymRow<'_>) -> String {
459
458
  .to_string()
460
459
  }
461
460
 
461
+ fn is_organism_taxon(row: &SynonymRow<'_>) -> bool {
462
+ row.types.iter().any(|category| {
463
+ category
464
+ .trim()
465
+ .trim_start_matches("biolink:")
466
+ .eq_ignore_ascii_case("OrganismTaxon")
467
+ })
468
+ }
469
+
462
470
  fn first_taxon(row: &SynonymRow<'_>) -> i32 {
463
471
  row.taxa
464
472
  .first()
@@ -469,6 +477,56 @@ fn first_taxon(row: &SynonymRow<'_>) -> i32 {
469
477
  .unwrap_or(0)
470
478
  }
471
479
 
480
+ fn parsed_taxa(row: &SynonymRow<'_>) -> Vec<i32> {
481
+ row.taxa
482
+ .iter()
483
+ .filter_map(|taxon| {
484
+ let taxon = taxon.trim();
485
+ let local = taxon
486
+ .split_once(':')
487
+ .filter(|(prefix, _)| prefix.eq_ignore_ascii_case("NCBITaxon"))
488
+ .map(|(_, local)| local)
489
+ .unwrap_or(taxon);
490
+ local.parse::<i32>().ok().filter(|id| *id > 0)
491
+ })
492
+ .collect()
493
+ }
494
+
495
+ fn allowlist_identity(taxa: &HashSet<i32>) -> String {
496
+ let mut ids: Vec<i32> = taxa.iter().copied().collect();
497
+ ids.sort_unstable();
498
+ let encoded = ids
499
+ .iter()
500
+ .map(i32::to_string)
501
+ .collect::<Vec<String>>()
502
+ .join(",");
503
+ format!(
504
+ "count={};xxh64={:016x}",
505
+ ids.len(),
506
+ xxh64(encoded.as_bytes(), 0)
507
+ )
508
+ }
509
+
510
+ fn retain_for_taxon_allowlist(row: &SynonymRow<'_>, allowlist: Option<&HashSet<i32>>) -> bool {
511
+ let Some(allowlist) = allowlist else {
512
+ return true;
513
+ };
514
+ if is_organism_taxon(row) {
515
+ return true;
516
+ }
517
+ let mut has_valid_taxon = false;
518
+ for taxon in parsed_taxa(row) {
519
+ has_valid_taxon = true;
520
+ if allowlist.contains(&taxon) {
521
+ return true;
522
+ }
523
+ }
524
+ // Taxonless rows remain available for categories whose BABEL synonym rows
525
+ // do not carry a positive taxon ID. This includes the conventional
526
+ // `NCBITaxon:0` sentinel and malformed/empty taxon values.
527
+ !has_valid_taxon
528
+ }
529
+
472
530
  fn split_curie(curie: &str) -> Option<(&str, &str)> {
473
531
  curie
474
532
  .split_once(':')
@@ -1358,6 +1416,7 @@ struct SynonymShared<'a> {
1358
1416
  curie_counter: &'a AtomicU32,
1359
1417
  equivalents: &'a EquivIndex,
1360
1418
  exclude_prefixes: &'a HashSet<String>,
1419
+ taxon_allowlist: Option<&'a HashSet<i32>>,
1361
1420
  spill_dir: &'a Path,
1362
1421
  local_spill: usize,
1363
1422
  curie_spill: usize,
@@ -1393,6 +1452,9 @@ fn process_row(
1393
1452
  let Some((prefix, local_id)) = split_curie(curie) else {
1394
1453
  return Ok(());
1395
1454
  };
1455
+ if !retain_for_taxon_allowlist(row, sh.taxon_allowlist) {
1456
+ return Ok(());
1457
+ }
1396
1458
  // Skip CURIEs whose prefix the caller excludes (opt-in via
1397
1459
  // TABLASSERT_FULLMAP_EXCLUDE_PREFIXES) — filtered out downstream anyway.
1398
1460
  if sh.exclude_prefixes.contains(prefix) {
@@ -1623,6 +1685,7 @@ fn process_synonyms(
1623
1685
  local_spill: usize,
1624
1686
  curie_spill: usize,
1625
1687
  exclude_prefixes: &HashSet<String>,
1688
+ taxon_allowlist: Option<&HashSet<i32>>,
1626
1689
  worker_count: usize,
1627
1690
  shard_count: usize,
1628
1691
  chunk_bytes: usize,
@@ -1671,6 +1734,7 @@ fn process_synonyms(
1671
1734
  curie_counter: &curie_counter,
1672
1735
  equivalents,
1673
1736
  exclude_prefixes,
1737
+ taxon_allowlist,
1674
1738
  spill_dir,
1675
1739
  local_spill,
1676
1740
  curie_spill,
@@ -1799,6 +1863,7 @@ fn write_final_database(
1799
1863
  insert_batch: usize,
1800
1864
  shard_count: usize,
1801
1865
  progress: Option<&Arc<Progress>>,
1866
+ taxon_allowlist_identity: Option<&str>,
1802
1867
  ) -> PyResult<()> {
1803
1868
  let build_id = new_build_id();
1804
1869
  let build_id_str = build_id.to_string();
@@ -1855,6 +1920,9 @@ fn write_final_database(
1855
1920
  let shard_count_str = shard_count.to_string();
1856
1921
  meta.insert("shards", shard_count_str.as_str())
1857
1922
  .map_err(py_err)?;
1923
+ if let Some(identity) = taxon_allowlist_identity {
1924
+ meta.insert("taxon_allowlist", identity).map_err(py_err)?;
1925
+ }
1858
1926
  }
1859
1927
  write.commit().map_err(py_err)?;
1860
1928
 
@@ -2229,6 +2297,8 @@ fn build_fullmap_inner(
2229
2297
  local_spill: usize,
2230
2298
  curie_spill: usize,
2231
2299
  exclude_prefixes: HashSet<String>,
2300
+ taxon_allowlist: Option<HashSet<i32>>,
2301
+ taxon_allowlist_identity: Option<String>,
2232
2302
  chunk_bytes: usize,
2233
2303
  producers: usize,
2234
2304
  cache_bytes: usize,
@@ -2258,6 +2328,7 @@ fn build_fullmap_inner(
2258
2328
  local_spill,
2259
2329
  curie_spill,
2260
2330
  &exclude_prefixes,
2331
+ taxon_allowlist.as_ref(),
2261
2332
  worker_count,
2262
2333
  shard_count,
2263
2334
  chunk_bytes,
@@ -2288,6 +2359,7 @@ fn build_fullmap_inner(
2288
2359
  insert_batch,
2289
2360
  shard_count,
2290
2361
  progress.as_ref(),
2362
+ taxon_allowlist_identity.as_deref(),
2291
2363
  )?;
2292
2364
 
2293
2365
  // Clean up spill runs on success (left in place on error for inspection).
@@ -2297,13 +2369,14 @@ fn build_fullmap_inner(
2297
2369
  }
2298
2370
 
2299
2371
  #[pyfunction]
2300
- #[pyo3(signature = (output, classes, synonyms, progress=None))]
2372
+ #[pyo3(signature = (output, classes, synonyms, progress=None, taxon_allowlist=None))]
2301
2373
  pub fn build_fullmap_db(
2302
2374
  py: Python<'_>,
2303
2375
  output: PathBuf,
2304
2376
  classes: Vec<PathBuf>,
2305
2377
  synonyms: Vec<PathBuf>,
2306
2378
  progress: Option<Py<PyAny>>,
2379
+ taxon_allowlist: Option<Vec<i32>>,
2307
2380
  ) -> PyResult<()> {
2308
2381
  if synonyms.is_empty() {
2309
2382
  return Err(PyValueError::new_err(
@@ -2407,6 +2480,9 @@ pub fn build_fullmap_db(
2407
2480
  PathBuf::from(p)
2408
2481
  });
2409
2482
 
2483
+ let taxon_allowlist: Option<HashSet<i32>> =
2484
+ taxon_allowlist.map(|ids| ids.into_iter().filter(|id| *id > 0).collect());
2485
+ let taxon_allowlist_identity: Option<String> = taxon_allowlist.as_ref().map(allowlist_identity);
2410
2486
  let progress = progress.map(|cb| Arc::new(Progress { cb }));
2411
2487
 
2412
2488
  // Release the GIL for the whole build so rich's Live display thread can
@@ -2422,6 +2498,8 @@ pub fn build_fullmap_db(
2422
2498
  local_spill,
2423
2499
  curie_spill,
2424
2500
  exclude_prefixes,
2501
+ taxon_allowlist,
2502
+ taxon_allowlist_identity,
2425
2503
  chunk_bytes,
2426
2504
  producers,
2427
2505
  cache_bytes,
@@ -3650,6 +3728,8 @@ mod tests {
3650
3728
  local_spill,
3651
3729
  1_000_000,
3652
3730
  HashSet::new(),
3731
+ None,
3732
+ None,
3653
3733
  DEFAULT_CHUNK_BYTES,
3654
3734
  2,
3655
3735
  64 * 1024 * 1024,
@@ -3678,10 +3758,12 @@ mod tests {
3678
3758
  }
3679
3759
 
3680
3760
  #[test]
3681
- fn token_qc_rejects_banned_tokens() {
3761
+ fn token_qc_rejects_banned_tokens_but_accepts_inchikey() {
3682
3762
  assert!(token_qc("brca1"));
3763
+ assert!(token_qc("inchikey=abc"));
3683
3764
  assert!(!token_qc("hypothetical protein"));
3684
3765
  assert!(!token_qc("line\nbreak"));
3766
+ assert!(!token_qc("tab\tvalue"));
3685
3767
  }
3686
3768
 
3687
3769
  #[test]
@@ -4028,6 +4110,7 @@ mod tests {
4028
4110
  30,
4029
4111
  1_000_000,
4030
4112
  &HashSet::new(),
4113
+ None,
4031
4114
  4,
4032
4115
  shard_count,
4033
4116
  DEFAULT_CHUNK_BYTES,
@@ -4085,6 +4168,8 @@ mod tests {
4085
4168
  100,
4086
4169
  1_000_000,
4087
4170
  HashSet::new(),
4171
+ None,
4172
+ None,
4088
4173
  DEFAULT_CHUNK_BYTES,
4089
4174
  2,
4090
4175
  64 * 1024 * 1024,
@@ -4102,6 +4187,8 @@ mod tests {
4102
4187
  100,
4103
4188
  1_000_000,
4104
4189
  HashSet::new(),
4190
+ None,
4191
+ None,
4105
4192
  DEFAULT_CHUNK_BYTES,
4106
4193
  2,
4107
4194
  64 * 1024 * 1024,
@@ -4519,6 +4606,8 @@ mod tests {
4519
4606
  4_000_000,
4520
4607
  1_000_000,
4521
4608
  HashSet::new(),
4609
+ None,
4610
+ None,
4522
4611
  DEFAULT_CHUNK_BYTES,
4523
4612
  2,
4524
4613
  64 * 1024 * 1024,
@@ -4588,7 +4677,7 @@ mod tests {
4588
4677
 
4589
4678
  std::env::set_var("TABLASSERT_FULLMAP_SHARDS", "2");
4590
4679
  let built = Python::attach(|py| {
4591
- build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None)
4680
+ build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None, None)
4592
4681
  });
4593
4682
  std::env::remove_var("TABLASSERT_FULLMAP_SHARDS");
4594
4683
  built.unwrap();
@@ -4649,6 +4738,8 @@ mod tests {
4649
4738
  4_000_000,
4650
4739
  1_000_000,
4651
4740
  HashSet::new(),
4741
+ None,
4742
+ None,
4652
4743
  DEFAULT_CHUNK_BYTES,
4653
4744
  2,
4654
4745
  64 * 1024 * 1024,
@@ -5356,6 +5447,8 @@ mod tests {
5356
5447
  4_000_000,
5357
5448
  2,
5358
5449
  HashSet::new(),
5450
+ None,
5451
+ None,
5359
5452
  DEFAULT_CHUNK_BYTES,
5360
5453
  2,
5361
5454
  64 * 1024 * 1024,
@@ -5446,6 +5539,8 @@ mod tests {
5446
5539
  4_000_000,
5447
5540
  1_000_000,
5448
5541
  exclude,
5542
+ None,
5543
+ None,
5449
5544
  DEFAULT_CHUNK_BYTES,
5450
5545
  2,
5451
5546
  64 * 1024 * 1024,
@@ -5505,6 +5600,8 @@ mod tests {
5505
5600
  4_000_000,
5506
5601
  1_000_000,
5507
5602
  HashSet::new(),
5603
+ None,
5604
+ None,
5508
5605
  8192,
5509
5606
  2,
5510
5607
  64 * 1024 * 1024,
@@ -5546,6 +5643,7 @@ mod tests {
5546
5643
  Vec::new(),
5547
5644
  Vec::new(),
5548
5645
  None,
5646
+ None,
5549
5647
  )
5550
5648
  .expect_err("empty synonyms should fail");
5551
5649
  assert!(err
@@ -1,3 +1,4 @@
1
+ use pyo3::exceptions::PyOSError;
1
2
  use pyo3::prelude::*;
2
3
 
3
4
  // mimalloc returns memory to the OS far better than glibc malloc under heavy
@@ -5,7 +6,10 @@ use pyo3::prelude::*;
5
6
  // making millions of small String/Vec allocations). Without it, per-thread
6
7
  // malloc arenas retain freed memory and inflate peak RSS several-fold.
7
8
  use mimalloc::MiMalloc;
8
- use xxhash_rust::xxh64::xxh64 as xxh64_digest;
9
+ use std::fs::File;
10
+ use std::io::Read;
11
+ use std::path::PathBuf;
12
+ use xxhash_rust::xxh64::{xxh64 as xxh64_digest, Xxh64};
9
13
 
10
14
  #[global_allocator]
11
15
  static GLOBAL: MiMalloc = MiMalloc;
@@ -26,6 +30,27 @@ fn xxh64(data: &str) -> String {
26
30
  format!("{:016x}", xxh64_digest(data.as_bytes(), 0))
27
31
  }
28
32
 
33
+ /// XXH64 hex digest (seed 0) of a file's raw bytes, streamed in fixed-size chunks.
34
+ #[pyfunction]
35
+ fn xxh64_file(path: PathBuf) -> PyResult<String> {
36
+ const CHUNK_SIZE: usize = 8 * 1024 * 1024;
37
+
38
+ let mut file = File::open(&path)
39
+ .map_err(|error| PyOSError::new_err(format!("{}: {}", path.display(), error)))?;
40
+ let mut hasher = Xxh64::new(0);
41
+ let mut chunk = vec![0; CHUNK_SIZE];
42
+ loop {
43
+ let bytes_read = file
44
+ .read(&mut chunk)
45
+ .map_err(|error| PyOSError::new_err(format!("{}: {}", path.display(), error)))?;
46
+ if bytes_read == 0 {
47
+ break;
48
+ }
49
+ hasher.update(&chunk[..bytes_read]);
50
+ }
51
+ Ok(format!("{:016x}", hasher.digest()))
52
+ }
53
+
29
54
  // Public re-exports of the fullmap `#[pyfunction]`s so Rust integration tests
30
55
  // (`rust/tests/`) and other non-Python embedders can drive the exact production
31
56
  // build/read path. The `fullmap` module itself stays private; only these
@@ -48,12 +73,13 @@ fn rs(module: &Bound<'_, PyModule>) -> PyResult<()> {
48
73
  module.add_function(wrap_pyfunction!(ndjson::dedup_ndjson, module)?)?;
49
74
  module.add_function(wrap_pyfunction!(uuid::namespace_uuid, module)?)?;
50
75
  module.add_function(wrap_pyfunction!(xxh64, module)?)?;
76
+ module.add_function(wrap_pyfunction!(xxh64_file, module)?)?;
51
77
  Ok(())
52
78
  }
53
79
 
54
80
  #[cfg(test)]
55
81
  mod tests {
56
- use super::xxh64;
82
+ use super::{xxh64, xxh64_file};
57
83
 
58
84
  #[test]
59
85
  fn xxh64_matches_known_digests() {
@@ -63,4 +89,33 @@ mod tests {
63
89
  assert_eq!(xxh64("hello"), "26c7827d889f6da3");
64
90
  assert_eq!(xxh64(""), "ef46db3751d8e999");
65
91
  }
92
+
93
+ #[test]
94
+ fn xxh64_file_matches_known_digests() {
95
+ // WHY: pin file hashing to the same seed-0 digest as the string primitive,
96
+ // including the empty-file identity value and UTF-8 bytes.
97
+ let dir = tempfile::tempdir().unwrap();
98
+ let path = dir.path().join("hello");
99
+ std::fs::write(&path, b"hello").unwrap();
100
+ assert_eq!(xxh64_file(path).unwrap(), "26c7827d889f6da3");
101
+
102
+ let empty = dir.path().join("empty");
103
+ std::fs::write(&empty, b"").unwrap();
104
+ assert_eq!(xxh64_file(empty).unwrap(), "ef46db3751d8e999");
105
+ }
106
+
107
+ #[test]
108
+ fn xxh64_file_reports_io_failures_as_os_error() {
109
+ // WHY: callers must distinguish I/O failures from valid digests and see
110
+ // the offending path, including for missing files and directories.
111
+ let dir = tempfile::tempdir().unwrap();
112
+ let missing = dir.path().join("missing");
113
+ let error = xxh64_file(missing.clone()).unwrap_err();
114
+ assert!(error.to_string().contains(&missing.display().to_string()));
115
+
116
+ let error = xxh64_file(dir.path().to_path_buf()).unwrap_err();
117
+ assert!(error
118
+ .to_string()
119
+ .contains(&dir.path().display().to_string()));
120
+ }
66
121
  }