tablassert 17.0.1__tar.gz → 18.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {tablassert-17.0.1 → tablassert-18.0.0}/PKG-INFO +7 -3
  2. {tablassert-17.0.1 → tablassert-18.0.0}/README.md +6 -2
  3. {tablassert-17.0.1 → tablassert-18.0.0}/pyproject.toml +1 -1
  4. {tablassert-17.0.1 → tablassert-18.0.0}/rust/src/lib.rs +57 -2
  5. {tablassert-17.0.1 → tablassert-18.0.0}/rust/src/ndjson.rs +159 -2
  6. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/biolink.py +21 -17
  7. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/cli.py +46 -19
  8. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/errors.py +22 -0
  9. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/lib.py +5 -15
  10. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/models.py +3 -4
  11. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/rs.pyi +1 -0
  12. tablassert-18.0.0/src/tablassert/utils.py +53 -0
  13. tablassert-17.0.1/src/tablassert/utils.py +0 -18
  14. {tablassert-17.0.1 → tablassert-18.0.0}/LICENSE +0 -0
  15. {tablassert-17.0.1 → tablassert-18.0.0}/rust/Cargo.lock +0 -0
  16. {tablassert-17.0.1 → tablassert-18.0.0}/rust/Cargo.toml +0 -0
  17. {tablassert-17.0.1 → tablassert-18.0.0}/rust/examples/count_tables.rs +0 -0
  18. {tablassert-17.0.1 → tablassert-18.0.0}/rust/src/fullmap.rs +0 -0
  19. {tablassert-17.0.1 → tablassert-18.0.0}/rust/src/json.rs +0 -0
  20. {tablassert-17.0.1 → tablassert-18.0.0}/rust/src/uuid.rs +0 -0
  21. {tablassert-17.0.1 → tablassert-18.0.0}/rust/tests/build_golden.rs +0 -0
  22. {tablassert-17.0.1 → tablassert-18.0.0}/rust/tests/common/mod.rs +0 -0
  23. {tablassert-17.0.1 → tablassert-18.0.0}/rust/tests/extract_prebuilt.rs +0 -0
  24. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/__init__.py +0 -0
  25. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/_lazy.py +0 -0
  26. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/agent.py +0 -0
  27. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/coerce.py +0 -0
  28. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/distill.py +0 -0
  29. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/enums.py +0 -0
  30. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/extras.py +0 -0
  31. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/fullmap.py +0 -0
  32. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/graph_target.py +0 -0
  33. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/ingests.py +0 -0
  34. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/log.py +0 -0
  35. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/nlp.py +0 -0
  36. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/progress.py +0 -0
  37. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/qc.py +0 -0
  38. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/rig.py +0 -0
  39. {tablassert-17.0.1 → tablassert-18.0.0}/src/tablassert/study.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 17.0.1
3
+ Version: 18.0.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -92,7 +92,7 @@ template:
92
92
  source:
93
93
  kind: text
94
94
  local: ./gene-disease.csv
95
- url: https://example.com/data.csv
95
+ url: [https://example.com/data.csv]
96
96
  row_slice: [1, auto]
97
97
  delimiter: ","
98
98
  statement:
@@ -178,8 +178,9 @@ CSV/TSV/Excel sources; optional extras add runtime and pipeline capabilities:
178
178
  | `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
179
179
  | `aria2` | bundled aria2c downloader for `build-fullmap --aria2c` (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
180
180
  | `qc` | four-stage QC audit (exact → fuzzy → abbreviation → SapBERT embeddings) | `pip install "tablassert[qc]"` |
181
- | `agent` | autonomous agent (smolagents, litellm, PDF context) | `pip install "tablassert[agent]"` |
181
+ | `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
182
182
  | `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
183
+ | `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
183
184
  | `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
184
185
 
185
186
  QC is opt-in at build time (`build-kg --qc`). Reaching a feature whose extra is not installed never
@@ -215,9 +216,12 @@ no LazyFrame setup or NLP preprocessing required. See the
215
216
  - **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
216
217
  - **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
217
218
  - **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
219
+ - **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
218
220
  - **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
219
221
  - **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
220
222
  - **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
223
+ - **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
224
+ - **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
221
225
 
222
226
  ## Developing
223
227
 
@@ -33,7 +33,7 @@ template:
33
33
  source:
34
34
  kind: text
35
35
  local: ./gene-disease.csv
36
- url: https://example.com/data.csv
36
+ url: [https://example.com/data.csv]
37
37
  row_slice: [1, auto]
38
38
  delimiter: ","
39
39
  statement:
@@ -119,8 +119,9 @@ CSV/TSV/Excel sources; optional extras add runtime and pipeline capabilities:
119
119
  | `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
120
120
  | `aria2` | bundled aria2c downloader for `build-fullmap --aria2c` (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
121
121
  | `qc` | four-stage QC audit (exact → fuzzy → abbreviation → SapBERT embeddings) | `pip install "tablassert[qc]"` |
122
- | `agent` | autonomous agent (smolagents, litellm, PDF context) | `pip install "tablassert[agent]"` |
122
+ | `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
123
123
  | `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
124
+ | `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
124
125
  | `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
125
126
 
126
127
  QC is opt-in at build time (`build-kg --qc`). Reaching a feature whose extra is not installed never
@@ -156,9 +157,12 @@ no LazyFrame setup or NLP preprocessing required. See the
156
157
  - **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
157
158
  - **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
158
159
  - **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
160
+ - **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
159
161
  - **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
160
162
  - **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
161
163
  - **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
164
+ - **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
165
+ - **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
162
166
 
163
167
  ## Developing
164
168
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "17.0.1"
3
+ version = "18.0.0"
4
4
  description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -1,3 +1,4 @@
1
+ use pyo3::exceptions::PyOSError;
1
2
  use pyo3::prelude::*;
2
3
 
3
4
  // mimalloc returns memory to the OS far better than glibc malloc under heavy
@@ -5,7 +6,10 @@ use pyo3::prelude::*;
5
6
  // making millions of small String/Vec allocations). Without it, per-thread
6
7
  // malloc arenas retain freed memory and inflate peak RSS several-fold.
7
8
  use mimalloc::MiMalloc;
8
- use xxhash_rust::xxh64::xxh64 as xxh64_digest;
9
+ use std::fs::File;
10
+ use std::io::Read;
11
+ use std::path::PathBuf;
12
+ use xxhash_rust::xxh64::{xxh64 as xxh64_digest, Xxh64};
9
13
 
10
14
  #[global_allocator]
11
15
  static GLOBAL: MiMalloc = MiMalloc;
@@ -26,6 +30,27 @@ fn xxh64(data: &str) -> String {
26
30
  format!("{:016x}", xxh64_digest(data.as_bytes(), 0))
27
31
  }
28
32
 
33
+ /// XXH64 hex digest (seed 0) of a file's raw bytes, streamed in fixed-size chunks.
34
+ #[pyfunction]
35
+ fn xxh64_file(path: PathBuf) -> PyResult<String> {
36
+ const CHUNK_SIZE: usize = 8 * 1024 * 1024;
37
+
38
+ let mut file = File::open(&path)
39
+ .map_err(|error| PyOSError::new_err(format!("{}: {}", path.display(), error)))?;
40
+ let mut hasher = Xxh64::new(0);
41
+ let mut chunk = vec![0; CHUNK_SIZE];
42
+ loop {
43
+ let bytes_read = file
44
+ .read(&mut chunk)
45
+ .map_err(|error| PyOSError::new_err(format!("{}: {}", path.display(), error)))?;
46
+ if bytes_read == 0 {
47
+ break;
48
+ }
49
+ hasher.update(&chunk[..bytes_read]);
50
+ }
51
+ Ok(format!("{:016x}", hasher.digest()))
52
+ }
53
+
29
54
  // Public re-exports of the fullmap `#[pyfunction]`s so Rust integration tests
30
55
  // (`rust/tests/`) and other non-Python embedders can drive the exact production
31
56
  // build/read path. The `fullmap` module itself stays private; only these
@@ -48,12 +73,13 @@ fn rs(module: &Bound<'_, PyModule>) -> PyResult<()> {
48
73
  module.add_function(wrap_pyfunction!(ndjson::dedup_ndjson, module)?)?;
49
74
  module.add_function(wrap_pyfunction!(uuid::namespace_uuid, module)?)?;
50
75
  module.add_function(wrap_pyfunction!(xxh64, module)?)?;
76
+ module.add_function(wrap_pyfunction!(xxh64_file, module)?)?;
51
77
  Ok(())
52
78
  }
53
79
 
54
80
  #[cfg(test)]
55
81
  mod tests {
56
- use super::xxh64;
82
+ use super::{xxh64, xxh64_file};
57
83
 
58
84
  #[test]
59
85
  fn xxh64_matches_known_digests() {
@@ -63,4 +89,33 @@ mod tests {
63
89
  assert_eq!(xxh64("hello"), "26c7827d889f6da3");
64
90
  assert_eq!(xxh64(""), "ef46db3751d8e999");
65
91
  }
92
+
93
+ #[test]
94
+ fn xxh64_file_matches_known_digests() {
95
+ // WHY: pin file hashing to the same seed-0 digest as the string primitive,
96
+ // including the empty-file identity value and UTF-8 bytes.
97
+ let dir = tempfile::tempdir().unwrap();
98
+ let path = dir.path().join("hello");
99
+ std::fs::write(&path, b"hello").unwrap();
100
+ assert_eq!(xxh64_file(path).unwrap(), "26c7827d889f6da3");
101
+
102
+ let empty = dir.path().join("empty");
103
+ std::fs::write(&empty, b"").unwrap();
104
+ assert_eq!(xxh64_file(empty).unwrap(), "ef46db3751d8e999");
105
+ }
106
+
107
+ #[test]
108
+ fn xxh64_file_reports_io_failures_as_os_error() {
109
+ // WHY: callers must distinguish I/O failures from valid digests and see
110
+ // the offending path, including for missing files and directories.
111
+ let dir = tempfile::tempdir().unwrap();
112
+ let missing = dir.path().join("missing");
113
+ let error = xxh64_file(missing.clone()).unwrap_err();
114
+ assert!(error.to_string().contains(&missing.display().to_string()));
115
+
116
+ let error = xxh64_file(dir.path().to_path_buf()).unwrap_err();
117
+ assert!(error
118
+ .to_string()
119
+ .contains(&dir.path().display().to_string()));
120
+ }
66
121
  }
@@ -208,6 +208,34 @@ struct MergeIndex {
208
208
  scalar_conflicts: u64,
209
209
  }
210
210
 
211
+ type OriginalValues = FxHashMap<String, FxHashSet<String>>;
212
+
213
+ fn collect_original_values(value: &Value) -> OriginalValues {
214
+ let mut values: OriginalValues = FxHashMap::default();
215
+ let Some(map) = value.as_object() else {
216
+ return values;
217
+ };
218
+ for (key, value) in map {
219
+ if key.starts_with("original_") {
220
+ if let Value::String(text) = value {
221
+ for part in text.split('|').filter(|part| !part.is_empty()) {
222
+ values
223
+ .entry(key.clone())
224
+ .or_default()
225
+ .insert(part.to_string());
226
+ }
227
+ }
228
+ }
229
+ }
230
+ values
231
+ }
232
+
233
+ fn format_original_values(values: &FxHashSet<String>) -> String {
234
+ let mut sorted: Vec<&str> = values.iter().map(String::as_str).collect();
235
+ sorted.sort_unstable_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
236
+ sorted.join("|")
237
+ }
238
+
211
239
  /// The canonical bytes of one list item, shared by every structure that needs them.
212
240
  ///
213
241
  /// ONE heap allocation per distinct item instead of one per consumer: `Rc<T>`'s `Hash`,
@@ -281,6 +309,8 @@ struct MergedRecord {
281
309
  hashes: FxHashSet<u64>,
282
310
  /// Union state for every field whose current value is an array.
283
311
  lists: FxHashMap<String, ListState>,
312
+ /// Distinct non-empty scalar source values for each `original_*` field.
313
+ original_values: OriginalValues,
284
314
  }
285
315
 
286
316
  impl MergedRecord {
@@ -298,6 +328,7 @@ impl MergedRecord {
298
328
  let mut hashes: FxHashSet<u64> = FxHashSet::default();
299
329
  hashes.insert(content);
300
330
  Ok(Self {
331
+ original_values: collect_original_values(&value),
301
332
  value,
302
333
  hashes,
303
334
  lists,
@@ -312,6 +343,11 @@ impl MergedRecord {
312
343
  let Some(map) = self.value.as_object_mut() else {
313
344
  return Err(runtime_error("expected JSON object"));
314
345
  };
346
+ for (key, values) in &self.original_values {
347
+ if !values.is_empty() {
348
+ map.insert(key.clone(), Value::String(format_original_values(values)));
349
+ }
350
+ }
315
351
  for (key, state) in &mut self.lists {
316
352
  if !state.unioned {
317
353
  continue;
@@ -362,7 +398,9 @@ impl MergedRecord {
362
398
  /// checked against the field's `ListState` set in O(1); stored items are never
363
399
  /// re-canonicalized, and the sort by canonical bytes is deferred to write-out
364
400
  /// (`MergedRecord::finish`), where it runs only for fields that saw a real union;
365
- /// - scalar fields: first-wins on conflict, counted;
401
+ /// - scalar fields: first-wins on conflict, counted, except `original_*` fields;
402
+ /// - `original_*` fields collect distinct non-empty strings and render sorted values as
403
+ /// `A`, `A|B`, or `A|B|C`;
366
404
  /// - fields only on `incoming`: copied over (only a conflict when both sides disagree);
367
405
  /// - `id` is never touched: both sides carry the same one by construction.
368
406
  ///
@@ -399,6 +437,15 @@ fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<u64> {
399
437
  stored
400
438
  .lists
401
439
  .insert(key.clone(), ListState::from_items(items)?);
440
+ } else if key.starts_with("original_") {
441
+ if let Value::String(text) = incoming_value {
442
+ let values = stored.original_values.entry(key.clone()).or_default();
443
+ values.extend(
444
+ text.split('|')
445
+ .filter(|part| !part.is_empty())
446
+ .map(str::to_owned),
447
+ );
448
+ }
402
449
  }
403
450
  stored_map.insert(key.clone(), incoming_value.clone());
404
451
  }
@@ -422,6 +469,26 @@ fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<u64> {
422
469
  stored_items.push(item.clone());
423
470
  }
424
471
  }
472
+ } else if key.starts_with("original_") {
473
+ if let (Value::String(stored_text), Value::String(incoming_text)) =
474
+ (&*stored_value, incoming_value)
475
+ {
476
+ let values = stored.original_values.entry(key.clone()).or_default();
477
+ values.extend(
478
+ stored_text
479
+ .split('|')
480
+ .filter(|part| !part.is_empty())
481
+ .map(str::to_owned),
482
+ );
483
+ values.extend(
484
+ incoming_text
485
+ .split('|')
486
+ .filter(|part| !part.is_empty())
487
+ .map(str::to_owned),
488
+ );
489
+ } else if stored_value != incoming_value {
490
+ conflicts += 1;
491
+ }
425
492
  } else if stored_value != incoming_value {
426
493
  conflicts += 1;
427
494
  }
@@ -529,6 +596,10 @@ fn merge_records_reference(stored: &mut Value, incoming: &Value) -> PyResult<u64
529
596
  return Err(runtime_error("expected JSON object"));
530
597
  };
531
598
  let mut conflicts: u64 = 0;
599
+ let mut original_values: OriginalValues = collect_original_values(stored);
600
+ for (key, values) in collect_original_values(incoming) {
601
+ original_values.entry(key).or_default().extend(values);
602
+ }
532
603
  // Read both counts BEFORE the fold: the fold may copy incoming's over a stored side
533
604
  // that lacks it, and the recompute rule below needs to know each side contributed one.
534
605
  let stored_cases: Option<Value> = stored.get("number_of_cases").cloned();
@@ -565,12 +636,21 @@ fn merge_records_reference(stored: &mut Value, incoming: &Value) -> PyResult<u64
565
636
  }
566
637
  keyed.sort_by(|left, right| left.0.cmp(&right.0));
567
638
  stored_items.extend(keyed.into_iter().map(|(_, item)| item));
639
+ } else if key.starts_with("original_") {
640
+ // Aggregated and rendered after this fold, once all values are known.
568
641
  } else if stored_value != incoming_value {
569
642
  conflicts += 1;
570
643
  }
571
644
  }
572
645
  }
573
646
  }
647
+ if let Some(map) = stored.as_object_mut() {
648
+ for (key, values) in &original_values {
649
+ if !values.is_empty() {
650
+ map.insert(key.clone(), Value::String(format_original_values(values)));
651
+ }
652
+ }
653
+ }
574
654
  // WHY: exact-unique `number_of_cases` semantics (see the docstring). Guarded on the
575
655
  // merged record actually carrying the ID list: a one-sided carrier is fine (the union
576
656
  // is just that side's list), while a carrier-less merge never recomputes.
@@ -759,8 +839,9 @@ pub fn dedup_ndjson(
759
839
 
760
840
  #[cfg(test)]
761
841
  mod tests {
762
- use super::{dedup_ndjson, differing_keys, record_if_new};
842
+ use super::{dedup_ndjson, differing_keys, format_original_values, record_if_new};
763
843
  use rustc_hash::FxHashMap;
844
+ use rustc_hash::FxHashSet;
764
845
  use serde_json::Value;
765
846
  use std::fs;
766
847
  use tempfile::tempdir;
@@ -1465,6 +1546,82 @@ mod tests {
1465
1546
  assert_eq!(sources[1]["resource_id"], serde_json::json!("infores:b"));
1466
1547
  }
1467
1548
 
1549
+ #[test]
1550
+ fn format_original_values_uses_sorted_pipe_joining() {
1551
+ let empty: FxHashSet<String> = FxHashSet::default();
1552
+ assert_eq!(format_original_values(&empty), "");
1553
+
1554
+ let one = FxHashSet::from_iter([String::from("only")]);
1555
+ assert_eq!(format_original_values(&one), "only");
1556
+
1557
+ let two = FxHashSet::from_iter([String::from("z"), String::from("a")]);
1558
+ assert_eq!(format_original_values(&two), "a|z");
1559
+
1560
+ let three = FxHashSet::from_iter([String::from("z"), String::from("a"), String::from("m")]);
1561
+ assert_eq!(format_original_values(&three), "a|m|z");
1562
+ }
1563
+
1564
+ #[test]
1565
+ fn merge_mode_aggregates_original_scalars_across_all_records() {
1566
+ let dir = tempdir().expect("tempdir");
1567
+ let (input, output) = write_edges(
1568
+ dir.path(),
1569
+ concat!(
1570
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"z\",\"original_object\":\"\"}\n",
1571
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"a\",\"original_object\":\"x\"}\n",
1572
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"m\",\"original_object\":\"y\"}\n",
1573
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"a\",\"original_object\":\"x\"}\n"
1574
+ ),
1575
+ );
1576
+ let (merged, conflicts) = dedup_ndjson(
1577
+ input,
1578
+ output.clone(),
1579
+ true,
1580
+ None,
1581
+ spo_fields(),
1582
+ Some("merge".to_string()),
1583
+ )
1584
+ .expect("merge dedup");
1585
+
1586
+ assert_eq!((merged, conflicts), (2, 0));
1587
+ let edge = merged_edge(&output);
1588
+ assert_eq!(edge["original_subject"], serde_json::json!("a|m|z"));
1589
+ assert_eq!(edge["original_object"], serde_json::json!("x|y"));
1590
+ }
1591
+
1592
+ #[test]
1593
+ fn merge_mode_ignores_empty_originals_and_is_order_independent() {
1594
+ let write = |dir: &std::path::Path, rows: &str| {
1595
+ let (input, output) = write_edges(dir, rows);
1596
+ dedup_ndjson(
1597
+ input,
1598
+ output.clone(),
1599
+ true,
1600
+ None,
1601
+ spo_fields(),
1602
+ Some("merge".to_string()),
1603
+ )
1604
+ .expect("merge dedup");
1605
+ merged_edge(&output)["original_subject"].clone()
1606
+ };
1607
+ let first = write(
1608
+ tempdir().expect("tempdir").path(),
1609
+ concat!(
1610
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"\"}\n",
1611
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"only\"}\n"
1612
+ ),
1613
+ );
1614
+ let second = write(
1615
+ tempdir().expect("tempdir").path(),
1616
+ concat!(
1617
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"only\"}\n",
1618
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"\"}\n"
1619
+ ),
1620
+ );
1621
+ assert_eq!(first, serde_json::json!("only"));
1622
+ assert_eq!(first, second);
1623
+ }
1624
+
1468
1625
  #[test]
1469
1626
  fn merge_mode_keeps_fields_only_the_second_record_carries() {
1470
1627
  // WHY: first-wins arbitrates CONFLICTS; a field absent from the first record is
@@ -257,9 +257,9 @@ def _association_model_fields() -> set[str]:
257
257
 
258
258
  Unions ``model_fields`` across the entire association family rather than only
259
259
  walking ``Association.__mro__``. Walking the base MRO alone silently excludes
260
- subclass-only evidence slots -- ``clinical_approval_status``,
261
- ``number_of_cases``, ``FDA_regulatory_approvals`` -- which then get demoted
262
- into ``supporting_text`` by :func:`lib.fold_unknown_to_supporting_text`.
260
+ subclass-only evidence slots -- ``clinical_approval_status`` and
261
+ ``number_of_cases`` -- which then get demoted into ``supporting_text`` by
262
+ :func:`lib.fold_unknown_to_supporting_text`.
263
263
  """
264
264
  fields: set[str] = set()
265
265
  for klass in _association_classes():
@@ -529,8 +529,8 @@ release from making ``species_context_qualifier`` silently emittable again.
529
529
 
530
530
 
531
531
  CLASS_FIELD_OVERRIDES: dict[str, frozenset[str]] = {
532
- "EntityToDiseaseAssociation": frozenset({"disease_context_qualifier"}),
533
- "EntityToPhenotypicFeatureAssociation": frozenset({"disease_context_qualifier"}),
532
+ "EntityToDiseaseAssociation": frozenset({"disease_context_qualifier", "regulatory_approvals"}),
533
+ "EntityToPhenotypicFeatureAssociation": frozenset({"disease_context_qualifier", "regulatory_approvals"}),
534
534
  }
535
535
  """Per-class grants of edge fields the resolved association class does not declare.
536
536
 
@@ -538,15 +538,14 @@ Keys are bare association class names (``association_class(cat).__name__``), val
538
538
  the slots ``lib.prune_to_class`` keeps on rows resolved to that class even though the
539
539
  installed model attaches them elsewhere.
540
540
 
541
- The motivating case is a DAKP contraindication edge: ``FDA_regulatory_approvals`` is
542
- declared only on the ``EntityToDisease`` / ``EntityToPhenotypicFeature`` classes the
543
- edge is pinned to, while ``disease_context_qualifier`` is declared only on the
541
+ The motivating case is a DAKP contraindication edge: ``regulatory_approvals`` is a
542
+ canonical slot not yet attached by the installed model, while
543
+ ``disease_context_qualifier`` is declared only on the
544
544
  ``ChemicalEntityToDiseaseOrPhenotypicFeatureAssociation`` lineage -- so one edge can
545
- natively carry one slot or the other, never both. Tablassert deliberately emits the
546
- qualifier on the pinned classes ahead of the pinned model (pending an upstream Biolink
547
- widening), exactly as :data:`KNOWN_PENDING_EDGE_FIELDS` emits KGX carryovers ahead of
548
- it. ``_validation_record`` strips granted fields before record validation so the
549
- deliberate gap is not reported as ``extra_forbidden``.
545
+ natively carry both only through these explicit class-scoped grants. Tablassert
546
+ deliberately emits the granted fields on the pinned classes ahead of the pinned model
547
+ (pending an upstream Biolink widening). ``_validation_record`` strips granted fields
548
+ before record validation so the deliberate gap is not reported as ``extra_forbidden``.
550
549
 
551
550
  A tripwire test asserts every granted field is still absent from its class: the moment
552
551
  a biolink-model release attaches the slot, the suite fails and the stale grant is
@@ -582,7 +581,12 @@ unresolved.
582
581
 
583
582
 
584
583
  ALLOWED_EDGE_FIELDS: frozenset[str] = (
585
- (frozenset(_association_model_fields()) | {q.value for q in Qualifiers} | TABLASERT_EDGE_EXTRAS)
584
+ (
585
+ frozenset(_association_model_fields())
586
+ | {q.value for q in Qualifiers}
587
+ | TABLASERT_EDGE_EXTRAS
588
+ | frozenset().union(*CLASS_FIELD_OVERRIDES.values())
589
+ )
586
590
  - UNSATISFIABLE_EDGE_FIELDS
587
591
  - DISABLED_EDGE_FIELDS
588
592
  )
@@ -591,9 +595,9 @@ ALLOWED_EDGE_FIELDS: frozenset[str] = (
591
595
  Any column on an edge frame that is not in this set is folded into the
592
596
  ``supporting_text`` ``list[str]`` field by ``lib.fold_unknown_to_supporting_text()``
593
597
  as a ``"column: value"`` string. Composed of the fields declared by *any* Biolink
594
- association class, the derived qualifier slot names, and the curated
595
- ``TABLASERT_EDGE_EXTRAS`` -- less the slots that no Pydantic class can hold and the
596
- fields disabled by Tablassert policy.
598
+ association class, the derived qualifier slot names, the curated
599
+ ``TABLASERT_EDGE_EXTRAS``, and explicit :data:`CLASS_FIELD_OVERRIDES` grants -- less the
600
+ slots that no Pydantic class can hold and the fields disabled by Tablassert policy.
597
601
 
598
602
  Note this is a per-*family* allow-list: a field being permitted here does not mean the
599
603
  specific association class chosen for a given edge accepts it. Per-record pruning
@@ -75,6 +75,44 @@ def _section_store_path(h: str, head: bool = False, release: bool = False, qc: b
75
75
  return STORE / f"{h}{suffix}.parquet"
76
76
 
77
77
 
78
+ def _section_store_key_for_build(
79
+ section: dict[str, Any], configuration_file: Path, content_hashes: dict[tuple[Path, int, int], str]
80
+ ) -> tuple[str, str]:
81
+ """Return a section's content-aware store key and config-only fallback label.
82
+
83
+ The memo belongs to one ``build_graph_pipeline`` call and is invalidated by any
84
+ ``(resolved path, mtime_ns, size)`` change. Progress and validation errors use the
85
+ content-aware key; the returned config-only hash labels ``SourceFileError`` when
86
+ hashing fails before a content-aware key can exist.
87
+ """
88
+ from tablassert.utils import file_content_hash, mkhash, section_store_key
89
+
90
+ config_hash: str = mkhash(section)
91
+ source: object = section.get("source")
92
+ local: object = source.get("local") if isinstance(source, dict) else None
93
+ if local is None:
94
+ return section_store_key(section), config_hash
95
+
96
+ resolved: Path = Path(str(local)).resolve()
97
+ config_value: object = section.get("config", "section")
98
+ section_label: str = f"{Path(str(config_value)).stem} · {config_hash[:8]}"
99
+ digest: str | None = None
100
+ try:
101
+ stat = resolved.stat()
102
+ except OSError:
103
+ # Reuse the contextual helper so missing/unreadable paths never leak a raw OS error.
104
+ digest = file_content_hash(resolved, config=configuration_file, section_label=section_label)
105
+ return section_store_key(section, content_digest=digest), config_hash
106
+
107
+ signature: tuple[Path, int, int] = (resolved, stat.st_mtime_ns, stat.st_size)
108
+ if digest is None:
109
+ digest = content_hashes.get(signature)
110
+ if digest is None:
111
+ digest = file_content_hash(resolved, config=configuration_file, section_label=section_label)
112
+ content_hashes[signature] = digest
113
+ return section_store_key(section, content_digest=digest), config_hash
114
+
115
+
78
116
  def _load_table_indexed(args: tuple[int, Path]) -> tuple[int, object]:
79
117
  """Load one table, tagged with its input index (multiprocessing worker).
80
118
 
@@ -139,17 +177,11 @@ def _load_graph(configuration_file: Path) -> Graph:
139
177
 
140
178
 
141
179
  def build_pipeline(
142
- configuration_file: Path,
143
- progress: PipelineProgress,
144
- release: bool = False,
145
- qc: bool = False,
146
- log: bool = False,
147
- head: bool = False,
148
- no_original: bool = False,
180
+ configuration_file: Path, progress: PipelineProgress, release: bool = False, qc: bool = False, log: bool = False, head: bool = False
149
181
  ) -> None:
150
182
  """Load a graph YAML and build it through the shared in-process core."""
151
183
  graph: Graph = _load_graph(configuration_file)
152
- build_graph_pipeline(graph, configuration_file, progress, release=release, qc=qc, log=log, head=head, no_original=no_original)
184
+ build_graph_pipeline(graph, configuration_file, progress, release=release, qc=qc, log=log, head=head)
153
185
 
154
186
 
155
187
  def build_graph_pipeline(
@@ -160,7 +192,6 @@ def build_graph_pipeline(
160
192
  qc: bool = False,
161
193
  log: bool = False,
162
194
  head: bool = False,
163
- no_original: bool = False,
164
195
  audit_sources: bool = True,
165
196
  ) -> None:
166
197
  """Build a validated :class:`Graph` without loading another graph YAML.
@@ -178,13 +209,10 @@ def build_graph_pipeline(
178
209
  qc: When ``True``, run quality-control audits and final study assertions.
179
210
  log: When ``True``, enable per-section verbose logging.
180
211
  head: When ``True``, build a random sample of up to five rows per section.
181
- no_original: When ``True``, omit the verbatim ``original_*`` source-cell
182
- copies from the final edge NDJSON.
183
212
  """
184
213
  from tablassert.fullmap import fullmap_db_path
185
214
  from tablassert.lib import Tcode, compile_graph, compile_subgraph
186
215
  from tablassert.progress import flatten_pydantic_error, format_section_compact
187
- from tablassert.utils import mkhash
188
216
 
189
217
  # Stage 1/6: load tables.
190
218
  progress.stage("Loading Tables")
@@ -223,8 +251,11 @@ def build_graph_pipeline(
223
251
  progress.stage("Building TCode")
224
252
  start, advance, _ = progress.section_loop(n, "TCode")
225
253
  tcode: list[Tcode] = []
254
+ # This memo is intentionally scoped to one build: a changed stat signature re-hashes,
255
+ # while repeated sections pointing to the same unchanged file hash only once.
256
+ content_hashes: dict[tuple[Path, int, int], str] = {}
226
257
  for s in sections:
227
- h: str = mkhash(s)
258
+ h, _ = _section_store_key_for_build(s, configuration_file, content_hashes)
228
259
  start(f"{Path(str(s['config'])).stem} · {h[:8]}")
229
260
  # Mode flags change the cached parquet's content, so each combination caches
230
261
  # to a distinct file and can never quick-exit another mode's build.
@@ -284,7 +315,6 @@ def build_graph_pipeline(
284
315
  section_sources if audit_sources else None,
285
316
  on_phase=sub_step,
286
317
  on_subgraph=advance,
287
- no_original=no_original,
288
318
  uuid_fields=g.uuid_fields,
289
319
  uuid_domain=g.uuid_namespace,
290
320
  uuid_on_collision=g.uuid_on_collision,
@@ -691,16 +721,13 @@ def build_kg(
691
721
  qc: Annotated[bool, cyclopts.Parameter(name=["--qc", "-q"], negative="")] = False,
692
722
  log: Annotated[bool, cyclopts.Parameter(name=["--log", "-l"], negative="")] = False,
693
723
  head: Annotated[bool, cyclopts.Parameter(name=["--head", "-hd"], negative="")] = False,
694
- no_original: Annotated[bool, cyclopts.Parameter(name=["--no-original", "-no"], negative="")] = False,
695
724
  ) -> None:
696
725
  """Build a knowledge graph from a YAML configuration file.
697
726
 
698
727
  The positional config is a Graph YAML that orchestrates one or more table
699
728
  configs into a single knowledge-graph build.
700
729
 
701
- ``--no-original`` omits the verbatim source-cell copies (``original_subject``,
702
- ``original_object``, and any other ``original_*`` fields) from the final edge
703
- NDJSON. ``--qc`` requires the ``[qc]`` extra (``pip install
730
+ ``--qc`` requires the ``[qc]`` extra (``pip install
704
731
  "tablassert[qc]"``); it is checked before the build starts, because the audit stage
705
732
  runs LAST and a missing extra would otherwise surface only after entity resolution
706
733
  has finished. It also runs a final study stage that asserts over the emitted NDJSON
@@ -711,7 +738,7 @@ def build_kg(
711
738
  """
712
739
  if qc:
713
740
  extras.require("qc", required_by="--qc")
714
- run(7 if qc else 6, build_pipeline, graph_configuration_file, release=release, qc=qc, log=log, head=head, no_original=no_original)
741
+ run(7 if qc else 6, build_pipeline, graph_configuration_file, release=release, qc=qc, log=log, head=head)
715
742
 
716
743
 
717
744
  @APP.command(name="validate")
@@ -12,6 +12,7 @@ TablassertErrorCodes = Literal[
12
12
  "graph-validation-failed",
13
13
  "section-validation-failed",
14
14
  "babel-download-failed",
15
+ "source-file-unreadable",
15
16
  "resolve-bad-specs",
16
17
  "config-rows-and-row-slice-conflict",
17
18
  "comparison-bad-comparator-type",
@@ -185,3 +186,24 @@ class BabelDownloadError(TablassertError):
185
186
  f"BABEL download failed after {retries} attempts: {url} (last error: {last_error}). Check network connectivity or pin a different BABEL version.",
186
187
  code="babel-download-failed",
187
188
  )
189
+
190
+
191
+ class SourceFileError(TablassertError):
192
+ """A table section's ``source.local`` file could not be read for content-hashing.
193
+
194
+ Notes:
195
+ ``config`` and ``section_label`` are optional keyword context: the utils layer
196
+ that first notices the failure often lacks build context, but callers that have
197
+ it (the CLI's section loop) should pass them so the message names the exact
198
+ table config and section label (config stem + 8-char section digest).
199
+ """
200
+
201
+ def __init__(self, path: Path, detail: str, *, config: Path | None = None, section_label: str | None = None) -> None:
202
+ context: str = ""
203
+ if config is not None and section_label is not None:
204
+ context = f" (config {config}, section {section_label})"
205
+ elif config is not None:
206
+ context = f" (config {config})"
207
+ elif section_label is not None:
208
+ context = f" (section {section_label})"
209
+ super().__init__(f"Source file unreadable{context}: {path} — {detail}", code="source-file-unreadable")
@@ -1592,7 +1592,9 @@ def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fi
1592
1592
  uuid_fields: Optional edge fields that constitute edge identity
1593
1593
  (``Graph.uuid_fields``). ``None`` hashes the whole record.
1594
1594
  on_collision: ``error`` aborts on two different edges deriving one id;
1595
- ``merge`` folds them into one edge (``Graph.uuid_on_collision``).
1595
+ ``merge`` folds them into one edge (``Graph.uuid_on_collision``). During a merge,
1596
+ distinct non-empty scalar ``original_*`` fields are sorted and rendered as
1597
+ ``A``, ``A|B``, or ``A|B|C``.
1596
1598
 
1597
1599
  Notes:
1598
1600
  Also adds UUIDs to edges. Edges deduplicate on their derived id, so the
@@ -1714,11 +1716,7 @@ def fold_unknown_to_supporting_text(lf: pl.LazyFrame) -> pl.LazyFrame:
1714
1716
 
1715
1717
 
1716
1718
  def _collect_subframes(
1717
- subgraphs: list[Path],
1718
- on_phase: Callable[[str], None] | None = None,
1719
- on_subgraph: Callable[[], None] | None = None,
1720
- infores_id: str | None = None,
1721
- no_original: bool = False,
1719
+ subgraphs: list[Path], on_phase: Callable[[str], None] | None = None, on_subgraph: Callable[[], None] | None = None, infores_id: str | None = None
1722
1720
  ) -> tuple[list[pl.LazyFrame], list[pl.LazyFrame]]:
1723
1721
  """Scan and normalize subgraph parquets into node/edge subframes.
1724
1722
 
@@ -1733,8 +1731,6 @@ def _collect_subframes(
1733
1731
  on_subgraph: Optional callback fired once after each subgraph is
1734
1732
  processed, used to tick the progress bar.
1735
1733
  infores_id: Graph-level infores CURIE recorded as node ``provided_by``.
1736
- no_original: When True, also drop the verbatim ``original_*``
1737
- source-cell copies from the final edge frames.
1738
1734
 
1739
1735
  Returns:
1740
1736
  Tuple of ``(subnodes, subedges)``: per-section node and edge
@@ -1759,9 +1755,6 @@ def _collect_subframes(
1759
1755
  subnodes.append(partial)
1760
1756
  # Drop internal pre-resolution snapshot columns from final edges.
1761
1757
  lf = lf.drop([c for c in lf.collect_schema().names() if c.endswith("_pre_resolution")])
1762
- # --no-original: drop the verbatim source-cell copies from final edges too.
1763
- if no_original:
1764
- lf = lf.drop([c for c in lf.collect_schema().names() if c.startswith("original_")])
1765
1758
  lf = fold_unknown_to_supporting_text(lf)
1766
1759
  subedges.append(lf)
1767
1760
  if on_subgraph is not None:
@@ -1853,7 +1846,6 @@ def compile_graph(
1853
1846
  section_sources: list[dict[str, object]] | None = None,
1854
1847
  on_phase: Callable[[str], None] | None = None,
1855
1848
  on_subgraph: Callable[[], None] | None = None,
1856
- no_original: bool = False,
1857
1849
  uuid_fields: list[str] | None = None,
1858
1850
  uuid_domain: str | None = None,
1859
1851
  uuid_on_collision: str = "error",
@@ -1875,8 +1867,6 @@ def compile_graph(
1875
1867
  ``write-edges`` / ``dedup`` / ``rig``), used to drive progress UX.
1876
1868
  on_subgraph: Optional callback fired once per processed subgraph,
1877
1869
  used to tick the progress bar.
1878
- no_original: When ``True``, omit the verbatim ``original_*``
1879
- source-cell copies from the final edge NDJSON.
1880
1870
  uuid_fields: Optional edge fields that constitute edge identity
1881
1871
  (``Graph.uuid_fields``). ``None`` hashes the whole edge record, so any
1882
1872
  change to any field re-mints the id.
@@ -1916,7 +1906,7 @@ def compile_graph(
1916
1906
 
1917
1907
  subnodes: list[pl.LazyFrame]
1918
1908
  subedges: list[pl.LazyFrame]
1919
- subnodes, subedges = _collect_subframes(subgraphs, on_phase, on_subgraph, rig_cfg.source_info.infores_id, no_original)
1909
+ subnodes, subedges = _collect_subframes(subgraphs, on_phase, on_subgraph, rig_cfg.source_info.infores_id)
1920
1910
  _write_ndjson(subnodes, subedges, n, e, name, version, rig_cfg, section_sources, on_phase, domain, uuid_fields, uuid_on_collision)
1921
1911
 
1922
1912
 
@@ -710,10 +710,9 @@ class Annotation(Encoding):
710
710
  def clean_annotation(cls, annotation: str) -> str:
711
711
  cleaned: str = annotation.strip()
712
712
  lowered: str = cleaned.lower()
713
- # Allow-listed slots may carry uppercase (Biolink's ``FDA_regulatory_approvals``):
714
- # any casing the author declares canonicalizes onto the allow-listed spelling
715
- # verbatim, so the emitted edge field preserves its case exactly instead of being
716
- # lowercased into an unknown name.
713
+ # Allow-listed slots may carry uppercase: any casing the author declares
714
+ # canonicalizes onto the allow-listed spelling verbatim, so the emitted edge
715
+ # field preserves its case instead of being lowercased into an unknown name.
717
716
  return next((field for field in ALLOWED_EDGE_FIELDS if field.lower() == lowered), lowered)
718
717
 
719
718
  @model_validator(mode="after")
@@ -19,3 +19,4 @@ def hydrate_sources(db: Path) -> list[str]: ...
19
19
  def lookup_fullmap_terms(db: Path, terms: list[str], return_format: str = "rows") -> list[dict[str, Any]]: ...
20
20
  def namespace_uuid(domain: str, values: list[str]) -> str: ...
21
21
  def xxh64(data: str) -> str: ...
22
+ def xxh64_file(path: str) -> str: ...
@@ -0,0 +1,53 @@
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+ from typing import Any
5
+
6
+ from tablassert import rs
7
+ from tablassert.errors import SourceFileError
8
+
9
+ BASE: Path = Path("./.tablassert")
10
+ STORE: Path = BASE / "store"
11
+ STORE.mkdir(parents=True, exist_ok=True)
12
+
13
+
14
+ def mkhash(x: Any) -> str:
15
+ # xxh64 (not xxh32): the digest is the content-addressed identity for section
16
+ # parquet stores and the section label in validation errors. 32 bits invites
17
+ # birthday collisions (~50% at ~77k sections) that would silently reuse another
18
+ # section's cached subgraph; the full 16-hex 64-bit digest avoids that.
19
+ return rs.xxh64(str(x))
20
+
21
+
22
+ def file_content_hash(path: Path, *, config: Path | None = None, section_label: str | None = None) -> str:
23
+ """XXH64 hex digest of a source file's raw bytes, or a loud :class:`SourceFileError`.
24
+
25
+ Pure: no caching, so the digest always reflects the file's bytes at call time and an
26
+ edited source always yields a new section store key. ``config`` / ``section_label``
27
+ are optional build context forwarded to the error message when available.
28
+ """
29
+ resolved: Path = path.resolve()
30
+ if not resolved.exists():
31
+ raise SourceFileError(resolved, "no such file", config=config, section_label=section_label)
32
+ if not resolved.is_file():
33
+ raise SourceFileError(resolved, "not a regular file", config=config, section_label=section_label)
34
+ try:
35
+ return rs.xxh64_file(str(resolved))
36
+ except OSError as e:
37
+ raise SourceFileError(resolved, str(e), config=config, section_label=section_label) from e
38
+
39
+
40
+ def section_store_key(section: Any, local: Path | None = None, *, content_digest: str | None = None) -> str:
41
+ """Content-addressed identity for a section's cached parquet store.
42
+
43
+ With a ``local`` source file or a precomputed ``content_digest`` the key mixes the
44
+ section config hash with the file's content digest, so editing the source file
45
+ invalidates the cache; without one the key is the plain section hash. Always a
46
+ single 16-hex xxh64 digest. ``content_digest`` lets a caller memoize file reads
47
+ without changing the key formula.
48
+ """
49
+ if content_digest is None:
50
+ if local is None:
51
+ return mkhash(section)
52
+ content_digest = file_content_hash(local)
53
+ return mkhash(f"{mkhash(section)}:{content_digest}")
@@ -1,18 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from pathlib import Path
4
- from typing import Any
5
-
6
- from tablassert import rs
7
-
8
- BASE: Path = Path("./.tablassert")
9
- STORE: Path = BASE / "store"
10
- STORE.mkdir(parents=True, exist_ok=True)
11
-
12
-
13
- def mkhash(x: Any) -> str:
14
- # xxh64 (not xxh32): the digest is the content-addressed identity for section
15
- # parquet stores and the section label in validation errors. 32 bits invites
16
- # birthday collisions (~50% at ~77k sections) that would silently reuse another
17
- # section's cached subgraph; the full 16-hex 64-bit digest avoids that.
18
- return rs.xxh64(str(x))
File without changes
File without changes
File without changes