tablassert 16.1.1__tar.gz → 16.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {tablassert-16.1.1 → tablassert-16.2.0}/PKG-INFO +1 -1
  2. {tablassert-16.1.1 → tablassert-16.2.0}/pyproject.toml +1 -1
  3. {tablassert-16.1.1 → tablassert-16.2.0}/rust/src/ndjson.rs +428 -15
  4. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/cli.py +1 -0
  5. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/errors.py +1 -0
  6. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/lib.py +24 -6
  7. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/models.py +12 -1
  8. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/rs.pyi +3 -1
  9. {tablassert-16.1.1 → tablassert-16.2.0}/LICENSE +0 -0
  10. {tablassert-16.1.1 → tablassert-16.2.0}/README.md +0 -0
  11. {tablassert-16.1.1 → tablassert-16.2.0}/rust/Cargo.lock +0 -0
  12. {tablassert-16.1.1 → tablassert-16.2.0}/rust/Cargo.toml +0 -0
  13. {tablassert-16.1.1 → tablassert-16.2.0}/rust/examples/count_tables.rs +0 -0
  14. {tablassert-16.1.1 → tablassert-16.2.0}/rust/src/fullmap.rs +0 -0
  15. {tablassert-16.1.1 → tablassert-16.2.0}/rust/src/json.rs +0 -0
  16. {tablassert-16.1.1 → tablassert-16.2.0}/rust/src/lib.rs +0 -0
  17. {tablassert-16.1.1 → tablassert-16.2.0}/rust/src/uuid.rs +0 -0
  18. {tablassert-16.1.1 → tablassert-16.2.0}/rust/tests/build_golden.rs +0 -0
  19. {tablassert-16.1.1 → tablassert-16.2.0}/rust/tests/common/mod.rs +0 -0
  20. {tablassert-16.1.1 → tablassert-16.2.0}/rust/tests/extract_prebuilt.rs +0 -0
  21. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/__init__.py +0 -0
  22. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/_lazy.py +0 -0
  23. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/agent.py +0 -0
  24. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/biolink.py +0 -0
  25. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/coerce.py +0 -0
  26. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/enums.py +0 -0
  27. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/extras.py +0 -0
  28. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/fullmap.py +0 -0
  29. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/graph_target.py +0 -0
  30. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/ingests.py +0 -0
  31. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/log.py +0 -0
  32. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/nlp.py +0 -0
  33. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/progress.py +0 -0
  34. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/qc.py +0 -0
  35. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/rig.py +0 -0
  36. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/study.py +0 -0
  37. {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 16.1.1
3
+ Version: 16.2.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "16.1.1"
3
+ version = "16.2.0"
4
4
  description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -4,6 +4,7 @@ use pyo3::exceptions::PyRuntimeError;
4
4
  use pyo3::prelude::*;
5
5
  use rustc_hash::FxHashMap;
6
6
  use serde_json::Value;
7
+ use std::collections::hash_map::Entry;
7
8
  use std::fs::File;
8
9
  use std::io::{BufRead, BufReader, BufWriter, Write};
9
10
  use std::path::{Path, PathBuf};
@@ -191,19 +192,161 @@ fn not_a_key_error(output: &Path, id: &str, incoming: &Value, fields: Option<&[S
191
192
  ))
192
193
  }
193
194
 
195
+ /// Merge-mode dedup state: derived id -> (hashes of every absorbed record, merged record),
196
+ /// plus the first-seen id order so the buffered output is deterministic.
197
+ ///
198
+ /// Unlike the default `EdgeIndex` (24 bytes per edge, streaming writes), this retains one
199
+ /// COMPLETE record per unique id and writes nothing until end-of-stream, because a
200
+ /// divergent record must be folded into the record that already claimed the id. That
201
+ /// memory cost is exactly why merge mode is opt-in (`uuid_on_collision: merge`).
202
+ #[derive(Default)]
203
+ struct MergeIndex {
204
+ records: FxHashMap<[u8; 16], (Vec<u64>, Value)>,
205
+ order: Vec<[u8; 16]>,
206
+ merged: u64,
207
+ scalar_conflicts: u64,
208
+ }
209
+
210
+ /// Fold `incoming` into `stored`, field-wise. Returns the number of conflicting scalar
211
+ /// fields (kept first-wins) so the caller can report them.
212
+ ///
213
+ /// - list fields: union, deduped by canonical JSON bytes (so two `sources` objects that
214
+ /// differ only in key order collapse), then sorted by canonical bytes so the merged
215
+ /// output is identical regardless of which record arrived first;
216
+ /// - scalar fields: first-wins on conflict, counted;
217
+ /// - fields only on `incoming`: copied over (only a conflict when both sides disagree);
218
+ /// - `id` is never touched: both sides carry the same one by construction.
219
+ fn merge_records(stored: &mut Value, incoming: &Value) -> PyResult<u64> {
220
+ let Some(incoming_map) = incoming.as_object() else {
221
+ return Err(runtime_error("expected JSON object"));
222
+ };
223
+ let mut conflicts: u64 = 0;
224
+ let Some(stored_map) = stored.as_object_mut() else {
225
+ return Err(runtime_error("expected JSON object"));
226
+ };
227
+ for (key, incoming_value) in incoming_map {
228
+ if key == "id" {
229
+ continue;
230
+ }
231
+ match stored_map.get_mut(key) {
232
+ None => {
233
+ stored_map.insert(key.clone(), incoming_value.clone());
234
+ }
235
+ Some(stored_value) => {
236
+ if let (Value::Array(stored_items), Value::Array(incoming_items)) =
237
+ (&mut *stored_value, incoming_value)
238
+ {
239
+ let mut seen: Vec<Vec<u8>> = Vec::with_capacity(stored_items.len());
240
+ for item in stored_items.iter() {
241
+ seen.push(canonical_json_bytes(item).map_err(runtime_error)?);
242
+ }
243
+ for item in incoming_items {
244
+ let bytes: Vec<u8> = canonical_json_bytes(item).map_err(runtime_error)?;
245
+ if !seen.contains(&bytes) {
246
+ seen.push(bytes);
247
+ stored_items.push(item.clone());
248
+ }
249
+ }
250
+ let mut keyed: Vec<(Vec<u8>, Value)> = Vec::with_capacity(stored_items.len());
251
+ for item in stored_items.drain(..) {
252
+ keyed.push((canonical_json_bytes(&item).map_err(runtime_error)?, item));
253
+ }
254
+ keyed.sort_by(|left, right| left.0.cmp(&right.0));
255
+ stored_items.extend(keyed.into_iter().map(|(_, item)| item));
256
+ } else if stored_value != incoming_value {
257
+ conflicts += 1;
258
+ }
259
+ }
260
+ }
261
+ }
262
+ Ok(conflicts)
263
+ }
264
+
265
+ impl MergeIndex {
266
+ fn absorb(&mut self, id: [u8; 16], value: Value, content: u64) -> PyResult<()> {
267
+ match self.records.entry(id) {
268
+ Entry::Vacant(slot) => {
269
+ slot.insert((vec![content], value));
270
+ self.order.push(id);
271
+ Ok(())
272
+ }
273
+ Entry::Occupied(mut slot) => {
274
+ let (hashes, stored_value) = slot.get_mut();
275
+ // Exact repeat of ANY record already folded into this id -- including a
276
+ // divergent one -- is suppressed, so the conflict summary never
277
+ // double-counts a re-seen row.
278
+ if hashes.contains(&content) {
279
+ return Ok(());
280
+ }
281
+ self.merged += 1;
282
+ self.scalar_conflicts += merge_records(stored_value, &value)?;
283
+ hashes.push(content);
284
+ Ok(())
285
+ }
286
+ }
287
+ }
288
+ }
289
+
290
+ /// Merge-mode edge pass: buffer every unique edge, fold divergent same-id records into the
291
+ /// first, then write in first-seen order. Returns (divergent records merged, conflicting
292
+ /// scalar fields) for the summary log. Runs ONLY under `uuid_on_collision: merge`; the
293
+ /// default path stays streaming and never buffers a record.
294
+ fn dedup_edges_merge(
295
+ reader: BufReader<File>,
296
+ mut writer: BufWriter<File>,
297
+ domain: &str,
298
+ fields: Option<&[String]>,
299
+ ) -> PyResult<(u64, u64)> {
300
+ let mut index: MergeIndex = MergeIndex::default();
301
+ for line in reader.lines() {
302
+ let line: String = line.map_err(runtime_error)?;
303
+ if line.trim().is_empty() {
304
+ continue;
305
+ }
306
+ let value: Value = serde_json::from_str(&line).map_err(runtime_error)?;
307
+ let Some(Finalized { value, content }) = finalize_record(value, true, domain, fields)?
308
+ else {
309
+ continue;
310
+ };
311
+ index.absorb(edge_id_bytes(&value)?, value, content)?;
312
+ }
313
+ for id in &index.order {
314
+ let (_, value) = &index.records[id];
315
+ writer
316
+ .write_all(&emitted_json_bytes(value).map_err(runtime_error)?)
317
+ .map_err(runtime_error)?;
318
+ writer.write_all(b"\n").map_err(runtime_error)?;
319
+ }
320
+ writer.flush().map_err(runtime_error)?;
321
+ Ok((index.merged, index.scalar_conflicts))
322
+ }
323
+
194
324
  #[pyfunction]
195
- #[pyo3(signature = (input, output, is_edges, domain=None, uuid_fields=None))]
325
+ #[pyo3(signature = (input, output, is_edges, domain=None, uuid_fields=None, on_collision=None))]
196
326
  pub fn dedup_ndjson(
197
327
  input: PathBuf,
198
328
  output: PathBuf,
199
329
  is_edges: bool,
200
330
  domain: Option<String>,
201
331
  uuid_fields: Option<Vec<String>>,
202
- ) -> PyResult<()> {
332
+ on_collision: Option<String>,
333
+ ) -> PyResult<(u64, u64)> {
334
+ let merge: bool = match on_collision.as_deref() {
335
+ None | Some("error") => false,
336
+ Some("merge") => true,
337
+ Some(other) => {
338
+ return Err(runtime_error(format!(
339
+ "unknown on_collision {other:?}: expected \"error\" or \"merge\""
340
+ )));
341
+ }
342
+ };
203
343
  let domain: String = domain.unwrap_or_else(|| "TABLASSERT".to_string());
204
344
  let fields: Option<&[String]> = uuid_fields.as_deref();
205
345
  let reader: BufReader<File> = BufReader::new(File::open(input).map_err(runtime_error)?);
206
346
  let mut writer: BufWriter<File> = BufWriter::new(File::create(&output).map_err(runtime_error)?);
347
+ if is_edges && merge {
348
+ return dedup_edges_merge(reader, writer, &domain, fields);
349
+ }
207
350
  let mut nodes: FxHashMap<u64, Vec<Vec<u8>>> = FxHashMap::default();
208
351
  let mut edges: EdgeIndex = EdgeIndex::default();
209
352
 
@@ -249,7 +392,8 @@ pub fn dedup_ndjson(
249
392
  // Flush explicitly and propagate failure: relying on BufWriter's drop-time
250
393
  // flush would swallow a final write error and report success with truncated
251
394
  // output.
252
- writer.flush().map_err(runtime_error)
395
+ writer.flush().map_err(runtime_error)?;
396
+ Ok((0, 0))
253
397
  }
254
398
 
255
399
  #[cfg(test)]
@@ -287,7 +431,7 @@ mod tests {
287
431
  let output = dir.path().join("nodes.ndjson");
288
432
  fs::write(&input, "{\"id\":\"A\"}\n{\"id\":\"B\"}\n{\"id\":\"A\"}\n").expect("write input");
289
433
 
290
- dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
434
+ dedup_ndjson(input, output.clone(), false, None, None, None).expect("dedup nodes");
291
435
 
292
436
  let lines: Vec<String> = fs::read_to_string(output)
293
437
  .expect("read output")
@@ -311,7 +455,7 @@ mod tests {
311
455
  )
312
456
  .expect("write input");
313
457
 
314
- dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
458
+ dedup_ndjson(input, output.clone(), false, None, None, None).expect("dedup nodes");
315
459
 
316
460
  let lines: Vec<String> = fs::read_to_string(output)
317
461
  .expect("read output")
@@ -338,6 +482,7 @@ mod tests {
338
482
  true,
339
483
  Some("TABLASSERT".to_string()),
340
484
  None,
485
+ None,
341
486
  )
342
487
  .expect("dedup edges");
343
488
 
@@ -367,6 +512,7 @@ mod tests {
367
512
  true,
368
513
  Some("TABLASSERT".to_string()),
369
514
  None,
515
+ None,
370
516
  )
371
517
  .expect("dedup edges");
372
518
 
@@ -392,7 +538,7 @@ mod tests {
392
538
  )
393
539
  .expect("write input");
394
540
 
395
- dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
541
+ dedup_ndjson(input, output.clone(), false, None, None, None).expect("dedup nodes");
396
542
 
397
543
  let line = fs::read_to_string(output).expect("read output");
398
544
  let value: Value = serde_json::from_str(line.trim()).expect("json");
@@ -407,7 +553,7 @@ mod tests {
407
553
  let output = dir.path().join("nodes.ndjson");
408
554
  fs::write(&input, "{}\n{\"drop\":\"NA\"}\n").expect("write input");
409
555
 
410
- dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
556
+ dedup_ndjson(input, output.clone(), false, None, None, None).expect("dedup nodes");
411
557
 
412
558
  assert_eq!(fs::read_to_string(output).expect("read output"), "");
413
559
  }
@@ -419,7 +565,7 @@ mod tests {
419
565
  let output = dir.path().join("nodes.ndjson");
420
566
  fs::write(&input, "\n \n{\"id\":\"A\"}\n\n{\"id\":\"A\"}\n \n").expect("write input");
421
567
 
422
- dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
568
+ dedup_ndjson(input, output.clone(), false, None, None, None).expect("dedup nodes");
423
569
 
424
570
  assert_eq!(
425
571
  fs::read_to_string(output).expect("read output"),
@@ -463,14 +609,22 @@ mod tests {
463
609
  dir.path(),
464
610
  "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
465
611
  );
466
- dedup_ndjson(before_in, before_out.clone(), true, None, fields.clone()).expect("dedup");
612
+ dedup_ndjson(
613
+ before_in,
614
+ before_out.clone(),
615
+ true,
616
+ None,
617
+ fields.clone(),
618
+ None,
619
+ )
620
+ .expect("dedup");
467
621
 
468
622
  let after_dir = tempdir().expect("tempdir");
469
623
  let (after_in, after_out) = write_edges(
470
624
  after_dir.path(),
471
625
  "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\",\"effect_size\":1.5}\n",
472
626
  );
473
- dedup_ndjson(after_in, after_out.clone(), true, None, fields).expect("dedup");
627
+ dedup_ndjson(after_in, after_out.clone(), true, None, fields, None).expect("dedup");
474
628
 
475
629
  assert_eq!(edge_ids(&before_out), edge_ids(&after_out));
476
630
  }
@@ -484,14 +638,14 @@ mod tests {
484
638
  dir.path(),
485
639
  "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
486
640
  );
487
- dedup_ndjson(before_in, before_out.clone(), true, None, None).expect("dedup");
641
+ dedup_ndjson(before_in, before_out.clone(), true, None, None, None).expect("dedup");
488
642
 
489
643
  let after_dir = tempdir().expect("tempdir");
490
644
  let (after_in, after_out) = write_edges(
491
645
  after_dir.path(),
492
646
  "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n",
493
647
  );
494
- dedup_ndjson(after_in, after_out.clone(), true, None, None).expect("dedup");
648
+ dedup_ndjson(after_in, after_out.clone(), true, None, None, None).expect("dedup");
495
649
 
496
650
  assert_ne!(edge_ids(&before_out), edge_ids(&after_out));
497
651
  }
@@ -514,7 +668,7 @@ mod tests {
514
668
  "predicate".to_string(),
515
669
  "object".to_string(),
516
670
  ]);
517
- let error = dedup_ndjson(input, output, true, None, fields).expect_err("not a key");
671
+ let error = dedup_ndjson(input, output, true, None, fields, None).expect_err("not a key");
518
672
  let message = error.to_string();
519
673
  assert!(message.contains("uuid-fields-not-a-key"), "{message}");
520
674
  assert!(message.contains("p_value"), "{message}");
@@ -534,7 +688,7 @@ mod tests {
534
688
  ),
535
689
  );
536
690
  let fields = Some(vec!["subject".to_string(), "object".to_string()]);
537
- dedup_ndjson(input, output.clone(), true, None, fields).expect("dedup edges");
691
+ dedup_ndjson(input, output.clone(), true, None, fields, None).expect("dedup edges");
538
692
  assert_eq!(edge_ids(&output).len(), 1);
539
693
  }
540
694
 
@@ -551,7 +705,7 @@ mod tests {
551
705
  "{\"object\":\"B\",\"subject\":\"A\"}\n"
552
706
  ),
553
707
  );
554
- dedup_ndjson(input, output.clone(), true, None, None).expect("dedup edges");
708
+ dedup_ndjson(input, output.clone(), true, None, None, None).expect("dedup edges");
555
709
  assert_eq!(edge_ids(&output).len(), 1);
556
710
  }
557
711
 
@@ -568,6 +722,7 @@ mod tests {
568
722
  true,
569
723
  Some("infores:left".to_string()),
570
724
  None,
725
+ None,
571
726
  )
572
727
  .expect("dedup");
573
728
 
@@ -580,6 +735,7 @@ mod tests {
580
735
  true,
581
736
  Some("infores:right".to_string()),
582
737
  None,
738
+ None,
583
739
  )
584
740
  .expect("dedup");
585
741
 
@@ -596,4 +752,261 @@ mod tests {
596
752
  vec!["effect_size", "p_value"]
597
753
  );
598
754
  }
755
+
756
+ fn merged_edge(output: &std::path::Path) -> Value {
757
+ let lines: Vec<String> = fs::read_to_string(output)
758
+ .expect("read output")
759
+ .lines()
760
+ .map(str::to_string)
761
+ .collect();
762
+ assert_eq!(lines.len(), 1, "expected one merged edge, got {lines:?}");
763
+ serde_json::from_str(&lines[0]).expect("json")
764
+ }
765
+
766
+ const SPO: &[&str] = &["subject", "predicate", "object"];
767
+
768
+ fn spo_fields() -> Option<Vec<String>> {
769
+ Some(SPO.iter().map(ToString::to_string).collect())
770
+ }
771
+
772
+ #[test]
773
+ fn merge_mode_folds_divergent_edges_into_one() {
774
+ // WHY: `uuid_on_collision: merge`. Two rows whose raw mention spellings resolve to
775
+ // the same CURIE derive one id; merge mode unions their evidence into a single
776
+ // edge instead of aborting with `uuid-fields-not-a-key`.
777
+ let dir = tempdir().expect("tempdir");
778
+ let (input, output) = write_edges(
779
+ dir.path(),
780
+ concat!(
781
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\",\"publications\":[\"PMID:2\",\"PMID:1\"]}\n",
782
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\",\"publications\":[\"PMID:3\",\"PMID:1\"]}\n"
783
+ ),
784
+ );
785
+ let (merged, conflicts) = dedup_ndjson(
786
+ input,
787
+ output.clone(),
788
+ true,
789
+ None,
790
+ spo_fields(),
791
+ Some("merge".to_string()),
792
+ )
793
+ .expect("merge dedup");
794
+
795
+ assert_eq!((merged, conflicts), (1, 1));
796
+ let edge = merged_edge(&output);
797
+ // List fields union, dedup, and sort; scalar conflicts keep the first value.
798
+ assert_eq!(
799
+ edge["publications"],
800
+ serde_json::json!(["PMID:1", "PMID:2", "PMID:3"])
801
+ );
802
+ assert_eq!(edge["p_value"], serde_json::json!("0.01"));
803
+ }
804
+
805
+ #[test]
806
+ fn merge_mode_suppresses_exact_repeats_without_remerging() {
807
+ // WHY: an exact repeat is a Duplicate even when it repeats a DIVERGENT record
808
+ // already folded into the merged edge -- re-merging it would leave the record
809
+ // unchanged but double-count the scalar conflict in the summary.
810
+ let dir = tempdir().expect("tempdir");
811
+ let (input, output) = write_edges(
812
+ dir.path(),
813
+ concat!(
814
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
815
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n",
816
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n"
817
+ ),
818
+ );
819
+ let (merged, conflicts) = dedup_ndjson(
820
+ input,
821
+ output.clone(),
822
+ true,
823
+ None,
824
+ spo_fields(),
825
+ Some("merge".to_string()),
826
+ )
827
+ .expect("merge dedup");
828
+
829
+ assert_eq!((merged, conflicts), (1, 1));
830
+ assert_eq!(merged_edge(&output)["p_value"], serde_json::json!("0.01"));
831
+ }
832
+
833
+ #[test]
834
+ fn merge_mode_output_is_order_independent() {
835
+ // WHY: merged output must not depend on which row the source happened to emit
836
+ // first, or two builds of one graph would diverge. List unions sort by canonical
837
+ // bytes, so both arrival orders produce byte-identical output.
838
+ let left_dir = tempdir().expect("tempdir");
839
+ let (left_in, left_out) = write_edges(
840
+ left_dir.path(),
841
+ concat!(
842
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:2\"]}\n",
843
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:1\",\"PMID:3\"]}\n"
844
+ ),
845
+ );
846
+ dedup_ndjson(
847
+ left_in,
848
+ left_out.clone(),
849
+ true,
850
+ None,
851
+ spo_fields(),
852
+ Some("merge".to_string()),
853
+ )
854
+ .expect("merge dedup");
855
+
856
+ let right_dir = tempdir().expect("tempdir");
857
+ let (right_in, right_out) = write_edges(
858
+ right_dir.path(),
859
+ concat!(
860
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:1\",\"PMID:3\"]}\n",
861
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:2\"]}\n"
862
+ ),
863
+ );
864
+ dedup_ndjson(
865
+ right_in,
866
+ right_out.clone(),
867
+ true,
868
+ None,
869
+ spo_fields(),
870
+ Some("merge".to_string()),
871
+ )
872
+ .expect("merge dedup");
873
+
874
+ assert_eq!(
875
+ fs::read_to_string(left_out).expect("read left"),
876
+ fs::read_to_string(right_out).expect("read right")
877
+ );
878
+ }
879
+
880
+ #[test]
881
+ fn merge_mode_writes_edges_in_first_seen_order() {
882
+ // WHY: buffering must not reorder the graph -- the first record to claim each id
883
+ // fixes its position in the output.
884
+ let dir = tempdir().expect("tempdir");
885
+ let (input, output) = write_edges(
886
+ dir.path(),
887
+ concat!(
888
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:1\"]}\n",
889
+ "{\"subject\":\"X\",\"predicate\":\"r\",\"object\":\"Y\"}\n",
890
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:2\"]}\n"
891
+ ),
892
+ );
893
+ dedup_ndjson(
894
+ input,
895
+ output.clone(),
896
+ true,
897
+ None,
898
+ spo_fields(),
899
+ Some("merge".to_string()),
900
+ )
901
+ .expect("merge dedup");
902
+
903
+ let lines: Vec<Value> = fs::read_to_string(output)
904
+ .expect("read output")
905
+ .lines()
906
+ .map(|line| serde_json::from_str(line).expect("json"))
907
+ .collect();
908
+ assert_eq!(lines.len(), 2);
909
+ assert_eq!(lines[0]["subject"], serde_json::json!("A"));
910
+ assert_eq!(
911
+ lines[0]["publications"],
912
+ serde_json::json!(["PMID:1", "PMID:2"])
913
+ );
914
+ assert_eq!(lines[1]["subject"], serde_json::json!("X"));
915
+ }
916
+
917
+ #[test]
918
+ fn merge_mode_dedups_list_objects_by_canonical_bytes() {
919
+ // WHY: `sources` entries are objects; two entries that differ only in key order
920
+ // are the same source and must collapse, and the union must sort so merged output
921
+ // is deterministic.
922
+ let dir = tempdir().expect("tempdir");
923
+ let (input, output) = write_edges(
924
+ dir.path(),
925
+ concat!(
926
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"sources\":[{\"resource_id\":\"infores:b\",\"resource_role\":\"primary_knowledge_source\"}]}\n",
927
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"sources\":[{\"resource_role\":\"primary_knowledge_source\",\"resource_id\":\"infores:b\"},{\"resource_id\":\"infores:a\",\"resource_role\":\"aggregator_knowledge_source\"}]}\n"
928
+ ),
929
+ );
930
+ dedup_ndjson(
931
+ input,
932
+ output.clone(),
933
+ true,
934
+ None,
935
+ spo_fields(),
936
+ Some("merge".to_string()),
937
+ )
938
+ .expect("merge dedup");
939
+
940
+ let edge = merged_edge(&output);
941
+ let sources = edge["sources"].as_array().expect("sources array");
942
+ assert_eq!(
943
+ sources.len(),
944
+ 2,
945
+ "key order alone must not keep both entries"
946
+ );
947
+ assert_eq!(sources[0]["resource_id"], serde_json::json!("infores:a"));
948
+ assert_eq!(sources[1]["resource_id"], serde_json::json!("infores:b"));
949
+ }
950
+
951
+ #[test]
952
+ fn merge_mode_keeps_fields_only_the_second_record_carries() {
953
+ // WHY: first-wins arbitrates CONFLICTS; a field absent from the first record is
954
+ // not a conflict, it is extra evidence, and dropping it would lose data.
955
+ let dir = tempdir().expect("tempdir");
956
+ let (input, output) = write_edges(
957
+ dir.path(),
958
+ concat!(
959
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\"}\n",
960
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"effect_size\":1.5}\n"
961
+ ),
962
+ );
963
+ let (merged, conflicts) = dedup_ndjson(
964
+ input,
965
+ output.clone(),
966
+ true,
967
+ None,
968
+ spo_fields(),
969
+ Some("merge".to_string()),
970
+ )
971
+ .expect("merge dedup");
972
+
973
+ assert_eq!((merged, conflicts), (1, 0));
974
+ assert_eq!(merged_edge(&output)["effect_size"], serde_json::json!(1.5));
975
+ }
976
+
977
+ #[test]
978
+ fn explicit_error_mode_still_aborts_on_divergent_edges() {
979
+ // WHY: `uuid_on_collision` defaults to `error`; naming it explicitly must behave
980
+ // exactly like the default.
981
+ let dir = tempdir().expect("tempdir");
982
+ let (input, output) = write_edges(
983
+ dir.path(),
984
+ concat!(
985
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
986
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n"
987
+ ),
988
+ );
989
+ let error = dedup_ndjson(
990
+ input,
991
+ output,
992
+ true,
993
+ None,
994
+ spo_fields(),
995
+ Some("error".to_string()),
996
+ )
997
+ .expect_err("not a key");
998
+ assert!(
999
+ error.to_string().contains("uuid-fields-not-a-key"),
1000
+ "{error}"
1001
+ );
1002
+ }
1003
+
1004
+ #[test]
1005
+ fn unknown_on_collision_is_rejected() {
1006
+ let dir = tempdir().expect("tempdir");
1007
+ let (input, output) = write_edges(dir.path(), "{\"subject\":\"A\",\"object\":\"B\"}\n");
1008
+ let error = dedup_ndjson(input, output, true, None, None, Some("bogus".to_string()))
1009
+ .expect_err("unknown mode");
1010
+ assert!(error.to_string().contains("bogus"), "{error}");
1011
+ }
599
1012
  }
@@ -265,6 +265,7 @@ def build_graph_pipeline(
265
265
  no_original=no_original,
266
266
  uuid_fields=g.uuid_fields,
267
267
  uuid_domain=g.uuid_namespace,
268
+ uuid_on_collision=g.uuid_on_collision,
268
269
  )
269
270
 
270
271
  # Stage 7/7 (only with --qc): assert over the final NDJSON files.
@@ -44,6 +44,7 @@ TablassertErrorCodes = Literal[
44
44
  "rig-validation-failed",
45
45
  "uuid-bad-fields",
46
46
  "uuid-fields-not-a-key",
47
+ "uuid-merge-without-fields",
47
48
  ]
48
49
 
49
50
 
@@ -1578,7 +1578,7 @@ def upstream_resource_ids(repo: Repositories) -> list[str]:
1578
1578
  return [InformationResources.PUBMED.value]
1579
1579
 
1580
1580
 
1581
- def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fields: list[str] | None = None) -> None:
1581
+ def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fields: list[str] | None = None, on_collision: str = "error") -> None:
1582
1582
  """Remove null values from and deduplicate an NDJSON stream.
1583
1583
 
1584
1584
  Args:
@@ -1588,12 +1588,17 @@ def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fi
1588
1588
  Ignored for nodes, whose ids are CURIEs rather than derived hashes.
1589
1589
  uuid_fields: Optional edge fields that constitute edge identity
1590
1590
  (``Graph.uuid_fields``). ``None`` hashes the whole record.
1591
+ on_collision: ``error`` aborts on two different edges deriving one id;
1592
+ ``merge`` folds them into one edge (``Graph.uuid_on_collision``).
1591
1593
 
1592
1594
  Notes:
1593
1595
  Also adds UUIDs to edges. Edges deduplicate on their derived id, so the
1594
1596
  output can never carry the same id twice: an exact repeat is collapsed,
1595
1597
  while two genuinely different edges deriving one id abort the build with
1596
- ``uuid-fields-not-a-key`` rather than shipping a duplicate.
1598
+ ``uuid-fields-not-a-key`` rather than shipping a duplicate -- unless
1599
+ ``on_collision="merge"``, which unions their list fields (sorted, so the
1600
+ output is merge-order independent), keeps the first value of conflicting
1601
+ scalars, and buffers one full record per unique id until end-of-stream.
1597
1602
 
1598
1603
  Returns:
1599
1604
  ``None``; writes the deduplicated stream alongside ``p_in`` with no
@@ -1604,7 +1609,13 @@ def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fi
1604
1609
  if p_out.is_file():
1605
1610
  p_out.unlink()
1606
1611
 
1607
- rs.dedup_ndjson(p_in, p_out, is_edges, domain, uuid_fields)
1612
+ merged, scalar_conflicts = rs.dedup_ndjson(p_in, p_out, is_edges, domain, uuid_fields, on_collision)
1613
+ if merged:
1614
+ logger.warning(
1615
+ "uuid_on_collision=merge folded {merged} divergent edge records into their first-seen edges; {conflicts} conflicting scalar fields kept the first value",
1616
+ merged=merged,
1617
+ conflicts=scalar_conflicts,
1618
+ )
1608
1619
 
1609
1620
  p_in.unlink()
1610
1621
 
@@ -1762,6 +1773,7 @@ def _write_ndjson(
1762
1773
  on_phase: Callable[[str], None] | None = None,
1763
1774
  domain: str = "TABLASSERT",
1764
1775
  uuid_fields: list[str] | None = None,
1776
+ on_collision: str = "error",
1765
1777
  ) -> None:
1766
1778
  """Write, dedup, and RIG the KGX NDJSON outputs.
1767
1779
 
@@ -1790,6 +1802,8 @@ def _write_ndjson(
1790
1802
  boundary, used to drive progress UX.
1791
1803
  domain: UUID namespace for derived edge ids (``Graph.uuid_namespace``).
1792
1804
  uuid_fields: Optional edge identity fields (``Graph.uuid_fields``).
1805
+ on_collision: Collision policy for divergent same-id edges
1806
+ (``Graph.uuid_on_collision``).
1793
1807
  """
1794
1808
  # Phase: write-nodes. Collection point: appending to output files.
1795
1809
  if on_phase is not None:
@@ -1810,7 +1824,7 @@ def _write_ndjson(
1810
1824
  # Phase: dedup.
1811
1825
  if on_phase is not None:
1812
1826
  on_phase("dedup")
1813
- dedup_stream(edges_tmp, is_edges=True, domain=domain, uuid_fields=uuid_fields)
1827
+ dedup_stream(edges_tmp, is_edges=True, domain=domain, uuid_fields=uuid_fields, on_collision=on_collision)
1814
1828
  # The deduper hashes the record WITH the literal `{edge_id}` placeholder still
1815
1829
  # in place, so edge ids stay deterministic regardless of this resolution pass.
1816
1830
  _resolve_edge_id_placeholders(edges_tmp.with_suffix(""))
@@ -1834,6 +1848,7 @@ def compile_graph(
1834
1848
  no_original: bool = False,
1835
1849
  uuid_fields: list[str] | None = None,
1836
1850
  uuid_domain: str | None = None,
1851
+ uuid_on_collision: str = "error",
1837
1852
  ) -> None:
1838
1853
  """Aggregate subgraph parquets for NDJSON KGX export using a lazy scan.
1839
1854
 
@@ -1861,6 +1876,8 @@ def compile_graph(
1861
1876
  infores when ``uuid_fields`` is set -- narrowing the hash inputs
1862
1877
  removes the accidental cross-graph uniqueness that full-record hashing
1863
1878
  provided -- and to ``TABLASSERT`` otherwise.
1879
+ uuid_on_collision: ``error`` aborts when two different edges derive one id;
1880
+ ``merge`` folds them into one edge instead (``Graph.uuid_on_collision``).
1864
1881
 
1865
1882
  Returns:
1866
1883
  ``None``; writes ``<name>_<version>.nodes.ndjson``,
@@ -1871,7 +1888,8 @@ def compile_graph(
1871
1888
  TablassertError: With code ``rig-validation-failed`` when the
1872
1889
  generated RIG fails its built-in audit (nothing is then written).
1873
1890
  RuntimeError: Tagged ``uuid-fields-not-a-key`` when two genuinely
1874
- different edges derive one id under the declared ``uuid_fields``.
1891
+ different edges derive one id under the declared ``uuid_fields``
1892
+ and ``uuid_on_collision`` is ``error``.
1875
1893
  """
1876
1894
  rig_cfg: RIGConfig = rig if isinstance(rig, RIGConfig) else RIGConfig.model_validate(rig)
1877
1895
  out_dir: Path = Path(rig_cfg.artifact_base_path)
@@ -1891,7 +1909,7 @@ def compile_graph(
1891
1909
  subnodes: list[pl.LazyFrame]
1892
1910
  subedges: list[pl.LazyFrame]
1893
1911
  subnodes, subedges = _collect_subframes(subgraphs, on_phase, on_subgraph, rig_cfg.source_info.infores_id, no_original)
1894
- _write_ndjson(subnodes, subedges, n, e, name, version, rig_cfg, section_sources, on_phase, domain, uuid_fields)
1912
+ _write_ndjson(subnodes, subedges, n, e, name, version, rig_cfg, section_sources, on_phase, domain, uuid_fields, uuid_on_collision)
1895
1913
 
1896
1914
 
1897
1915
  def resolve_many(
@@ -1139,6 +1139,10 @@ class Graph(TablaBase):
1139
1139
  description="Explicit UUID namespace. Defaults to `rig.source_info.infores_id` when `uuid_fields` is set, `TABLASSERT` otherwise. Set it only when graphs must deliberately share an id space.",
1140
1140
  examples=["infores:multiomicskg"],
1141
1141
  )
1142
+ uuid_on_collision: Literal["error", "merge"] = Field(
1143
+ default="error",
1144
+ description="What to do when two different edges derive one id under `uuid_fields`: `error` aborts the build with `uuid-fields-not-a-key`; `merge` folds the records into one edge (list fields unioned and sorted, scalars first-wins). Requires `uuid_fields`, and buffers one full record per unique id, which is why it is opt-in.",
1145
+ )
1142
1146
 
1143
1147
  @model_validator(mode="after")
1144
1148
  def validate_uuid_fields(self: Self) -> Self:
@@ -1147,9 +1151,16 @@ class Graph(TablaBase):
1147
1151
  Every entry must be a real emittable edge field, or the id would silently derive
1148
1152
  from nothing and every edge in the graph would collide. `id` itself is rejected
1149
1153
  because it is the value being derived. Casing follows `Annotation.clean_annotation`
1150
- so `Subject` and `subject` both work and mixed-case Biolink slots survive.
1154
+ so `Subject` and `subject` both work and mixed-case Biolink slots survive. Also
1155
+ rejects `uuid_on_collision: merge` without `uuid_fields`: under the whole-record
1156
+ hash no two different records can share an id, so there is nothing to merge.
1151
1157
  """
1152
1158
  if self.uuid_fields is None:
1159
+ if self.uuid_on_collision == "merge":
1160
+ raise TablassertValidationError(
1161
+ "`uuid_on_collision: merge` requires `uuid_fields`. With the whole-record hash every field is an identity field, so two different records can never derive one id and there is nothing to merge.",
1162
+ code="uuid-merge-without-fields",
1163
+ )
1153
1164
  return self
1154
1165
  if not self.uuid_fields:
1155
1166
  raise TablassertValidationError(
@@ -7,7 +7,9 @@ from typing import Any
7
7
  def build_fullmap_db(
8
8
  output: Path, classes: list[Path], synonyms: list[Path], threads: int | None = None, progress: Callable[[int, int, int, str], None] | None = None
9
9
  ) -> None: ...
10
- def dedup_ndjson(input: Path, output: Path, is_edges: bool, domain: str | None = None, uuid_fields: list[str] | None = None) -> None: ...
10
+ def dedup_ndjson(
11
+ input: Path, output: Path, is_edges: bool, domain: str | None = None, uuid_fields: list[str] | None = None, on_collision: str | None = None
12
+ ) -> tuple[int, int]: ...
11
13
  def extract_prebuilt_fullmap(archive: Path, output: Path, progress: Callable[[str], None] | None = None) -> None: ...
12
14
  def fullmap_source_version() -> str: ...
13
15
  def hydrate_categories(db: Path) -> list[str]: ...
File without changes
File without changes
File without changes
File without changes
File without changes