tablassert 16.1.1__tar.gz → 16.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-16.1.1 → tablassert-16.2.0}/PKG-INFO +1 -1
- {tablassert-16.1.1 → tablassert-16.2.0}/pyproject.toml +1 -1
- {tablassert-16.1.1 → tablassert-16.2.0}/rust/src/ndjson.rs +428 -15
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/cli.py +1 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/errors.py +1 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/lib.py +24 -6
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/models.py +12 -1
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/rs.pyi +3 -1
- {tablassert-16.1.1 → tablassert-16.2.0}/LICENSE +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/README.md +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/rust/Cargo.lock +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/rust/Cargo.toml +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/rust/src/fullmap.rs +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/rust/src/json.rs +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/rust/src/lib.rs +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/rust/src/uuid.rs +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/__init__.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/agent.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/biolink.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/coerce.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/enums.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/extras.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/fullmap.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/ingests.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/log.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/nlp.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/progress.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/qc.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/rig.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/study.py +0 -0
- {tablassert-16.1.1 → tablassert-16.2.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "16.
|
|
3
|
+
version = "16.2.0"
|
|
4
4
|
description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -4,6 +4,7 @@ use pyo3::exceptions::PyRuntimeError;
|
|
|
4
4
|
use pyo3::prelude::*;
|
|
5
5
|
use rustc_hash::FxHashMap;
|
|
6
6
|
use serde_json::Value;
|
|
7
|
+
use std::collections::hash_map::Entry;
|
|
7
8
|
use std::fs::File;
|
|
8
9
|
use std::io::{BufRead, BufReader, BufWriter, Write};
|
|
9
10
|
use std::path::{Path, PathBuf};
|
|
@@ -191,19 +192,161 @@ fn not_a_key_error(output: &Path, id: &str, incoming: &Value, fields: Option<&[S
|
|
|
191
192
|
))
|
|
192
193
|
}
|
|
193
194
|
|
|
195
|
+
/// Merge-mode dedup state: derived id -> (hashes of every absorbed record, merged record),
|
|
196
|
+
/// plus the first-seen id order so the buffered output is deterministic.
|
|
197
|
+
///
|
|
198
|
+
/// Unlike the default `EdgeIndex` (24 bytes per edge, streaming writes), this retains one
|
|
199
|
+
/// COMPLETE record per unique id and writes nothing until end-of-stream, because a
|
|
200
|
+
/// divergent record must be folded into the record that already claimed the id. That
|
|
201
|
+
/// memory cost is exactly why merge mode is opt-in (`uuid_on_collision: merge`).
|
|
202
|
+
#[derive(Default)]
|
|
203
|
+
struct MergeIndex {
|
|
204
|
+
records: FxHashMap<[u8; 16], (Vec<u64>, Value)>,
|
|
205
|
+
order: Vec<[u8; 16]>,
|
|
206
|
+
merged: u64,
|
|
207
|
+
scalar_conflicts: u64,
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/// Fold `incoming` into `stored`, field-wise. Returns the number of conflicting scalar
|
|
211
|
+
/// fields (kept first-wins) so the caller can report them.
|
|
212
|
+
///
|
|
213
|
+
/// - list fields: union, deduped by canonical JSON bytes (so two `sources` objects that
|
|
214
|
+
/// differ only in key order collapse), then sorted by canonical bytes so the merged
|
|
215
|
+
/// output is identical regardless of which record arrived first;
|
|
216
|
+
/// - scalar fields: first-wins on conflict, counted;
|
|
217
|
+
/// - fields only on `incoming`: copied over (only a conflict when both sides disagree);
|
|
218
|
+
/// - `id` is never touched: both sides carry the same one by construction.
|
|
219
|
+
fn merge_records(stored: &mut Value, incoming: &Value) -> PyResult<u64> {
|
|
220
|
+
let Some(incoming_map) = incoming.as_object() else {
|
|
221
|
+
return Err(runtime_error("expected JSON object"));
|
|
222
|
+
};
|
|
223
|
+
let mut conflicts: u64 = 0;
|
|
224
|
+
let Some(stored_map) = stored.as_object_mut() else {
|
|
225
|
+
return Err(runtime_error("expected JSON object"));
|
|
226
|
+
};
|
|
227
|
+
for (key, incoming_value) in incoming_map {
|
|
228
|
+
if key == "id" {
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
match stored_map.get_mut(key) {
|
|
232
|
+
None => {
|
|
233
|
+
stored_map.insert(key.clone(), incoming_value.clone());
|
|
234
|
+
}
|
|
235
|
+
Some(stored_value) => {
|
|
236
|
+
if let (Value::Array(stored_items), Value::Array(incoming_items)) =
|
|
237
|
+
(&mut *stored_value, incoming_value)
|
|
238
|
+
{
|
|
239
|
+
let mut seen: Vec<Vec<u8>> = Vec::with_capacity(stored_items.len());
|
|
240
|
+
for item in stored_items.iter() {
|
|
241
|
+
seen.push(canonical_json_bytes(item).map_err(runtime_error)?);
|
|
242
|
+
}
|
|
243
|
+
for item in incoming_items {
|
|
244
|
+
let bytes: Vec<u8> = canonical_json_bytes(item).map_err(runtime_error)?;
|
|
245
|
+
if !seen.contains(&bytes) {
|
|
246
|
+
seen.push(bytes);
|
|
247
|
+
stored_items.push(item.clone());
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
let mut keyed: Vec<(Vec<u8>, Value)> = Vec::with_capacity(stored_items.len());
|
|
251
|
+
for item in stored_items.drain(..) {
|
|
252
|
+
keyed.push((canonical_json_bytes(&item).map_err(runtime_error)?, item));
|
|
253
|
+
}
|
|
254
|
+
keyed.sort_by(|left, right| left.0.cmp(&right.0));
|
|
255
|
+
stored_items.extend(keyed.into_iter().map(|(_, item)| item));
|
|
256
|
+
} else if stored_value != incoming_value {
|
|
257
|
+
conflicts += 1;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
Ok(conflicts)
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
impl MergeIndex {
|
|
266
|
+
fn absorb(&mut self, id: [u8; 16], value: Value, content: u64) -> PyResult<()> {
|
|
267
|
+
match self.records.entry(id) {
|
|
268
|
+
Entry::Vacant(slot) => {
|
|
269
|
+
slot.insert((vec![content], value));
|
|
270
|
+
self.order.push(id);
|
|
271
|
+
Ok(())
|
|
272
|
+
}
|
|
273
|
+
Entry::Occupied(mut slot) => {
|
|
274
|
+
let (hashes, stored_value) = slot.get_mut();
|
|
275
|
+
// Exact repeat of ANY record already folded into this id -- including a
|
|
276
|
+
// divergent one -- is suppressed, so the conflict summary never
|
|
277
|
+
// double-counts a re-seen row.
|
|
278
|
+
if hashes.contains(&content) {
|
|
279
|
+
return Ok(());
|
|
280
|
+
}
|
|
281
|
+
self.merged += 1;
|
|
282
|
+
self.scalar_conflicts += merge_records(stored_value, &value)?;
|
|
283
|
+
hashes.push(content);
|
|
284
|
+
Ok(())
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/// Merge-mode edge pass: buffer every unique edge, fold divergent same-id records into the
|
|
291
|
+
/// first, then write in first-seen order. Returns (divergent records merged, conflicting
|
|
292
|
+
/// scalar fields) for the summary log. Runs ONLY under `uuid_on_collision: merge`; the
|
|
293
|
+
/// default path stays streaming and never buffers a record.
|
|
294
|
+
fn dedup_edges_merge(
|
|
295
|
+
reader: BufReader<File>,
|
|
296
|
+
mut writer: BufWriter<File>,
|
|
297
|
+
domain: &str,
|
|
298
|
+
fields: Option<&[String]>,
|
|
299
|
+
) -> PyResult<(u64, u64)> {
|
|
300
|
+
let mut index: MergeIndex = MergeIndex::default();
|
|
301
|
+
for line in reader.lines() {
|
|
302
|
+
let line: String = line.map_err(runtime_error)?;
|
|
303
|
+
if line.trim().is_empty() {
|
|
304
|
+
continue;
|
|
305
|
+
}
|
|
306
|
+
let value: Value = serde_json::from_str(&line).map_err(runtime_error)?;
|
|
307
|
+
let Some(Finalized { value, content }) = finalize_record(value, true, domain, fields)?
|
|
308
|
+
else {
|
|
309
|
+
continue;
|
|
310
|
+
};
|
|
311
|
+
index.absorb(edge_id_bytes(&value)?, value, content)?;
|
|
312
|
+
}
|
|
313
|
+
for id in &index.order {
|
|
314
|
+
let (_, value) = &index.records[id];
|
|
315
|
+
writer
|
|
316
|
+
.write_all(&emitted_json_bytes(value).map_err(runtime_error)?)
|
|
317
|
+
.map_err(runtime_error)?;
|
|
318
|
+
writer.write_all(b"\n").map_err(runtime_error)?;
|
|
319
|
+
}
|
|
320
|
+
writer.flush().map_err(runtime_error)?;
|
|
321
|
+
Ok((index.merged, index.scalar_conflicts))
|
|
322
|
+
}
|
|
323
|
+
|
|
194
324
|
#[pyfunction]
|
|
195
|
-
#[pyo3(signature = (input, output, is_edges, domain=None, uuid_fields=None))]
|
|
325
|
+
#[pyo3(signature = (input, output, is_edges, domain=None, uuid_fields=None, on_collision=None))]
|
|
196
326
|
pub fn dedup_ndjson(
|
|
197
327
|
input: PathBuf,
|
|
198
328
|
output: PathBuf,
|
|
199
329
|
is_edges: bool,
|
|
200
330
|
domain: Option<String>,
|
|
201
331
|
uuid_fields: Option<Vec<String>>,
|
|
202
|
-
|
|
332
|
+
on_collision: Option<String>,
|
|
333
|
+
) -> PyResult<(u64, u64)> {
|
|
334
|
+
let merge: bool = match on_collision.as_deref() {
|
|
335
|
+
None | Some("error") => false,
|
|
336
|
+
Some("merge") => true,
|
|
337
|
+
Some(other) => {
|
|
338
|
+
return Err(runtime_error(format!(
|
|
339
|
+
"unknown on_collision {other:?}: expected \"error\" or \"merge\""
|
|
340
|
+
)));
|
|
341
|
+
}
|
|
342
|
+
};
|
|
203
343
|
let domain: String = domain.unwrap_or_else(|| "TABLASSERT".to_string());
|
|
204
344
|
let fields: Option<&[String]> = uuid_fields.as_deref();
|
|
205
345
|
let reader: BufReader<File> = BufReader::new(File::open(input).map_err(runtime_error)?);
|
|
206
346
|
let mut writer: BufWriter<File> = BufWriter::new(File::create(&output).map_err(runtime_error)?);
|
|
347
|
+
if is_edges && merge {
|
|
348
|
+
return dedup_edges_merge(reader, writer, &domain, fields);
|
|
349
|
+
}
|
|
207
350
|
let mut nodes: FxHashMap<u64, Vec<Vec<u8>>> = FxHashMap::default();
|
|
208
351
|
let mut edges: EdgeIndex = EdgeIndex::default();
|
|
209
352
|
|
|
@@ -249,7 +392,8 @@ pub fn dedup_ndjson(
|
|
|
249
392
|
// Flush explicitly and propagate failure: relying on BufWriter's drop-time
|
|
250
393
|
// flush would swallow a final write error and report success with truncated
|
|
251
394
|
// output.
|
|
252
|
-
writer.flush().map_err(runtime_error)
|
|
395
|
+
writer.flush().map_err(runtime_error)?;
|
|
396
|
+
Ok((0, 0))
|
|
253
397
|
}
|
|
254
398
|
|
|
255
399
|
#[cfg(test)]
|
|
@@ -287,7 +431,7 @@ mod tests {
|
|
|
287
431
|
let output = dir.path().join("nodes.ndjson");
|
|
288
432
|
fs::write(&input, "{\"id\":\"A\"}\n{\"id\":\"B\"}\n{\"id\":\"A\"}\n").expect("write input");
|
|
289
433
|
|
|
290
|
-
dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
|
|
434
|
+
dedup_ndjson(input, output.clone(), false, None, None, None).expect("dedup nodes");
|
|
291
435
|
|
|
292
436
|
let lines: Vec<String> = fs::read_to_string(output)
|
|
293
437
|
.expect("read output")
|
|
@@ -311,7 +455,7 @@ mod tests {
|
|
|
311
455
|
)
|
|
312
456
|
.expect("write input");
|
|
313
457
|
|
|
314
|
-
dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
|
|
458
|
+
dedup_ndjson(input, output.clone(), false, None, None, None).expect("dedup nodes");
|
|
315
459
|
|
|
316
460
|
let lines: Vec<String> = fs::read_to_string(output)
|
|
317
461
|
.expect("read output")
|
|
@@ -338,6 +482,7 @@ mod tests {
|
|
|
338
482
|
true,
|
|
339
483
|
Some("TABLASSERT".to_string()),
|
|
340
484
|
None,
|
|
485
|
+
None,
|
|
341
486
|
)
|
|
342
487
|
.expect("dedup edges");
|
|
343
488
|
|
|
@@ -367,6 +512,7 @@ mod tests {
|
|
|
367
512
|
true,
|
|
368
513
|
Some("TABLASSERT".to_string()),
|
|
369
514
|
None,
|
|
515
|
+
None,
|
|
370
516
|
)
|
|
371
517
|
.expect("dedup edges");
|
|
372
518
|
|
|
@@ -392,7 +538,7 @@ mod tests {
|
|
|
392
538
|
)
|
|
393
539
|
.expect("write input");
|
|
394
540
|
|
|
395
|
-
dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
|
|
541
|
+
dedup_ndjson(input, output.clone(), false, None, None, None).expect("dedup nodes");
|
|
396
542
|
|
|
397
543
|
let line = fs::read_to_string(output).expect("read output");
|
|
398
544
|
let value: Value = serde_json::from_str(line.trim()).expect("json");
|
|
@@ -407,7 +553,7 @@ mod tests {
|
|
|
407
553
|
let output = dir.path().join("nodes.ndjson");
|
|
408
554
|
fs::write(&input, "{}\n{\"drop\":\"NA\"}\n").expect("write input");
|
|
409
555
|
|
|
410
|
-
dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
|
|
556
|
+
dedup_ndjson(input, output.clone(), false, None, None, None).expect("dedup nodes");
|
|
411
557
|
|
|
412
558
|
assert_eq!(fs::read_to_string(output).expect("read output"), "");
|
|
413
559
|
}
|
|
@@ -419,7 +565,7 @@ mod tests {
|
|
|
419
565
|
let output = dir.path().join("nodes.ndjson");
|
|
420
566
|
fs::write(&input, "\n \n{\"id\":\"A\"}\n\n{\"id\":\"A\"}\n \n").expect("write input");
|
|
421
567
|
|
|
422
|
-
dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
|
|
568
|
+
dedup_ndjson(input, output.clone(), false, None, None, None).expect("dedup nodes");
|
|
423
569
|
|
|
424
570
|
assert_eq!(
|
|
425
571
|
fs::read_to_string(output).expect("read output"),
|
|
@@ -463,14 +609,22 @@ mod tests {
|
|
|
463
609
|
dir.path(),
|
|
464
610
|
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
|
|
465
611
|
);
|
|
466
|
-
dedup_ndjson(
|
|
612
|
+
dedup_ndjson(
|
|
613
|
+
before_in,
|
|
614
|
+
before_out.clone(),
|
|
615
|
+
true,
|
|
616
|
+
None,
|
|
617
|
+
fields.clone(),
|
|
618
|
+
None,
|
|
619
|
+
)
|
|
620
|
+
.expect("dedup");
|
|
467
621
|
|
|
468
622
|
let after_dir = tempdir().expect("tempdir");
|
|
469
623
|
let (after_in, after_out) = write_edges(
|
|
470
624
|
after_dir.path(),
|
|
471
625
|
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\",\"effect_size\":1.5}\n",
|
|
472
626
|
);
|
|
473
|
-
dedup_ndjson(after_in, after_out.clone(), true, None, fields).expect("dedup");
|
|
627
|
+
dedup_ndjson(after_in, after_out.clone(), true, None, fields, None).expect("dedup");
|
|
474
628
|
|
|
475
629
|
assert_eq!(edge_ids(&before_out), edge_ids(&after_out));
|
|
476
630
|
}
|
|
@@ -484,14 +638,14 @@ mod tests {
|
|
|
484
638
|
dir.path(),
|
|
485
639
|
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
|
|
486
640
|
);
|
|
487
|
-
dedup_ndjson(before_in, before_out.clone(), true, None, None).expect("dedup");
|
|
641
|
+
dedup_ndjson(before_in, before_out.clone(), true, None, None, None).expect("dedup");
|
|
488
642
|
|
|
489
643
|
let after_dir = tempdir().expect("tempdir");
|
|
490
644
|
let (after_in, after_out) = write_edges(
|
|
491
645
|
after_dir.path(),
|
|
492
646
|
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n",
|
|
493
647
|
);
|
|
494
|
-
dedup_ndjson(after_in, after_out.clone(), true, None, None).expect("dedup");
|
|
648
|
+
dedup_ndjson(after_in, after_out.clone(), true, None, None, None).expect("dedup");
|
|
495
649
|
|
|
496
650
|
assert_ne!(edge_ids(&before_out), edge_ids(&after_out));
|
|
497
651
|
}
|
|
@@ -514,7 +668,7 @@ mod tests {
|
|
|
514
668
|
"predicate".to_string(),
|
|
515
669
|
"object".to_string(),
|
|
516
670
|
]);
|
|
517
|
-
let error = dedup_ndjson(input, output, true, None, fields).expect_err("not a key");
|
|
671
|
+
let error = dedup_ndjson(input, output, true, None, fields, None).expect_err("not a key");
|
|
518
672
|
let message = error.to_string();
|
|
519
673
|
assert!(message.contains("uuid-fields-not-a-key"), "{message}");
|
|
520
674
|
assert!(message.contains("p_value"), "{message}");
|
|
@@ -534,7 +688,7 @@ mod tests {
|
|
|
534
688
|
),
|
|
535
689
|
);
|
|
536
690
|
let fields = Some(vec!["subject".to_string(), "object".to_string()]);
|
|
537
|
-
dedup_ndjson(input, output.clone(), true, None, fields).expect("dedup edges");
|
|
691
|
+
dedup_ndjson(input, output.clone(), true, None, fields, None).expect("dedup edges");
|
|
538
692
|
assert_eq!(edge_ids(&output).len(), 1);
|
|
539
693
|
}
|
|
540
694
|
|
|
@@ -551,7 +705,7 @@ mod tests {
|
|
|
551
705
|
"{\"object\":\"B\",\"subject\":\"A\"}\n"
|
|
552
706
|
),
|
|
553
707
|
);
|
|
554
|
-
dedup_ndjson(input, output.clone(), true, None, None).expect("dedup edges");
|
|
708
|
+
dedup_ndjson(input, output.clone(), true, None, None, None).expect("dedup edges");
|
|
555
709
|
assert_eq!(edge_ids(&output).len(), 1);
|
|
556
710
|
}
|
|
557
711
|
|
|
@@ -568,6 +722,7 @@ mod tests {
|
|
|
568
722
|
true,
|
|
569
723
|
Some("infores:left".to_string()),
|
|
570
724
|
None,
|
|
725
|
+
None,
|
|
571
726
|
)
|
|
572
727
|
.expect("dedup");
|
|
573
728
|
|
|
@@ -580,6 +735,7 @@ mod tests {
|
|
|
580
735
|
true,
|
|
581
736
|
Some("infores:right".to_string()),
|
|
582
737
|
None,
|
|
738
|
+
None,
|
|
583
739
|
)
|
|
584
740
|
.expect("dedup");
|
|
585
741
|
|
|
@@ -596,4 +752,261 @@ mod tests {
|
|
|
596
752
|
vec!["effect_size", "p_value"]
|
|
597
753
|
);
|
|
598
754
|
}
|
|
755
|
+
|
|
756
|
+
fn merged_edge(output: &std::path::Path) -> Value {
|
|
757
|
+
let lines: Vec<String> = fs::read_to_string(output)
|
|
758
|
+
.expect("read output")
|
|
759
|
+
.lines()
|
|
760
|
+
.map(str::to_string)
|
|
761
|
+
.collect();
|
|
762
|
+
assert_eq!(lines.len(), 1, "expected one merged edge, got {lines:?}");
|
|
763
|
+
serde_json::from_str(&lines[0]).expect("json")
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
const SPO: &[&str] = &["subject", "predicate", "object"];
|
|
767
|
+
|
|
768
|
+
fn spo_fields() -> Option<Vec<String>> {
|
|
769
|
+
Some(SPO.iter().map(ToString::to_string).collect())
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
#[test]
|
|
773
|
+
fn merge_mode_folds_divergent_edges_into_one() {
|
|
774
|
+
// WHY: `uuid_on_collision: merge`. Two rows whose raw mention spellings resolve to
|
|
775
|
+
// the same CURIE derive one id; merge mode unions their evidence into a single
|
|
776
|
+
// edge instead of aborting with `uuid-fields-not-a-key`.
|
|
777
|
+
let dir = tempdir().expect("tempdir");
|
|
778
|
+
let (input, output) = write_edges(
|
|
779
|
+
dir.path(),
|
|
780
|
+
concat!(
|
|
781
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\",\"publications\":[\"PMID:2\",\"PMID:1\"]}\n",
|
|
782
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\",\"publications\":[\"PMID:3\",\"PMID:1\"]}\n"
|
|
783
|
+
),
|
|
784
|
+
);
|
|
785
|
+
let (merged, conflicts) = dedup_ndjson(
|
|
786
|
+
input,
|
|
787
|
+
output.clone(),
|
|
788
|
+
true,
|
|
789
|
+
None,
|
|
790
|
+
spo_fields(),
|
|
791
|
+
Some("merge".to_string()),
|
|
792
|
+
)
|
|
793
|
+
.expect("merge dedup");
|
|
794
|
+
|
|
795
|
+
assert_eq!((merged, conflicts), (1, 1));
|
|
796
|
+
let edge = merged_edge(&output);
|
|
797
|
+
// List fields union, dedup, and sort; scalar conflicts keep the first value.
|
|
798
|
+
assert_eq!(
|
|
799
|
+
edge["publications"],
|
|
800
|
+
serde_json::json!(["PMID:1", "PMID:2", "PMID:3"])
|
|
801
|
+
);
|
|
802
|
+
assert_eq!(edge["p_value"], serde_json::json!("0.01"));
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
#[test]
|
|
806
|
+
fn merge_mode_suppresses_exact_repeats_without_remerging() {
|
|
807
|
+
// WHY: an exact repeat is a Duplicate even when it repeats a DIVERGENT record
|
|
808
|
+
// already folded into the merged edge -- re-merging it would leave the record
|
|
809
|
+
// unchanged but double-count the scalar conflict in the summary.
|
|
810
|
+
let dir = tempdir().expect("tempdir");
|
|
811
|
+
let (input, output) = write_edges(
|
|
812
|
+
dir.path(),
|
|
813
|
+
concat!(
|
|
814
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
|
|
815
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n",
|
|
816
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n"
|
|
817
|
+
),
|
|
818
|
+
);
|
|
819
|
+
let (merged, conflicts) = dedup_ndjson(
|
|
820
|
+
input,
|
|
821
|
+
output.clone(),
|
|
822
|
+
true,
|
|
823
|
+
None,
|
|
824
|
+
spo_fields(),
|
|
825
|
+
Some("merge".to_string()),
|
|
826
|
+
)
|
|
827
|
+
.expect("merge dedup");
|
|
828
|
+
|
|
829
|
+
assert_eq!((merged, conflicts), (1, 1));
|
|
830
|
+
assert_eq!(merged_edge(&output)["p_value"], serde_json::json!("0.01"));
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
#[test]
|
|
834
|
+
fn merge_mode_output_is_order_independent() {
|
|
835
|
+
// WHY: merged output must not depend on which row the source happened to emit
|
|
836
|
+
// first, or two builds of one graph would diverge. List unions sort by canonical
|
|
837
|
+
// bytes, so both arrival orders produce byte-identical output.
|
|
838
|
+
let left_dir = tempdir().expect("tempdir");
|
|
839
|
+
let (left_in, left_out) = write_edges(
|
|
840
|
+
left_dir.path(),
|
|
841
|
+
concat!(
|
|
842
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:2\"]}\n",
|
|
843
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:1\",\"PMID:3\"]}\n"
|
|
844
|
+
),
|
|
845
|
+
);
|
|
846
|
+
dedup_ndjson(
|
|
847
|
+
left_in,
|
|
848
|
+
left_out.clone(),
|
|
849
|
+
true,
|
|
850
|
+
None,
|
|
851
|
+
spo_fields(),
|
|
852
|
+
Some("merge".to_string()),
|
|
853
|
+
)
|
|
854
|
+
.expect("merge dedup");
|
|
855
|
+
|
|
856
|
+
let right_dir = tempdir().expect("tempdir");
|
|
857
|
+
let (right_in, right_out) = write_edges(
|
|
858
|
+
right_dir.path(),
|
|
859
|
+
concat!(
|
|
860
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:1\",\"PMID:3\"]}\n",
|
|
861
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:2\"]}\n"
|
|
862
|
+
),
|
|
863
|
+
);
|
|
864
|
+
dedup_ndjson(
|
|
865
|
+
right_in,
|
|
866
|
+
right_out.clone(),
|
|
867
|
+
true,
|
|
868
|
+
None,
|
|
869
|
+
spo_fields(),
|
|
870
|
+
Some("merge".to_string()),
|
|
871
|
+
)
|
|
872
|
+
.expect("merge dedup");
|
|
873
|
+
|
|
874
|
+
assert_eq!(
|
|
875
|
+
fs::read_to_string(left_out).expect("read left"),
|
|
876
|
+
fs::read_to_string(right_out).expect("read right")
|
|
877
|
+
);
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
#[test]
|
|
881
|
+
fn merge_mode_writes_edges_in_first_seen_order() {
|
|
882
|
+
// WHY: buffering must not reorder the graph -- the first record to claim each id
|
|
883
|
+
// fixes its position in the output.
|
|
884
|
+
let dir = tempdir().expect("tempdir");
|
|
885
|
+
let (input, output) = write_edges(
|
|
886
|
+
dir.path(),
|
|
887
|
+
concat!(
|
|
888
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:1\"]}\n",
|
|
889
|
+
"{\"subject\":\"X\",\"predicate\":\"r\",\"object\":\"Y\"}\n",
|
|
890
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"publications\":[\"PMID:2\"]}\n"
|
|
891
|
+
),
|
|
892
|
+
);
|
|
893
|
+
dedup_ndjson(
|
|
894
|
+
input,
|
|
895
|
+
output.clone(),
|
|
896
|
+
true,
|
|
897
|
+
None,
|
|
898
|
+
spo_fields(),
|
|
899
|
+
Some("merge".to_string()),
|
|
900
|
+
)
|
|
901
|
+
.expect("merge dedup");
|
|
902
|
+
|
|
903
|
+
let lines: Vec<Value> = fs::read_to_string(output)
|
|
904
|
+
.expect("read output")
|
|
905
|
+
.lines()
|
|
906
|
+
.map(|line| serde_json::from_str(line).expect("json"))
|
|
907
|
+
.collect();
|
|
908
|
+
assert_eq!(lines.len(), 2);
|
|
909
|
+
assert_eq!(lines[0]["subject"], serde_json::json!("A"));
|
|
910
|
+
assert_eq!(
|
|
911
|
+
lines[0]["publications"],
|
|
912
|
+
serde_json::json!(["PMID:1", "PMID:2"])
|
|
913
|
+
);
|
|
914
|
+
assert_eq!(lines[1]["subject"], serde_json::json!("X"));
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
#[test]
|
|
918
|
+
fn merge_mode_dedups_list_objects_by_canonical_bytes() {
|
|
919
|
+
// WHY: `sources` entries are objects; two entries that differ only in key order
|
|
920
|
+
// are the same source and must collapse, and the union must sort so merged output
|
|
921
|
+
// is deterministic.
|
|
922
|
+
let dir = tempdir().expect("tempdir");
|
|
923
|
+
let (input, output) = write_edges(
|
|
924
|
+
dir.path(),
|
|
925
|
+
concat!(
|
|
926
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"sources\":[{\"resource_id\":\"infores:b\",\"resource_role\":\"primary_knowledge_source\"}]}\n",
|
|
927
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"sources\":[{\"resource_role\":\"primary_knowledge_source\",\"resource_id\":\"infores:b\"},{\"resource_id\":\"infores:a\",\"resource_role\":\"aggregator_knowledge_source\"}]}\n"
|
|
928
|
+
),
|
|
929
|
+
);
|
|
930
|
+
dedup_ndjson(
|
|
931
|
+
input,
|
|
932
|
+
output.clone(),
|
|
933
|
+
true,
|
|
934
|
+
None,
|
|
935
|
+
spo_fields(),
|
|
936
|
+
Some("merge".to_string()),
|
|
937
|
+
)
|
|
938
|
+
.expect("merge dedup");
|
|
939
|
+
|
|
940
|
+
let edge = merged_edge(&output);
|
|
941
|
+
let sources = edge["sources"].as_array().expect("sources array");
|
|
942
|
+
assert_eq!(
|
|
943
|
+
sources.len(),
|
|
944
|
+
2,
|
|
945
|
+
"key order alone must not keep both entries"
|
|
946
|
+
);
|
|
947
|
+
assert_eq!(sources[0]["resource_id"], serde_json::json!("infores:a"));
|
|
948
|
+
assert_eq!(sources[1]["resource_id"], serde_json::json!("infores:b"));
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
#[test]
|
|
952
|
+
fn merge_mode_keeps_fields_only_the_second_record_carries() {
|
|
953
|
+
// WHY: first-wins arbitrates CONFLICTS; a field absent from the first record is
|
|
954
|
+
// not a conflict, it is extra evidence, and dropping it would lose data.
|
|
955
|
+
let dir = tempdir().expect("tempdir");
|
|
956
|
+
let (input, output) = write_edges(
|
|
957
|
+
dir.path(),
|
|
958
|
+
concat!(
|
|
959
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\"}\n",
|
|
960
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"effect_size\":1.5}\n"
|
|
961
|
+
),
|
|
962
|
+
);
|
|
963
|
+
let (merged, conflicts) = dedup_ndjson(
|
|
964
|
+
input,
|
|
965
|
+
output.clone(),
|
|
966
|
+
true,
|
|
967
|
+
None,
|
|
968
|
+
spo_fields(),
|
|
969
|
+
Some("merge".to_string()),
|
|
970
|
+
)
|
|
971
|
+
.expect("merge dedup");
|
|
972
|
+
|
|
973
|
+
assert_eq!((merged, conflicts), (1, 0));
|
|
974
|
+
assert_eq!(merged_edge(&output)["effect_size"], serde_json::json!(1.5));
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
#[test]
|
|
978
|
+
fn explicit_error_mode_still_aborts_on_divergent_edges() {
|
|
979
|
+
// WHY: `uuid_on_collision` defaults to `error`; naming it explicitly must behave
|
|
980
|
+
// exactly like the default.
|
|
981
|
+
let dir = tempdir().expect("tempdir");
|
|
982
|
+
let (input, output) = write_edges(
|
|
983
|
+
dir.path(),
|
|
984
|
+
concat!(
|
|
985
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
|
|
986
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n"
|
|
987
|
+
),
|
|
988
|
+
);
|
|
989
|
+
let error = dedup_ndjson(
|
|
990
|
+
input,
|
|
991
|
+
output,
|
|
992
|
+
true,
|
|
993
|
+
None,
|
|
994
|
+
spo_fields(),
|
|
995
|
+
Some("error".to_string()),
|
|
996
|
+
)
|
|
997
|
+
.expect_err("not a key");
|
|
998
|
+
assert!(
|
|
999
|
+
error.to_string().contains("uuid-fields-not-a-key"),
|
|
1000
|
+
"{error}"
|
|
1001
|
+
);
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
#[test]
|
|
1005
|
+
fn unknown_on_collision_is_rejected() {
|
|
1006
|
+
let dir = tempdir().expect("tempdir");
|
|
1007
|
+
let (input, output) = write_edges(dir.path(), "{\"subject\":\"A\",\"object\":\"B\"}\n");
|
|
1008
|
+
let error = dedup_ndjson(input, output, true, None, None, Some("bogus".to_string()))
|
|
1009
|
+
.expect_err("unknown mode");
|
|
1010
|
+
assert!(error.to_string().contains("bogus"), "{error}");
|
|
1011
|
+
}
|
|
599
1012
|
}
|
|
@@ -1578,7 +1578,7 @@ def upstream_resource_ids(repo: Repositories) -> list[str]:
|
|
|
1578
1578
|
return [InformationResources.PUBMED.value]
|
|
1579
1579
|
|
|
1580
1580
|
|
|
1581
|
-
def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fields: list[str] | None = None) -> None:
|
|
1581
|
+
def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fields: list[str] | None = None, on_collision: str = "error") -> None:
|
|
1582
1582
|
"""Remove null values from and deduplicate an NDJSON stream.
|
|
1583
1583
|
|
|
1584
1584
|
Args:
|
|
@@ -1588,12 +1588,17 @@ def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fi
|
|
|
1588
1588
|
Ignored for nodes, whose ids are CURIEs rather than derived hashes.
|
|
1589
1589
|
uuid_fields: Optional edge fields that constitute edge identity
|
|
1590
1590
|
(``Graph.uuid_fields``). ``None`` hashes the whole record.
|
|
1591
|
+
on_collision: ``error`` aborts on two different edges deriving one id;
|
|
1592
|
+
``merge`` folds them into one edge (``Graph.uuid_on_collision``).
|
|
1591
1593
|
|
|
1592
1594
|
Notes:
|
|
1593
1595
|
Also adds UUIDs to edges. Edges deduplicate on their derived id, so the
|
|
1594
1596
|
output can never carry the same id twice: an exact repeat is collapsed,
|
|
1595
1597
|
while two genuinely different edges deriving one id abort the build with
|
|
1596
|
-
``uuid-fields-not-a-key`` rather than shipping a duplicate
|
|
1598
|
+
``uuid-fields-not-a-key`` rather than shipping a duplicate -- unless
|
|
1599
|
+
``on_collision="merge"``, which unions their list fields (sorted, so the
|
|
1600
|
+
output is merge-order independent), keeps the first value of conflicting
|
|
1601
|
+
scalars, and buffers one full record per unique id until end-of-stream.
|
|
1597
1602
|
|
|
1598
1603
|
Returns:
|
|
1599
1604
|
``None``; writes the deduplicated stream alongside ``p_in`` with no
|
|
@@ -1604,7 +1609,13 @@ def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fi
|
|
|
1604
1609
|
if p_out.is_file():
|
|
1605
1610
|
p_out.unlink()
|
|
1606
1611
|
|
|
1607
|
-
rs.dedup_ndjson(p_in, p_out, is_edges, domain, uuid_fields)
|
|
1612
|
+
merged, scalar_conflicts = rs.dedup_ndjson(p_in, p_out, is_edges, domain, uuid_fields, on_collision)
|
|
1613
|
+
if merged:
|
|
1614
|
+
logger.warning(
|
|
1615
|
+
"uuid_on_collision=merge folded {merged} divergent edge records into their first-seen edges; {conflicts} conflicting scalar fields kept the first value",
|
|
1616
|
+
merged=merged,
|
|
1617
|
+
conflicts=scalar_conflicts,
|
|
1618
|
+
)
|
|
1608
1619
|
|
|
1609
1620
|
p_in.unlink()
|
|
1610
1621
|
|
|
@@ -1762,6 +1773,7 @@ def _write_ndjson(
|
|
|
1762
1773
|
on_phase: Callable[[str], None] | None = None,
|
|
1763
1774
|
domain: str = "TABLASSERT",
|
|
1764
1775
|
uuid_fields: list[str] | None = None,
|
|
1776
|
+
on_collision: str = "error",
|
|
1765
1777
|
) -> None:
|
|
1766
1778
|
"""Write, dedup, and RIG the KGX NDJSON outputs.
|
|
1767
1779
|
|
|
@@ -1790,6 +1802,8 @@ def _write_ndjson(
|
|
|
1790
1802
|
boundary, used to drive progress UX.
|
|
1791
1803
|
domain: UUID namespace for derived edge ids (``Graph.uuid_namespace``).
|
|
1792
1804
|
uuid_fields: Optional edge identity fields (``Graph.uuid_fields``).
|
|
1805
|
+
on_collision: Collision policy for divergent same-id edges
|
|
1806
|
+
(``Graph.uuid_on_collision``).
|
|
1793
1807
|
"""
|
|
1794
1808
|
# Phase: write-nodes. Collection point: appending to output files.
|
|
1795
1809
|
if on_phase is not None:
|
|
@@ -1810,7 +1824,7 @@ def _write_ndjson(
|
|
|
1810
1824
|
# Phase: dedup.
|
|
1811
1825
|
if on_phase is not None:
|
|
1812
1826
|
on_phase("dedup")
|
|
1813
|
-
dedup_stream(edges_tmp, is_edges=True, domain=domain, uuid_fields=uuid_fields)
|
|
1827
|
+
dedup_stream(edges_tmp, is_edges=True, domain=domain, uuid_fields=uuid_fields, on_collision=on_collision)
|
|
1814
1828
|
# The deduper hashes the record WITH the literal `{edge_id}` placeholder still
|
|
1815
1829
|
# in place, so edge ids stay deterministic regardless of this resolution pass.
|
|
1816
1830
|
_resolve_edge_id_placeholders(edges_tmp.with_suffix(""))
|
|
@@ -1834,6 +1848,7 @@ def compile_graph(
|
|
|
1834
1848
|
no_original: bool = False,
|
|
1835
1849
|
uuid_fields: list[str] | None = None,
|
|
1836
1850
|
uuid_domain: str | None = None,
|
|
1851
|
+
uuid_on_collision: str = "error",
|
|
1837
1852
|
) -> None:
|
|
1838
1853
|
"""Aggregate subgraph parquets for NDJSON KGX export using a lazy scan.
|
|
1839
1854
|
|
|
@@ -1861,6 +1876,8 @@ def compile_graph(
|
|
|
1861
1876
|
infores when ``uuid_fields`` is set -- narrowing the hash inputs
|
|
1862
1877
|
removes the accidental cross-graph uniqueness that full-record hashing
|
|
1863
1878
|
provided -- and to ``TABLASSERT`` otherwise.
|
|
1879
|
+
uuid_on_collision: ``error`` aborts when two different edges derive one id;
|
|
1880
|
+
``merge`` folds them into one edge instead (``Graph.uuid_on_collision``).
|
|
1864
1881
|
|
|
1865
1882
|
Returns:
|
|
1866
1883
|
``None``; writes ``<name>_<version>.nodes.ndjson``,
|
|
@@ -1871,7 +1888,8 @@ def compile_graph(
|
|
|
1871
1888
|
TablassertError: With code ``rig-validation-failed`` when the
|
|
1872
1889
|
generated RIG fails its built-in audit (nothing is then written).
|
|
1873
1890
|
RuntimeError: Tagged ``uuid-fields-not-a-key`` when two genuinely
|
|
1874
|
-
different edges derive one id under the declared ``uuid_fields
|
|
1891
|
+
different edges derive one id under the declared ``uuid_fields``
|
|
1892
|
+
and ``uuid_on_collision`` is ``error``.
|
|
1875
1893
|
"""
|
|
1876
1894
|
rig_cfg: RIGConfig = rig if isinstance(rig, RIGConfig) else RIGConfig.model_validate(rig)
|
|
1877
1895
|
out_dir: Path = Path(rig_cfg.artifact_base_path)
|
|
@@ -1891,7 +1909,7 @@ def compile_graph(
|
|
|
1891
1909
|
subnodes: list[pl.LazyFrame]
|
|
1892
1910
|
subedges: list[pl.LazyFrame]
|
|
1893
1911
|
subnodes, subedges = _collect_subframes(subgraphs, on_phase, on_subgraph, rig_cfg.source_info.infores_id, no_original)
|
|
1894
|
-
_write_ndjson(subnodes, subedges, n, e, name, version, rig_cfg, section_sources, on_phase, domain, uuid_fields)
|
|
1912
|
+
_write_ndjson(subnodes, subedges, n, e, name, version, rig_cfg, section_sources, on_phase, domain, uuid_fields, uuid_on_collision)
|
|
1895
1913
|
|
|
1896
1914
|
|
|
1897
1915
|
def resolve_many(
|
|
@@ -1139,6 +1139,10 @@ class Graph(TablaBase):
|
|
|
1139
1139
|
description="Explicit UUID namespace. Defaults to `rig.source_info.infores_id` when `uuid_fields` is set, `TABLASSERT` otherwise. Set it only when graphs must deliberately share an id space.",
|
|
1140
1140
|
examples=["infores:multiomicskg"],
|
|
1141
1141
|
)
|
|
1142
|
+
uuid_on_collision: Literal["error", "merge"] = Field(
|
|
1143
|
+
default="error",
|
|
1144
|
+
description="What to do when two different edges derive one id under `uuid_fields`: `error` aborts the build with `uuid-fields-not-a-key`; `merge` folds the records into one edge (list fields unioned and sorted, scalars first-wins). Requires `uuid_fields`, and buffers one full record per unique id, which is why it is opt-in.",
|
|
1145
|
+
)
|
|
1142
1146
|
|
|
1143
1147
|
@model_validator(mode="after")
|
|
1144
1148
|
def validate_uuid_fields(self: Self) -> Self:
|
|
@@ -1147,9 +1151,16 @@ class Graph(TablaBase):
|
|
|
1147
1151
|
Every entry must be a real emittable edge field, or the id would silently derive
|
|
1148
1152
|
from nothing and every edge in the graph would collide. `id` itself is rejected
|
|
1149
1153
|
because it is the value being derived. Casing follows `Annotation.clean_annotation`
|
|
1150
|
-
so `Subject` and `subject` both work and mixed-case Biolink slots survive.
|
|
1154
|
+
so `Subject` and `subject` both work and mixed-case Biolink slots survive. Also
|
|
1155
|
+
rejects `uuid_on_collision: merge` without `uuid_fields`: under the whole-record
|
|
1156
|
+
hash no two different records can share an id, so there is nothing to merge.
|
|
1151
1157
|
"""
|
|
1152
1158
|
if self.uuid_fields is None:
|
|
1159
|
+
if self.uuid_on_collision == "merge":
|
|
1160
|
+
raise TablassertValidationError(
|
|
1161
|
+
"`uuid_on_collision: merge` requires `uuid_fields`. With the whole-record hash every field is an identity field, so two different records can never derive one id and there is nothing to merge.",
|
|
1162
|
+
code="uuid-merge-without-fields",
|
|
1163
|
+
)
|
|
1153
1164
|
return self
|
|
1154
1165
|
if not self.uuid_fields:
|
|
1155
1166
|
raise TablassertValidationError(
|
|
@@ -7,7 +7,9 @@ from typing import Any
|
|
|
7
7
|
def build_fullmap_db(
|
|
8
8
|
output: Path, classes: list[Path], synonyms: list[Path], threads: int | None = None, progress: Callable[[int, int, int, str], None] | None = None
|
|
9
9
|
) -> None: ...
|
|
10
|
-
def dedup_ndjson(
|
|
10
|
+
def dedup_ndjson(
|
|
11
|
+
input: Path, output: Path, is_edges: bool, domain: str | None = None, uuid_fields: list[str] | None = None, on_collision: str | None = None
|
|
12
|
+
) -> tuple[int, int]: ...
|
|
11
13
|
def extract_prebuilt_fullmap(archive: Path, output: Path, progress: Callable[[str], None] | None = None) -> None: ...
|
|
12
14
|
def fullmap_source_version() -> str: ...
|
|
13
15
|
def hydrate_categories(db: Path) -> list[str]: ...
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|