tablassert 15.0.0__tar.gz → 16.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {tablassert-15.0.0 → tablassert-16.0.0}/PKG-INFO +1 -1
  2. {tablassert-15.0.0 → tablassert-16.0.0}/pyproject.toml +1 -1
  3. {tablassert-15.0.0 → tablassert-16.0.0}/rust/src/json.rs +31 -1
  4. tablassert-16.0.0/rust/src/ndjson.rs +599 -0
  5. tablassert-16.0.0/rust/src/uuid.rs +345 -0
  6. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/biolink.py +50 -10
  7. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/cli.py +11 -1
  8. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/coerce.py +14 -0
  9. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/errors.py +2 -0
  10. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/ingests.py +14 -1
  11. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/lib.py +40 -8
  12. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/models.py +69 -0
  13. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/rig.py +7 -9
  14. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/rs.pyi +1 -1
  15. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/study.py +19 -2
  16. tablassert-15.0.0/rust/src/ndjson.rs +0 -254
  17. tablassert-15.0.0/rust/src/uuid.rs +0 -146
  18. {tablassert-15.0.0 → tablassert-16.0.0}/LICENSE +0 -0
  19. {tablassert-15.0.0 → tablassert-16.0.0}/README.md +0 -0
  20. {tablassert-15.0.0 → tablassert-16.0.0}/rust/Cargo.lock +0 -0
  21. {tablassert-15.0.0 → tablassert-16.0.0}/rust/Cargo.toml +0 -0
  22. {tablassert-15.0.0 → tablassert-16.0.0}/rust/examples/count_tables.rs +0 -0
  23. {tablassert-15.0.0 → tablassert-16.0.0}/rust/src/fullmap.rs +0 -0
  24. {tablassert-15.0.0 → tablassert-16.0.0}/rust/src/lib.rs +0 -0
  25. {tablassert-15.0.0 → tablassert-16.0.0}/rust/tests/build_golden.rs +0 -0
  26. {tablassert-15.0.0 → tablassert-16.0.0}/rust/tests/common/mod.rs +0 -0
  27. {tablassert-15.0.0 → tablassert-16.0.0}/rust/tests/extract_prebuilt.rs +0 -0
  28. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/__init__.py +0 -0
  29. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/_lazy.py +0 -0
  30. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/agent.py +0 -0
  31. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/enums.py +0 -0
  32. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/extras.py +0 -0
  33. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/fullmap.py +0 -0
  34. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/graph_target.py +0 -0
  35. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/log.py +0 -0
  36. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/nlp.py +0 -0
  37. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/progress.py +0 -0
  38. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/qc.py +0 -0
  39. {tablassert-15.0.0 → tablassert-16.0.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 15.0.0
3
+ Version: 16.0.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "15.0.0"
3
+ version = "16.0.0"
4
4
  description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -65,10 +65,40 @@ pub fn strip_nulls(value: &Value) -> Value {
65
65
  }
66
66
  }
67
67
 
68
- pub fn stable_json_bytes(value: &Value) -> serde_json::Result<Vec<u8>> {
68
+ /// Serialize in the record's own key order. This is the EMITTED form -- what actually
69
+ /// gets written to the NDJSON -- so it must not reorder anything. It is deliberately not
70
+ /// a canonical form: use `canonical_json_bytes` when comparing two records for equality.
71
+ pub fn emitted_json_bytes(value: &Value) -> serde_json::Result<Vec<u8>> {
69
72
  serde_json::to_vec(value)
70
73
  }
71
74
 
75
+ /// Serialize with every object's keys sorted, recursively.
76
+ ///
77
+ /// `serde_json` is built with `preserve_order`, so plain `to_vec` leaks insertion order:
78
+ /// two logically identical records that arrived with different key order produce
79
+ /// different bytes. The edge deduper compares records for equality, so it needs a form
80
+ /// where "same content" means "same bytes"; array order is preserved because it is
81
+ /// semantic.
82
+ pub fn canonical_json_bytes(value: &Value) -> serde_json::Result<Vec<u8>> {
83
+ serde_json::to_vec(&canonical_value(value))
84
+ }
85
+
86
+ fn canonical_value(value: &Value) -> Value {
87
+ match value {
88
+ Value::Object(entries) => {
89
+ let mut keys: Vec<&String> = entries.keys().collect();
90
+ keys.sort_unstable();
91
+ let mut sorted: Map<String, Value> = Map::with_capacity(entries.len());
92
+ for key in keys {
93
+ sorted.insert(key.clone(), canonical_value(&entries[key]));
94
+ }
95
+ Value::Object(sorted)
96
+ }
97
+ Value::Array(items) => Value::Array(items.iter().map(canonical_value).collect()),
98
+ _ => value.clone(),
99
+ }
100
+ }
101
+
72
102
  #[cfg(test)]
73
103
  mod tests {
74
104
  use super::strip_nulls;
@@ -0,0 +1,599 @@
1
+ use crate::json::{canonical_json_bytes, emitted_json_bytes, strip_nulls};
2
+ use crate::uuid::uuid_for_json_object;
3
+ use pyo3::exceptions::PyRuntimeError;
4
+ use pyo3::prelude::*;
5
+ use rustc_hash::FxHashMap;
6
+ use serde_json::Value;
7
+ use std::fs::File;
8
+ use std::io::{BufRead, BufReader, BufWriter, Write};
9
+ use std::path::{Path, PathBuf};
10
+ use uuid::Uuid;
11
+ use xxhash_rust::xxh64::xxh64;
12
+
13
+ fn runtime_error(error: impl ToString) -> PyErr {
14
+ PyRuntimeError::new_err(error.to_string())
15
+ }
16
+
17
+ /// Collision-safe dedup decision for the NODES stream. Records are bucketed by xxh64
18
+ /// but a record is suppressed ONLY when its canonical bytes exactly match an existing
19
+ /// entry in the bucket, so two DISTINCT records whose xxh64 collides both survive.
20
+ /// (The former `HashSet<u64>` keyed on the hash alone silently dropped the second record
21
+ /// on a collision — data loss at scale.) Returns true when `bytes` is new and recorded.
22
+ ///
23
+ /// Edges do NOT use this path: their `id` is a derived hash, so they dedup on the id
24
+ /// itself (see `EdgeIndex`), which both guarantees id uniqueness and keeps 16 bytes per
25
+ /// edge instead of the whole record.
26
+ fn record_if_new(seen: &mut FxHashMap<u64, Vec<Vec<u8>>>, bytes: &[u8]) -> bool {
27
+ let bucket = seen.entry(xxh64(bytes, 0)).or_default();
28
+ if bucket.iter().any(|existing| existing.as_slice() == bytes) {
29
+ false
30
+ } else {
31
+ bucket.push(bytes.to_vec());
32
+ true
33
+ }
34
+ }
35
+
36
+ fn label_edge(mut value: Value, domain: &str, fields: Option<&[String]>) -> PyResult<Value> {
37
+ let id: String = uuid_for_json_object(domain, &value, fields)
38
+ .ok_or_else(|| runtime_error("expected JSON object"))?;
39
+ value
40
+ .as_object_mut()
41
+ .ok_or_else(|| runtime_error("expected JSON object"))?
42
+ .insert("id".to_string(), Value::String(id));
43
+ Ok(value)
44
+ }
45
+
46
+ /// A record ready to write, plus (for edges) the content hash of the record as it stood
47
+ /// BEFORE its `id` was inserted.
48
+ ///
49
+ /// Hashing pre-insertion matters twice over: the id is a pure function of the other
50
+ /// fields, so a post-insertion hash would agree whenever the ids agree -- exactly the
51
+ /// divergence this is meant to detect -- and it avoids cloning every record just to strip
52
+ /// one key back out.
53
+ struct Finalized {
54
+ value: Value,
55
+ content: u64,
56
+ }
57
+
58
+ fn finalize_record(
59
+ value: Value,
60
+ is_edges: bool,
61
+ domain: &str,
62
+ fields: Option<&[String]>,
63
+ ) -> PyResult<Option<Finalized>> {
64
+ let cleaned: Value = strip_nulls(&value);
65
+ let is_empty_object: bool = matches!(&cleaned, Value::Object(map) if map.is_empty());
66
+ if is_empty_object {
67
+ return Ok(None);
68
+ }
69
+ if !is_edges {
70
+ return Ok(Some(Finalized {
71
+ value: cleaned,
72
+ content: 0,
73
+ }));
74
+ }
75
+ let content: u64 = xxh64(&canonical_json_bytes(&cleaned).map_err(runtime_error)?, 0);
76
+ label_edge(cleaned, domain, fields).map(|value| Some(Finalized { value, content }))
77
+ }
78
+
79
+ /// Edge dedup state: derived id -> hash of that edge's canonical, id-free content.
80
+ ///
81
+ /// Keying on the id (not on the whole record, as the nodes path does) is what makes
82
+ /// `uuid_fields` safe: once the hash covers only a subset of the record, two records can
83
+ /// share an id while differing in bytes, and byte-keyed dedup would happily emit both --
84
+ /// duplicate edge ids in the output. It is also ~20-25x cheaper: 24 bytes per edge
85
+ /// instead of a full copy of every record (~800 bytes each at the 10M-edge scale).
86
+ #[derive(Default)]
87
+ struct EdgeIndex {
88
+ seen: FxHashMap<[u8; 16], u64>,
89
+ }
90
+
91
+ /// What to do with an edge whose id has been seen before.
92
+ enum EdgeVerdict {
93
+ /// First sighting of this id -- write it.
94
+ Fresh,
95
+ /// Byte-identical to the record that already claimed this id -- suppress it.
96
+ Duplicate,
97
+ /// A DIFFERENT record claims this id: the declared `uuid_fields` are not a key.
98
+ Divergent,
99
+ }
100
+
101
+ impl EdgeIndex {
102
+ fn classify(&mut self, id: [u8; 16], content: u64) -> EdgeVerdict {
103
+ match self.seen.get(&id) {
104
+ None => {
105
+ self.seen.insert(id, content);
106
+ EdgeVerdict::Fresh
107
+ }
108
+ Some(existing) if *existing == content => EdgeVerdict::Duplicate,
109
+ Some(_) => EdgeVerdict::Divergent,
110
+ }
111
+ }
112
+ }
113
+
114
+ fn edge_id_bytes(value: &Value) -> PyResult<[u8; 16]> {
115
+ let id: &str = value
116
+ .get("id")
117
+ .and_then(Value::as_str)
118
+ .ok_or_else(|| runtime_error("labeled edge is missing its id"))?;
119
+ Uuid::parse_str(id)
120
+ .map(Uuid::into_bytes)
121
+ .map_err(|error| runtime_error(format!("edge id {id} is not a UUID: {error}")))
122
+ }
123
+
124
+ /// Recover the record that first claimed `id` so the error can name what actually
125
+ /// differs. Runs ONLY on the failure path -- the build is about to abort, so re-reading
126
+ /// the partial output costs nothing in the happy case.
127
+ fn find_written_edge(output: &Path, id: &str) -> Option<Value> {
128
+ let reader = BufReader::new(File::open(output).ok()?);
129
+ reader
130
+ .lines()
131
+ .map_while(Result::ok)
132
+ .filter(|line| line.contains(id))
133
+ .find_map(|line| {
134
+ let value: Value = serde_json::from_str(&line).ok()?;
135
+ (value.get("id").and_then(Value::as_str) == Some(id)).then_some(value)
136
+ })
137
+ }
138
+
139
+ /// Top-level keys whose values differ between two records.
140
+ fn differing_keys(left: &Value, right: &Value) -> Vec<String> {
141
+ let (Some(left), Some(right)) = (left.as_object(), right.as_object()) else {
142
+ return Vec::new();
143
+ };
144
+ let mut keys: Vec<String> = left
145
+ .keys()
146
+ .chain(right.keys())
147
+ .filter(|key| key.as_str() != "id")
148
+ .filter(|key| left.get(*key) != right.get(*key))
149
+ .cloned()
150
+ .collect();
151
+ keys.sort_unstable();
152
+ keys.dedup();
153
+ keys
154
+ }
155
+
156
+ /// Build the `uuid-fields-not-a-key` diagnostic.
157
+ ///
158
+ /// The author's declared `uuid_fields` do not uniquely identify an edge in this graph, so
159
+ /// two genuinely different records derived the same id. Emitting both would ship duplicate
160
+ /// ids; silently dropping one would lose data. Fail, and name the fields that would fix it.
161
+ fn not_a_key_error(output: &Path, id: &str, incoming: &Value, fields: Option<&[String]>) -> PyErr {
162
+ let declared: String = match fields {
163
+ Some(fields) => fields.join(", "),
164
+ None => "<all fields>".to_string(),
165
+ };
166
+ let differing: String = find_written_edge(output, id)
167
+ .map(|existing| differing_keys(&existing, incoming))
168
+ .filter(|keys| !keys.is_empty())
169
+ .map_or_else(
170
+ || " (could not recover the first record to diff)".to_string(),
171
+ |keys| format!(" they differ in: {}", keys.join(", ")),
172
+ );
173
+ let context: String = ["subject", "predicate", "object"]
174
+ .iter()
175
+ .filter_map(|key| {
176
+ incoming
177
+ .get(*key)
178
+ .and_then(Value::as_str)
179
+ .map(|value| format!("{key}={value}"))
180
+ })
181
+ .collect::<Vec<String>>()
182
+ .join(" ");
183
+ runtime_error(format!(
184
+ "uuid-fields-not-a-key: declared uuid_fields are not a key for this graph.\n\
185
+ \x20 id {id} is claimed by 2 different edges.\n\
186
+ {differing}\n\
187
+ \x20 declared uuid_fields: {declared}\n\
188
+ \x20 offending edge: {context}\n\
189
+ Add a discriminating field to `uuid_fields` (a qualifier, `has_supporting_studies` \
190
+ for the source row, or the statistic that actually differs)."
191
+ ))
192
+ }
193
+
194
+ #[pyfunction]
195
+ #[pyo3(signature = (input, output, is_edges, domain=None, uuid_fields=None))]
196
+ pub fn dedup_ndjson(
197
+ input: PathBuf,
198
+ output: PathBuf,
199
+ is_edges: bool,
200
+ domain: Option<String>,
201
+ uuid_fields: Option<Vec<String>>,
202
+ ) -> PyResult<()> {
203
+ let domain: String = domain.unwrap_or_else(|| "TABLASSERT".to_string());
204
+ let fields: Option<&[String]> = uuid_fields.as_deref();
205
+ let reader: BufReader<File> = BufReader::new(File::open(input).map_err(runtime_error)?);
206
+ let mut writer: BufWriter<File> = BufWriter::new(File::create(&output).map_err(runtime_error)?);
207
+ let mut nodes: FxHashMap<u64, Vec<Vec<u8>>> = FxHashMap::default();
208
+ let mut edges: EdgeIndex = EdgeIndex::default();
209
+
210
+ reader
211
+ .lines()
212
+ .map(|line| line.map_err(runtime_error))
213
+ .filter(|line| line.as_ref().map_or(true, |line| !line.trim().is_empty()))
214
+ .map(|line| {
215
+ line.and_then(|line| serde_json::from_str::<Value>(&line).map_err(runtime_error))
216
+ })
217
+ .map(|value| value.and_then(|value| finalize_record(value, is_edges, &domain, fields)))
218
+ .try_for_each(|record| -> PyResult<()> {
219
+ let Some(Finalized { value, content }) = record? else {
220
+ return Ok(());
221
+ };
222
+ let write: bool = if is_edges {
223
+ // Edges dedup on their derived id, so the output can never carry the same
224
+ // id twice: an exact repeat is suppressed and a genuine divergence aborts
225
+ // the build rather than shipping a duplicate.
226
+ match edges.classify(edge_id_bytes(&value)?, content) {
227
+ EdgeVerdict::Fresh => true,
228
+ EdgeVerdict::Duplicate => false,
229
+ EdgeVerdict::Divergent => {
230
+ writer.flush().map_err(runtime_error)?;
231
+ let id: &str = value["id"].as_str().unwrap_or_default();
232
+ return Err(not_a_key_error(&output, id, &value, fields));
233
+ }
234
+ }
235
+ } else {
236
+ record_if_new(
237
+ &mut nodes,
238
+ &emitted_json_bytes(&value).map_err(runtime_error)?,
239
+ )
240
+ };
241
+ if write {
242
+ writer
243
+ .write_all(&emitted_json_bytes(&value).map_err(runtime_error)?)
244
+ .map_err(runtime_error)?;
245
+ writer.write_all(b"\n").map_err(runtime_error)?;
246
+ }
247
+ Ok(())
248
+ })?;
249
+ // Flush explicitly and propagate failure: relying on BufWriter's drop-time
250
+ // flush would swallow a final write error and report success with truncated
251
+ // output.
252
+ writer.flush().map_err(runtime_error)
253
+ }
254
+
255
+ #[cfg(test)]
256
+ mod tests {
257
+ use super::{dedup_ndjson, differing_keys, record_if_new};
258
+ use rustc_hash::FxHashMap;
259
+ use serde_json::Value;
260
+ use std::fs;
261
+ use tempfile::tempdir;
262
+ use uuid::Uuid;
263
+
264
+ #[test]
265
+ fn record_if_new_suppresses_only_exact_byte_duplicates() {
266
+ // WHY: dedup must be collision-safe. Keying on xxh64 alone (the old
267
+ // HashSet<u64>) would drop a distinct record that hashes into an
268
+ // occupied bucket; suppression must require an exact byte match.
269
+ let mut seen: FxHashMap<u64, Vec<Vec<u8>>> = FxHashMap::default();
270
+ // First sighting of a record is new.
271
+ assert!(record_if_new(&mut seen, b"{\"id\":\"A\"}"));
272
+ // A byte-identical record is suppressed.
273
+ assert!(!record_if_new(&mut seen, b"{\"id\":\"A\"}"));
274
+ // Distinct records survive even when they land in the same hash bucket
275
+ // (a real xxh64 collision is impractical to force, so this exercises the
276
+ // byte-exact equality path that decides suppression directly).
277
+ assert!(record_if_new(&mut seen, b"{\"id\":\"B\"}"));
278
+ assert!(!record_if_new(&mut seen, b"{\"id\":\"B\"}"));
279
+ }
280
+
281
+ #[test]
282
+ fn dedup_ndjson_keeps_distinct_records() {
283
+ // WHY: two different records must both survive dedup; only an identical
284
+ // duplicate is collapsed.
285
+ let dir = tempdir().expect("tempdir");
286
+ let input = dir.path().join("nodes.ndjson.tmp");
287
+ let output = dir.path().join("nodes.ndjson");
288
+ fs::write(&input, "{\"id\":\"A\"}\n{\"id\":\"B\"}\n{\"id\":\"A\"}\n").expect("write input");
289
+
290
+ dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
291
+
292
+ let lines: Vec<String> = fs::read_to_string(output)
293
+ .expect("read output")
294
+ .lines()
295
+ .map(str::to_string)
296
+ .collect();
297
+ assert_eq!(
298
+ lines,
299
+ vec!["{\"id\":\"A\"}".to_string(), "{\"id\":\"B\"}".to_string()]
300
+ );
301
+ }
302
+
303
+ #[test]
304
+ fn dedup_ndjson_deduplicates_nodes() {
305
+ let dir = tempdir().expect("tempdir");
306
+ let input = dir.path().join("nodes.ndjson.tmp");
307
+ let output = dir.path().join("nodes.ndjson");
308
+ fs::write(
309
+ &input,
310
+ "{\"id\":\"A\",\"drop\":\"NA\"}\n{\"id\":\"A\",\"drop\":\"NA\"}\n{}",
311
+ )
312
+ .expect("write input");
313
+
314
+ dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
315
+
316
+ let lines: Vec<String> = fs::read_to_string(output)
317
+ .expect("read output")
318
+ .lines()
319
+ .map(str::to_string)
320
+ .collect();
321
+ assert_eq!(lines, vec!["{\"id\":\"A\"}".to_string()]);
322
+ }
323
+
324
+ #[test]
325
+ fn dedup_ndjson_labels_edges() {
326
+ let dir = tempdir().expect("tempdir");
327
+ let input = dir.path().join("edges.ndjson.tmp");
328
+ let output = dir.path().join("edges.ndjson");
329
+ fs::write(
330
+ &input,
331
+ "{\"subject\":\"A\",\"object\":\"B\",\"predicate\":\"r\"}\n",
332
+ )
333
+ .expect("write input");
334
+
335
+ dedup_ndjson(
336
+ input,
337
+ output.clone(),
338
+ true,
339
+ Some("TABLASSERT".to_string()),
340
+ None,
341
+ )
342
+ .expect("dedup edges");
343
+
344
+ let line = fs::read_to_string(output).expect("read output");
345
+ let value: Value = serde_json::from_str(line.trim()).expect("json");
346
+ let id = value.get("id").and_then(Value::as_str).expect("edge id");
347
+ Uuid::parse_str(id).expect("valid UUID");
348
+ }
349
+
350
+ #[test]
351
+ fn dedup_ndjson_duplicate_edges_get_stable_single_id() {
352
+ let dir = tempdir().expect("tempdir");
353
+ let input = dir.path().join("edges.ndjson.tmp");
354
+ let output = dir.path().join("edges.ndjson");
355
+ fs::write(
356
+ &input,
357
+ concat!(
358
+ "{\"subject\":\"A\",\"object\":\"B\",\"predicate\":\"r\"}\n",
359
+ "{\"subject\":\"A\",\"object\":\"B\",\"predicate\":\"r\"}\n"
360
+ ),
361
+ )
362
+ .expect("write input");
363
+
364
+ dedup_ndjson(
365
+ input,
366
+ output.clone(),
367
+ true,
368
+ Some("TABLASSERT".to_string()),
369
+ None,
370
+ )
371
+ .expect("dedup edges");
372
+
373
+ let lines: Vec<String> = fs::read_to_string(output)
374
+ .expect("read output")
375
+ .lines()
376
+ .map(str::to_string)
377
+ .collect();
378
+ assert_eq!(lines.len(), 1);
379
+ let value: Value = serde_json::from_str(&lines[0]).expect("json");
380
+ let id = value.get("id").and_then(Value::as_str).expect("edge id");
381
+ Uuid::parse_str(id).expect("valid UUID");
382
+ }
383
+
384
+ #[test]
385
+ fn dedup_ndjson_strips_nested_null_like_values() {
386
+ let dir = tempdir().expect("tempdir");
387
+ let input = dir.path().join("nodes.ndjson.tmp");
388
+ let output = dir.path().join("nodes.ndjson");
389
+ fs::write(
390
+ &input,
391
+ "{\"id\":\"A\",\"meta\":{\"drop\":\"none\",\"keep\":\"yes\"},\"items\":[{\"x\":\"NA\",\"y\":\"z\"}]}\n",
392
+ )
393
+ .expect("write input");
394
+
395
+ dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
396
+
397
+ let line = fs::read_to_string(output).expect("read output");
398
+ let value: Value = serde_json::from_str(line.trim()).expect("json");
399
+ assert_eq!(value["meta"], serde_json::json!({"keep":"yes"}));
400
+ assert_eq!(value["items"], serde_json::json!([{"y":"z"}]));
401
+ }
402
+
403
+ #[test]
404
+ fn dedup_ndjson_empty_object_only_writes_empty_output() {
405
+ let dir = tempdir().expect("tempdir");
406
+ let input = dir.path().join("nodes.ndjson.tmp");
407
+ let output = dir.path().join("nodes.ndjson");
408
+ fs::write(&input, "{}\n{\"drop\":\"NA\"}\n").expect("write input");
409
+
410
+ dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
411
+
412
+ assert_eq!(fs::read_to_string(output).expect("read output"), "");
413
+ }
414
+
415
+ #[test]
416
+ fn dedup_ndjson_skips_blank_lines() {
417
+ let dir = tempdir().expect("tempdir");
418
+ let input = dir.path().join("nodes.ndjson.tmp");
419
+ let output = dir.path().join("nodes.ndjson");
420
+ fs::write(&input, "\n \n{\"id\":\"A\"}\n\n{\"id\":\"A\"}\n \n").expect("write input");
421
+
422
+ dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
423
+
424
+ assert_eq!(
425
+ fs::read_to_string(output).expect("read output"),
426
+ "{\"id\":\"A\"}\n"
427
+ );
428
+ }
429
+
430
+ fn write_edges(dir: &std::path::Path, body: &str) -> (std::path::PathBuf, std::path::PathBuf) {
431
+ let input = dir.join("edges.ndjson.tmp");
432
+ let output = dir.join("edges.ndjson");
433
+ fs::write(&input, body).expect("write input");
434
+ (input, output)
435
+ }
436
+
437
+ fn edge_ids(output: &std::path::Path) -> Vec<String> {
438
+ fs::read_to_string(output)
439
+ .expect("read output")
440
+ .lines()
441
+ .map(|line| {
442
+ serde_json::from_str::<Value>(line).expect("json")["id"]
443
+ .as_str()
444
+ .expect("edge id")
445
+ .to_string()
446
+ })
447
+ .collect()
448
+ }
449
+
450
+ #[test]
451
+ fn declared_uuid_fields_hold_the_id_still_across_attribute_edits() {
452
+ // WHY: the whole point of `uuid_fields`. Two builds whose only difference is a
453
+ // p_value must produce the SAME edge id, so downstream sees one edge updated
454
+ // rather than one retired and one created.
455
+ let dir = tempdir().expect("tempdir");
456
+ let fields = Some(vec![
457
+ "subject".to_string(),
458
+ "predicate".to_string(),
459
+ "object".to_string(),
460
+ ]);
461
+
462
+ let (before_in, before_out) = write_edges(
463
+ dir.path(),
464
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
465
+ );
466
+ dedup_ndjson(before_in, before_out.clone(), true, None, fields.clone()).expect("dedup");
467
+
468
+ let after_dir = tempdir().expect("tempdir");
469
+ let (after_in, after_out) = write_edges(
470
+ after_dir.path(),
471
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\",\"effect_size\":1.5}\n",
472
+ );
473
+ dedup_ndjson(after_in, after_out.clone(), true, None, fields).expect("dedup");
474
+
475
+ assert_eq!(edge_ids(&before_out), edge_ids(&after_out));
476
+ }
477
+
478
+ #[test]
479
+ fn undeclared_uuid_fields_let_the_id_drift() {
480
+ // WHY: the contrast case -- with no `uuid_fields`, the same attribute edit moves
481
+ // the id. This is the pre-16.0.0 behavior the default still preserves.
482
+ let dir = tempdir().expect("tempdir");
483
+ let (before_in, before_out) = write_edges(
484
+ dir.path(),
485
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
486
+ );
487
+ dedup_ndjson(before_in, before_out.clone(), true, None, None).expect("dedup");
488
+
489
+ let after_dir = tempdir().expect("tempdir");
490
+ let (after_in, after_out) = write_edges(
491
+ after_dir.path(),
492
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n",
493
+ );
494
+ dedup_ndjson(after_in, after_out.clone(), true, None, None).expect("dedup");
495
+
496
+ assert_ne!(edge_ids(&before_out), edge_ids(&after_out));
497
+ }
498
+
499
+ #[test]
500
+ fn uuid_fields_that_are_not_a_key_abort_the_build() {
501
+ // WHY: narrowing the hash inputs can make two DIFFERENT edges share an id.
502
+ // Shipping both would emit duplicate ids; dropping one would lose data. Fail,
503
+ // and name the fields that would disambiguate.
504
+ let dir = tempdir().expect("tempdir");
505
+ let (input, output) = write_edges(
506
+ dir.path(),
507
+ concat!(
508
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
509
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n"
510
+ ),
511
+ );
512
+ let fields = Some(vec![
513
+ "subject".to_string(),
514
+ "predicate".to_string(),
515
+ "object".to_string(),
516
+ ]);
517
+ let error = dedup_ndjson(input, output, true, None, fields).expect_err("not a key");
518
+ let message = error.to_string();
519
+ assert!(message.contains("uuid-fields-not-a-key"), "{message}");
520
+ assert!(message.contains("p_value"), "{message}");
521
+ assert!(message.contains("subject=A"), "{message}");
522
+ }
523
+
524
+ #[test]
525
+ fn identical_edges_still_collapse_under_declared_uuid_fields() {
526
+ // WHY: a true repeat is not a key violation -- it is the duplicate the deduper
527
+ // exists to collapse.
528
+ let dir = tempdir().expect("tempdir");
529
+ let (input, output) = write_edges(
530
+ dir.path(),
531
+ concat!(
532
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
533
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n"
534
+ ),
535
+ );
536
+ let fields = Some(vec!["subject".to_string(), "object".to_string()]);
537
+ dedup_ndjson(input, output.clone(), true, None, fields).expect("dedup edges");
538
+ assert_eq!(edge_ids(&output).len(), 1);
539
+ }
540
+
541
+ #[test]
542
+ fn key_order_alone_no_longer_ships_a_duplicate_id() {
543
+ // WHY: `uuid_for_json_object` sorts keys but the emitted bytes preserve insertion
544
+ // order, so byte-keyed dedup used to keep BOTH of these -- same id, two lines.
545
+ // Keying on the id collapses them.
546
+ let dir = tempdir().expect("tempdir");
547
+ let (input, output) = write_edges(
548
+ dir.path(),
549
+ concat!(
550
+ "{\"subject\":\"A\",\"object\":\"B\"}\n",
551
+ "{\"object\":\"B\",\"subject\":\"A\"}\n"
552
+ ),
553
+ );
554
+ dedup_ndjson(input, output.clone(), true, None, None).expect("dedup edges");
555
+ assert_eq!(edge_ids(&output).len(), 1);
556
+ }
557
+
558
+ #[test]
559
+ fn the_domain_separates_identical_edges_across_graphs() {
560
+ // WHY: two graphs asserting the same triple must not mint the same id, which is
561
+ // what makes a narrow `uuid_fields` safe across KGs.
562
+ let left_dir = tempdir().expect("tempdir");
563
+ let (left_in, left_out) =
564
+ write_edges(left_dir.path(), "{\"subject\":\"A\",\"object\":\"B\"}\n");
565
+ dedup_ndjson(
566
+ left_in,
567
+ left_out.clone(),
568
+ true,
569
+ Some("infores:left".to_string()),
570
+ None,
571
+ )
572
+ .expect("dedup");
573
+
574
+ let right_dir = tempdir().expect("tempdir");
575
+ let (right_in, right_out) =
576
+ write_edges(right_dir.path(), "{\"subject\":\"A\",\"object\":\"B\"}\n");
577
+ dedup_ndjson(
578
+ right_in,
579
+ right_out.clone(),
580
+ true,
581
+ Some("infores:right".to_string()),
582
+ None,
583
+ )
584
+ .expect("dedup");
585
+
586
+ assert_ne!(edge_ids(&left_out), edge_ids(&right_out));
587
+ }
588
+
589
+ #[test]
590
+ fn differing_keys_reports_only_real_differences_and_never_the_id() {
591
+ let left = serde_json::json!({"id": "x", "subject": "A", "p_value": "0.01"});
592
+ let right =
593
+ serde_json::json!({"id": "y", "subject": "A", "p_value": "0.99", "effect_size": 1.0});
594
+ assert_eq!(
595
+ differing_keys(&left, &right),
596
+ vec!["effect_size", "p_value"]
597
+ );
598
+ }
599
+ }