tablassert 15.1.0__tar.gz → 16.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-15.1.0 → tablassert-16.1.0}/PKG-INFO +1 -1
- {tablassert-15.1.0 → tablassert-16.1.0}/pyproject.toml +1 -1
- {tablassert-15.1.0 → tablassert-16.1.0}/rust/src/json.rs +31 -1
- tablassert-16.1.0/rust/src/ndjson.rs +599 -0
- tablassert-16.1.0/rust/src/uuid.rs +345 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/cli.py +24 -4
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/errors.py +2 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/ingests.py +14 -1
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/lib.py +68 -7
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/models.py +69 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/rig.py +7 -9
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/rs.pyi +1 -1
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/study.py +19 -2
- tablassert-15.1.0/rust/src/ndjson.rs +0 -254
- tablassert-15.1.0/rust/src/uuid.rs +0 -146
- {tablassert-15.1.0 → tablassert-16.1.0}/LICENSE +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/README.md +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/rust/Cargo.lock +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/rust/Cargo.toml +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/rust/src/fullmap.rs +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/rust/src/lib.rs +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/__init__.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/agent.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/biolink.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/coerce.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/enums.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/extras.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/fullmap.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/log.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/nlp.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/progress.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/qc.py +0 -0
- {tablassert-15.1.0 → tablassert-16.1.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "
|
|
3
|
+
version = "16.1.0"
|
|
4
4
|
description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -65,10 +65,40 @@ pub fn strip_nulls(value: &Value) -> Value {
|
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
|
|
68
|
+
/// Serialize in the record's own key order. This is the EMITTED form -- what actually
|
|
69
|
+
/// gets written to the NDJSON -- so it must not reorder anything. It is deliberately not
|
|
70
|
+
/// a canonical form: use `canonical_json_bytes` when comparing two records for equality.
|
|
71
|
+
pub fn emitted_json_bytes(value: &Value) -> serde_json::Result<Vec<u8>> {
|
|
69
72
|
serde_json::to_vec(value)
|
|
70
73
|
}
|
|
71
74
|
|
|
75
|
+
/// Serialize with every object's keys sorted, recursively.
|
|
76
|
+
///
|
|
77
|
+
/// `serde_json` is built with `preserve_order`, so plain `to_vec` leaks insertion order:
|
|
78
|
+
/// two logically identical records that arrived with different key order produce
|
|
79
|
+
/// different bytes. The edge deduper compares records for equality, so it needs a form
|
|
80
|
+
/// where "same content" means "same bytes"; array order is preserved because it is
|
|
81
|
+
/// semantic.
|
|
82
|
+
pub fn canonical_json_bytes(value: &Value) -> serde_json::Result<Vec<u8>> {
|
|
83
|
+
serde_json::to_vec(&canonical_value(value))
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
fn canonical_value(value: &Value) -> Value {
|
|
87
|
+
match value {
|
|
88
|
+
Value::Object(entries) => {
|
|
89
|
+
let mut keys: Vec<&String> = entries.keys().collect();
|
|
90
|
+
keys.sort_unstable();
|
|
91
|
+
let mut sorted: Map<String, Value> = Map::with_capacity(entries.len());
|
|
92
|
+
for key in keys {
|
|
93
|
+
sorted.insert(key.clone(), canonical_value(&entries[key]));
|
|
94
|
+
}
|
|
95
|
+
Value::Object(sorted)
|
|
96
|
+
}
|
|
97
|
+
Value::Array(items) => Value::Array(items.iter().map(canonical_value).collect()),
|
|
98
|
+
_ => value.clone(),
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
72
102
|
#[cfg(test)]
|
|
73
103
|
mod tests {
|
|
74
104
|
use super::strip_nulls;
|
|
@@ -0,0 +1,599 @@
|
|
|
1
|
+
use crate::json::{canonical_json_bytes, emitted_json_bytes, strip_nulls};
|
|
2
|
+
use crate::uuid::uuid_for_json_object;
|
|
3
|
+
use pyo3::exceptions::PyRuntimeError;
|
|
4
|
+
use pyo3::prelude::*;
|
|
5
|
+
use rustc_hash::FxHashMap;
|
|
6
|
+
use serde_json::Value;
|
|
7
|
+
use std::fs::File;
|
|
8
|
+
use std::io::{BufRead, BufReader, BufWriter, Write};
|
|
9
|
+
use std::path::{Path, PathBuf};
|
|
10
|
+
use uuid::Uuid;
|
|
11
|
+
use xxhash_rust::xxh64::xxh64;
|
|
12
|
+
|
|
13
|
+
fn runtime_error(error: impl ToString) -> PyErr {
|
|
14
|
+
PyRuntimeError::new_err(error.to_string())
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/// Collision-safe dedup decision for the NODES stream. Records are bucketed by xxh64
|
|
18
|
+
/// but a record is suppressed ONLY when its canonical bytes exactly match an existing
|
|
19
|
+
/// entry in the bucket, so two DISTINCT records whose xxh64 collides both survive.
|
|
20
|
+
/// (The former `HashSet<u64>` keyed on the hash alone silently dropped the second record
|
|
21
|
+
/// on a collision — data loss at scale.) Returns true when `bytes` is new and recorded.
|
|
22
|
+
///
|
|
23
|
+
/// Edges do NOT use this path: their `id` is a derived hash, so they dedup on the id
|
|
24
|
+
/// itself (see `EdgeIndex`), which both guarantees id uniqueness and keeps 16 bytes per
|
|
25
|
+
/// edge instead of the whole record.
|
|
26
|
+
fn record_if_new(seen: &mut FxHashMap<u64, Vec<Vec<u8>>>, bytes: &[u8]) -> bool {
|
|
27
|
+
let bucket = seen.entry(xxh64(bytes, 0)).or_default();
|
|
28
|
+
if bucket.iter().any(|existing| existing.as_slice() == bytes) {
|
|
29
|
+
false
|
|
30
|
+
} else {
|
|
31
|
+
bucket.push(bytes.to_vec());
|
|
32
|
+
true
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
fn label_edge(mut value: Value, domain: &str, fields: Option<&[String]>) -> PyResult<Value> {
|
|
37
|
+
let id: String = uuid_for_json_object(domain, &value, fields)
|
|
38
|
+
.ok_or_else(|| runtime_error("expected JSON object"))?;
|
|
39
|
+
value
|
|
40
|
+
.as_object_mut()
|
|
41
|
+
.ok_or_else(|| runtime_error("expected JSON object"))?
|
|
42
|
+
.insert("id".to_string(), Value::String(id));
|
|
43
|
+
Ok(value)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/// A record ready to write, plus (for edges) the content hash of the record as it stood
|
|
47
|
+
/// BEFORE its `id` was inserted.
|
|
48
|
+
///
|
|
49
|
+
/// Hashing pre-insertion matters twice over: the id is a pure function of the other
|
|
50
|
+
/// fields, so a post-insertion hash would agree whenever the ids agree -- exactly the
|
|
51
|
+
/// divergence this is meant to detect -- and it avoids cloning every record just to strip
|
|
52
|
+
/// one key back out.
|
|
53
|
+
struct Finalized {
|
|
54
|
+
value: Value,
|
|
55
|
+
content: u64,
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
fn finalize_record(
|
|
59
|
+
value: Value,
|
|
60
|
+
is_edges: bool,
|
|
61
|
+
domain: &str,
|
|
62
|
+
fields: Option<&[String]>,
|
|
63
|
+
) -> PyResult<Option<Finalized>> {
|
|
64
|
+
let cleaned: Value = strip_nulls(&value);
|
|
65
|
+
let is_empty_object: bool = matches!(&cleaned, Value::Object(map) if map.is_empty());
|
|
66
|
+
if is_empty_object {
|
|
67
|
+
return Ok(None);
|
|
68
|
+
}
|
|
69
|
+
if !is_edges {
|
|
70
|
+
return Ok(Some(Finalized {
|
|
71
|
+
value: cleaned,
|
|
72
|
+
content: 0,
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
let content: u64 = xxh64(&canonical_json_bytes(&cleaned).map_err(runtime_error)?, 0);
|
|
76
|
+
label_edge(cleaned, domain, fields).map(|value| Some(Finalized { value, content }))
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/// Edge dedup state: derived id -> hash of that edge's canonical, id-free content.
|
|
80
|
+
///
|
|
81
|
+
/// Keying on the id (not on the whole record, as the nodes path does) is what makes
|
|
82
|
+
/// `uuid_fields` safe: once the hash covers only a subset of the record, two records can
|
|
83
|
+
/// share an id while differing in bytes, and byte-keyed dedup would happily emit both --
|
|
84
|
+
/// duplicate edge ids in the output. It is also ~20-25x cheaper: 24 bytes per edge
|
|
85
|
+
/// instead of a full copy of every record (~800 bytes each at the 10M-edge scale).
|
|
86
|
+
#[derive(Default)]
|
|
87
|
+
struct EdgeIndex {
|
|
88
|
+
seen: FxHashMap<[u8; 16], u64>,
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/// What to do with an edge whose id has been seen before.
|
|
92
|
+
enum EdgeVerdict {
|
|
93
|
+
/// First sighting of this id -- write it.
|
|
94
|
+
Fresh,
|
|
95
|
+
/// Byte-identical to the record that already claimed this id -- suppress it.
|
|
96
|
+
Duplicate,
|
|
97
|
+
/// A DIFFERENT record claims this id: the declared `uuid_fields` are not a key.
|
|
98
|
+
Divergent,
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
impl EdgeIndex {
|
|
102
|
+
fn classify(&mut self, id: [u8; 16], content: u64) -> EdgeVerdict {
|
|
103
|
+
match self.seen.get(&id) {
|
|
104
|
+
None => {
|
|
105
|
+
self.seen.insert(id, content);
|
|
106
|
+
EdgeVerdict::Fresh
|
|
107
|
+
}
|
|
108
|
+
Some(existing) if *existing == content => EdgeVerdict::Duplicate,
|
|
109
|
+
Some(_) => EdgeVerdict::Divergent,
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
fn edge_id_bytes(value: &Value) -> PyResult<[u8; 16]> {
|
|
115
|
+
let id: &str = value
|
|
116
|
+
.get("id")
|
|
117
|
+
.and_then(Value::as_str)
|
|
118
|
+
.ok_or_else(|| runtime_error("labeled edge is missing its id"))?;
|
|
119
|
+
Uuid::parse_str(id)
|
|
120
|
+
.map(Uuid::into_bytes)
|
|
121
|
+
.map_err(|error| runtime_error(format!("edge id {id} is not a UUID: {error}")))
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/// Recover the record that first claimed `id` so the error can name what actually
|
|
125
|
+
/// differs. Runs ONLY on the failure path -- the build is about to abort, so re-reading
|
|
126
|
+
/// the partial output costs nothing in the happy case.
|
|
127
|
+
fn find_written_edge(output: &Path, id: &str) -> Option<Value> {
|
|
128
|
+
let reader = BufReader::new(File::open(output).ok()?);
|
|
129
|
+
reader
|
|
130
|
+
.lines()
|
|
131
|
+
.map_while(Result::ok)
|
|
132
|
+
.filter(|line| line.contains(id))
|
|
133
|
+
.find_map(|line| {
|
|
134
|
+
let value: Value = serde_json::from_str(&line).ok()?;
|
|
135
|
+
(value.get("id").and_then(Value::as_str) == Some(id)).then_some(value)
|
|
136
|
+
})
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/// Top-level keys whose values differ between two records.
|
|
140
|
+
fn differing_keys(left: &Value, right: &Value) -> Vec<String> {
|
|
141
|
+
let (Some(left), Some(right)) = (left.as_object(), right.as_object()) else {
|
|
142
|
+
return Vec::new();
|
|
143
|
+
};
|
|
144
|
+
let mut keys: Vec<String> = left
|
|
145
|
+
.keys()
|
|
146
|
+
.chain(right.keys())
|
|
147
|
+
.filter(|key| key.as_str() != "id")
|
|
148
|
+
.filter(|key| left.get(*key) != right.get(*key))
|
|
149
|
+
.cloned()
|
|
150
|
+
.collect();
|
|
151
|
+
keys.sort_unstable();
|
|
152
|
+
keys.dedup();
|
|
153
|
+
keys
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/// Build the `uuid-fields-not-a-key` diagnostic.
|
|
157
|
+
///
|
|
158
|
+
/// The author's declared `uuid_fields` do not uniquely identify an edge in this graph, so
|
|
159
|
+
/// two genuinely different records derived the same id. Emitting both would ship duplicate
|
|
160
|
+
/// ids; silently dropping one would lose data. Fail, and name the fields that would fix it.
|
|
161
|
+
fn not_a_key_error(output: &Path, id: &str, incoming: &Value, fields: Option<&[String]>) -> PyErr {
|
|
162
|
+
let declared: String = match fields {
|
|
163
|
+
Some(fields) => fields.join(", "),
|
|
164
|
+
None => "<all fields>".to_string(),
|
|
165
|
+
};
|
|
166
|
+
let differing: String = find_written_edge(output, id)
|
|
167
|
+
.map(|existing| differing_keys(&existing, incoming))
|
|
168
|
+
.filter(|keys| !keys.is_empty())
|
|
169
|
+
.map_or_else(
|
|
170
|
+
|| " (could not recover the first record to diff)".to_string(),
|
|
171
|
+
|keys| format!(" they differ in: {}", keys.join(", ")),
|
|
172
|
+
);
|
|
173
|
+
let context: String = ["subject", "predicate", "object"]
|
|
174
|
+
.iter()
|
|
175
|
+
.filter_map(|key| {
|
|
176
|
+
incoming
|
|
177
|
+
.get(*key)
|
|
178
|
+
.and_then(Value::as_str)
|
|
179
|
+
.map(|value| format!("{key}={value}"))
|
|
180
|
+
})
|
|
181
|
+
.collect::<Vec<String>>()
|
|
182
|
+
.join(" ");
|
|
183
|
+
runtime_error(format!(
|
|
184
|
+
"uuid-fields-not-a-key: declared uuid_fields are not a key for this graph.\n\
|
|
185
|
+
\x20 id {id} is claimed by 2 different edges.\n\
|
|
186
|
+
{differing}\n\
|
|
187
|
+
\x20 declared uuid_fields: {declared}\n\
|
|
188
|
+
\x20 offending edge: {context}\n\
|
|
189
|
+
Add a discriminating field to `uuid_fields` (a qualifier, `has_supporting_studies` \
|
|
190
|
+
for the source row, or the statistic that actually differs)."
|
|
191
|
+
))
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
#[pyfunction]
|
|
195
|
+
#[pyo3(signature = (input, output, is_edges, domain=None, uuid_fields=None))]
|
|
196
|
+
pub fn dedup_ndjson(
|
|
197
|
+
input: PathBuf,
|
|
198
|
+
output: PathBuf,
|
|
199
|
+
is_edges: bool,
|
|
200
|
+
domain: Option<String>,
|
|
201
|
+
uuid_fields: Option<Vec<String>>,
|
|
202
|
+
) -> PyResult<()> {
|
|
203
|
+
let domain: String = domain.unwrap_or_else(|| "TABLASSERT".to_string());
|
|
204
|
+
let fields: Option<&[String]> = uuid_fields.as_deref();
|
|
205
|
+
let reader: BufReader<File> = BufReader::new(File::open(input).map_err(runtime_error)?);
|
|
206
|
+
let mut writer: BufWriter<File> = BufWriter::new(File::create(&output).map_err(runtime_error)?);
|
|
207
|
+
let mut nodes: FxHashMap<u64, Vec<Vec<u8>>> = FxHashMap::default();
|
|
208
|
+
let mut edges: EdgeIndex = EdgeIndex::default();
|
|
209
|
+
|
|
210
|
+
reader
|
|
211
|
+
.lines()
|
|
212
|
+
.map(|line| line.map_err(runtime_error))
|
|
213
|
+
.filter(|line| line.as_ref().map_or(true, |line| !line.trim().is_empty()))
|
|
214
|
+
.map(|line| {
|
|
215
|
+
line.and_then(|line| serde_json::from_str::<Value>(&line).map_err(runtime_error))
|
|
216
|
+
})
|
|
217
|
+
.map(|value| value.and_then(|value| finalize_record(value, is_edges, &domain, fields)))
|
|
218
|
+
.try_for_each(|record| -> PyResult<()> {
|
|
219
|
+
let Some(Finalized { value, content }) = record? else {
|
|
220
|
+
return Ok(());
|
|
221
|
+
};
|
|
222
|
+
let write: bool = if is_edges {
|
|
223
|
+
// Edges dedup on their derived id, so the output can never carry the same
|
|
224
|
+
// id twice: an exact repeat is suppressed and a genuine divergence aborts
|
|
225
|
+
// the build rather than shipping a duplicate.
|
|
226
|
+
match edges.classify(edge_id_bytes(&value)?, content) {
|
|
227
|
+
EdgeVerdict::Fresh => true,
|
|
228
|
+
EdgeVerdict::Duplicate => false,
|
|
229
|
+
EdgeVerdict::Divergent => {
|
|
230
|
+
writer.flush().map_err(runtime_error)?;
|
|
231
|
+
let id: &str = value["id"].as_str().unwrap_or_default();
|
|
232
|
+
return Err(not_a_key_error(&output, id, &value, fields));
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
} else {
|
|
236
|
+
record_if_new(
|
|
237
|
+
&mut nodes,
|
|
238
|
+
&emitted_json_bytes(&value).map_err(runtime_error)?,
|
|
239
|
+
)
|
|
240
|
+
};
|
|
241
|
+
if write {
|
|
242
|
+
writer
|
|
243
|
+
.write_all(&emitted_json_bytes(&value).map_err(runtime_error)?)
|
|
244
|
+
.map_err(runtime_error)?;
|
|
245
|
+
writer.write_all(b"\n").map_err(runtime_error)?;
|
|
246
|
+
}
|
|
247
|
+
Ok(())
|
|
248
|
+
})?;
|
|
249
|
+
// Flush explicitly and propagate failure: relying on BufWriter's drop-time
|
|
250
|
+
// flush would swallow a final write error and report success with truncated
|
|
251
|
+
// output.
|
|
252
|
+
writer.flush().map_err(runtime_error)
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
#[cfg(test)]
|
|
256
|
+
mod tests {
|
|
257
|
+
use super::{dedup_ndjson, differing_keys, record_if_new};
|
|
258
|
+
use rustc_hash::FxHashMap;
|
|
259
|
+
use serde_json::Value;
|
|
260
|
+
use std::fs;
|
|
261
|
+
use tempfile::tempdir;
|
|
262
|
+
use uuid::Uuid;
|
|
263
|
+
|
|
264
|
+
#[test]
|
|
265
|
+
fn record_if_new_suppresses_only_exact_byte_duplicates() {
|
|
266
|
+
// WHY: dedup must be collision-safe. Keying on xxh64 alone (the old
|
|
267
|
+
// HashSet<u64>) would drop a distinct record that hashes into an
|
|
268
|
+
// occupied bucket; suppression must require an exact byte match.
|
|
269
|
+
let mut seen: FxHashMap<u64, Vec<Vec<u8>>> = FxHashMap::default();
|
|
270
|
+
// First sighting of a record is new.
|
|
271
|
+
assert!(record_if_new(&mut seen, b"{\"id\":\"A\"}"));
|
|
272
|
+
// A byte-identical record is suppressed.
|
|
273
|
+
assert!(!record_if_new(&mut seen, b"{\"id\":\"A\"}"));
|
|
274
|
+
// Distinct records survive even when they land in the same hash bucket
|
|
275
|
+
// (a real xxh64 collision is impractical to force, so this exercises the
|
|
276
|
+
// byte-exact equality path that decides suppression directly).
|
|
277
|
+
assert!(record_if_new(&mut seen, b"{\"id\":\"B\"}"));
|
|
278
|
+
assert!(!record_if_new(&mut seen, b"{\"id\":\"B\"}"));
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
#[test]
|
|
282
|
+
fn dedup_ndjson_keeps_distinct_records() {
|
|
283
|
+
// WHY: two different records must both survive dedup; only an identical
|
|
284
|
+
// duplicate is collapsed.
|
|
285
|
+
let dir = tempdir().expect("tempdir");
|
|
286
|
+
let input = dir.path().join("nodes.ndjson.tmp");
|
|
287
|
+
let output = dir.path().join("nodes.ndjson");
|
|
288
|
+
fs::write(&input, "{\"id\":\"A\"}\n{\"id\":\"B\"}\n{\"id\":\"A\"}\n").expect("write input");
|
|
289
|
+
|
|
290
|
+
dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
|
|
291
|
+
|
|
292
|
+
let lines: Vec<String> = fs::read_to_string(output)
|
|
293
|
+
.expect("read output")
|
|
294
|
+
.lines()
|
|
295
|
+
.map(str::to_string)
|
|
296
|
+
.collect();
|
|
297
|
+
assert_eq!(
|
|
298
|
+
lines,
|
|
299
|
+
vec!["{\"id\":\"A\"}".to_string(), "{\"id\":\"B\"}".to_string()]
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
#[test]
|
|
304
|
+
fn dedup_ndjson_deduplicates_nodes() {
|
|
305
|
+
let dir = tempdir().expect("tempdir");
|
|
306
|
+
let input = dir.path().join("nodes.ndjson.tmp");
|
|
307
|
+
let output = dir.path().join("nodes.ndjson");
|
|
308
|
+
fs::write(
|
|
309
|
+
&input,
|
|
310
|
+
"{\"id\":\"A\",\"drop\":\"NA\"}\n{\"id\":\"A\",\"drop\":\"NA\"}\n{}",
|
|
311
|
+
)
|
|
312
|
+
.expect("write input");
|
|
313
|
+
|
|
314
|
+
dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
|
|
315
|
+
|
|
316
|
+
let lines: Vec<String> = fs::read_to_string(output)
|
|
317
|
+
.expect("read output")
|
|
318
|
+
.lines()
|
|
319
|
+
.map(str::to_string)
|
|
320
|
+
.collect();
|
|
321
|
+
assert_eq!(lines, vec!["{\"id\":\"A\"}".to_string()]);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
#[test]
|
|
325
|
+
fn dedup_ndjson_labels_edges() {
|
|
326
|
+
let dir = tempdir().expect("tempdir");
|
|
327
|
+
let input = dir.path().join("edges.ndjson.tmp");
|
|
328
|
+
let output = dir.path().join("edges.ndjson");
|
|
329
|
+
fs::write(
|
|
330
|
+
&input,
|
|
331
|
+
"{\"subject\":\"A\",\"object\":\"B\",\"predicate\":\"r\"}\n",
|
|
332
|
+
)
|
|
333
|
+
.expect("write input");
|
|
334
|
+
|
|
335
|
+
dedup_ndjson(
|
|
336
|
+
input,
|
|
337
|
+
output.clone(),
|
|
338
|
+
true,
|
|
339
|
+
Some("TABLASSERT".to_string()),
|
|
340
|
+
None,
|
|
341
|
+
)
|
|
342
|
+
.expect("dedup edges");
|
|
343
|
+
|
|
344
|
+
let line = fs::read_to_string(output).expect("read output");
|
|
345
|
+
let value: Value = serde_json::from_str(line.trim()).expect("json");
|
|
346
|
+
let id = value.get("id").and_then(Value::as_str).expect("edge id");
|
|
347
|
+
Uuid::parse_str(id).expect("valid UUID");
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
#[test]
|
|
351
|
+
fn dedup_ndjson_duplicate_edges_get_stable_single_id() {
|
|
352
|
+
let dir = tempdir().expect("tempdir");
|
|
353
|
+
let input = dir.path().join("edges.ndjson.tmp");
|
|
354
|
+
let output = dir.path().join("edges.ndjson");
|
|
355
|
+
fs::write(
|
|
356
|
+
&input,
|
|
357
|
+
concat!(
|
|
358
|
+
"{\"subject\":\"A\",\"object\":\"B\",\"predicate\":\"r\"}\n",
|
|
359
|
+
"{\"subject\":\"A\",\"object\":\"B\",\"predicate\":\"r\"}\n"
|
|
360
|
+
),
|
|
361
|
+
)
|
|
362
|
+
.expect("write input");
|
|
363
|
+
|
|
364
|
+
dedup_ndjson(
|
|
365
|
+
input,
|
|
366
|
+
output.clone(),
|
|
367
|
+
true,
|
|
368
|
+
Some("TABLASSERT".to_string()),
|
|
369
|
+
None,
|
|
370
|
+
)
|
|
371
|
+
.expect("dedup edges");
|
|
372
|
+
|
|
373
|
+
let lines: Vec<String> = fs::read_to_string(output)
|
|
374
|
+
.expect("read output")
|
|
375
|
+
.lines()
|
|
376
|
+
.map(str::to_string)
|
|
377
|
+
.collect();
|
|
378
|
+
assert_eq!(lines.len(), 1);
|
|
379
|
+
let value: Value = serde_json::from_str(&lines[0]).expect("json");
|
|
380
|
+
let id = value.get("id").and_then(Value::as_str).expect("edge id");
|
|
381
|
+
Uuid::parse_str(id).expect("valid UUID");
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
#[test]
|
|
385
|
+
fn dedup_ndjson_strips_nested_null_like_values() {
|
|
386
|
+
let dir = tempdir().expect("tempdir");
|
|
387
|
+
let input = dir.path().join("nodes.ndjson.tmp");
|
|
388
|
+
let output = dir.path().join("nodes.ndjson");
|
|
389
|
+
fs::write(
|
|
390
|
+
&input,
|
|
391
|
+
"{\"id\":\"A\",\"meta\":{\"drop\":\"none\",\"keep\":\"yes\"},\"items\":[{\"x\":\"NA\",\"y\":\"z\"}]}\n",
|
|
392
|
+
)
|
|
393
|
+
.expect("write input");
|
|
394
|
+
|
|
395
|
+
dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
|
|
396
|
+
|
|
397
|
+
let line = fs::read_to_string(output).expect("read output");
|
|
398
|
+
let value: Value = serde_json::from_str(line.trim()).expect("json");
|
|
399
|
+
assert_eq!(value["meta"], serde_json::json!({"keep":"yes"}));
|
|
400
|
+
assert_eq!(value["items"], serde_json::json!([{"y":"z"}]));
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
#[test]
|
|
404
|
+
fn dedup_ndjson_empty_object_only_writes_empty_output() {
|
|
405
|
+
let dir = tempdir().expect("tempdir");
|
|
406
|
+
let input = dir.path().join("nodes.ndjson.tmp");
|
|
407
|
+
let output = dir.path().join("nodes.ndjson");
|
|
408
|
+
fs::write(&input, "{}\n{\"drop\":\"NA\"}\n").expect("write input");
|
|
409
|
+
|
|
410
|
+
dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
|
|
411
|
+
|
|
412
|
+
assert_eq!(fs::read_to_string(output).expect("read output"), "");
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
#[test]
|
|
416
|
+
fn dedup_ndjson_skips_blank_lines() {
|
|
417
|
+
let dir = tempdir().expect("tempdir");
|
|
418
|
+
let input = dir.path().join("nodes.ndjson.tmp");
|
|
419
|
+
let output = dir.path().join("nodes.ndjson");
|
|
420
|
+
fs::write(&input, "\n \n{\"id\":\"A\"}\n\n{\"id\":\"A\"}\n \n").expect("write input");
|
|
421
|
+
|
|
422
|
+
dedup_ndjson(input, output.clone(), false, None, None).expect("dedup nodes");
|
|
423
|
+
|
|
424
|
+
assert_eq!(
|
|
425
|
+
fs::read_to_string(output).expect("read output"),
|
|
426
|
+
"{\"id\":\"A\"}\n"
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
fn write_edges(dir: &std::path::Path, body: &str) -> (std::path::PathBuf, std::path::PathBuf) {
|
|
431
|
+
let input = dir.join("edges.ndjson.tmp");
|
|
432
|
+
let output = dir.join("edges.ndjson");
|
|
433
|
+
fs::write(&input, body).expect("write input");
|
|
434
|
+
(input, output)
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
fn edge_ids(output: &std::path::Path) -> Vec<String> {
|
|
438
|
+
fs::read_to_string(output)
|
|
439
|
+
.expect("read output")
|
|
440
|
+
.lines()
|
|
441
|
+
.map(|line| {
|
|
442
|
+
serde_json::from_str::<Value>(line).expect("json")["id"]
|
|
443
|
+
.as_str()
|
|
444
|
+
.expect("edge id")
|
|
445
|
+
.to_string()
|
|
446
|
+
})
|
|
447
|
+
.collect()
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
#[test]
|
|
451
|
+
fn declared_uuid_fields_hold_the_id_still_across_attribute_edits() {
|
|
452
|
+
// WHY: the whole point of `uuid_fields`. Two builds whose only difference is a
|
|
453
|
+
// p_value must produce the SAME edge id, so downstream sees one edge updated
|
|
454
|
+
// rather than one retired and one created.
|
|
455
|
+
let dir = tempdir().expect("tempdir");
|
|
456
|
+
let fields = Some(vec![
|
|
457
|
+
"subject".to_string(),
|
|
458
|
+
"predicate".to_string(),
|
|
459
|
+
"object".to_string(),
|
|
460
|
+
]);
|
|
461
|
+
|
|
462
|
+
let (before_in, before_out) = write_edges(
|
|
463
|
+
dir.path(),
|
|
464
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
|
|
465
|
+
);
|
|
466
|
+
dedup_ndjson(before_in, before_out.clone(), true, None, fields.clone()).expect("dedup");
|
|
467
|
+
|
|
468
|
+
let after_dir = tempdir().expect("tempdir");
|
|
469
|
+
let (after_in, after_out) = write_edges(
|
|
470
|
+
after_dir.path(),
|
|
471
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\",\"effect_size\":1.5}\n",
|
|
472
|
+
);
|
|
473
|
+
dedup_ndjson(after_in, after_out.clone(), true, None, fields).expect("dedup");
|
|
474
|
+
|
|
475
|
+
assert_eq!(edge_ids(&before_out), edge_ids(&after_out));
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
#[test]
|
|
479
|
+
fn undeclared_uuid_fields_let_the_id_drift() {
|
|
480
|
+
// WHY: the contrast case -- with no `uuid_fields`, the same attribute edit moves
|
|
481
|
+
// the id. This is the pre-16.0.0 behavior the default still preserves.
|
|
482
|
+
let dir = tempdir().expect("tempdir");
|
|
483
|
+
let (before_in, before_out) = write_edges(
|
|
484
|
+
dir.path(),
|
|
485
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
|
|
486
|
+
);
|
|
487
|
+
dedup_ndjson(before_in, before_out.clone(), true, None, None).expect("dedup");
|
|
488
|
+
|
|
489
|
+
let after_dir = tempdir().expect("tempdir");
|
|
490
|
+
let (after_in, after_out) = write_edges(
|
|
491
|
+
after_dir.path(),
|
|
492
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n",
|
|
493
|
+
);
|
|
494
|
+
dedup_ndjson(after_in, after_out.clone(), true, None, None).expect("dedup");
|
|
495
|
+
|
|
496
|
+
assert_ne!(edge_ids(&before_out), edge_ids(&after_out));
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
#[test]
|
|
500
|
+
fn uuid_fields_that_are_not_a_key_abort_the_build() {
|
|
501
|
+
// WHY: narrowing the hash inputs can make two DIFFERENT edges share an id.
|
|
502
|
+
// Shipping both would emit duplicate ids; dropping one would lose data. Fail,
|
|
503
|
+
// and name the fields that would disambiguate.
|
|
504
|
+
let dir = tempdir().expect("tempdir");
|
|
505
|
+
let (input, output) = write_edges(
|
|
506
|
+
dir.path(),
|
|
507
|
+
concat!(
|
|
508
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
|
|
509
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.99\"}\n"
|
|
510
|
+
),
|
|
511
|
+
);
|
|
512
|
+
let fields = Some(vec![
|
|
513
|
+
"subject".to_string(),
|
|
514
|
+
"predicate".to_string(),
|
|
515
|
+
"object".to_string(),
|
|
516
|
+
]);
|
|
517
|
+
let error = dedup_ndjson(input, output, true, None, fields).expect_err("not a key");
|
|
518
|
+
let message = error.to_string();
|
|
519
|
+
assert!(message.contains("uuid-fields-not-a-key"), "{message}");
|
|
520
|
+
assert!(message.contains("p_value"), "{message}");
|
|
521
|
+
assert!(message.contains("subject=A"), "{message}");
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
#[test]
|
|
525
|
+
fn identical_edges_still_collapse_under_declared_uuid_fields() {
|
|
526
|
+
// WHY: a true repeat is not a key violation -- it is the duplicate the deduper
|
|
527
|
+
// exists to collapse.
|
|
528
|
+
let dir = tempdir().expect("tempdir");
|
|
529
|
+
let (input, output) = write_edges(
|
|
530
|
+
dir.path(),
|
|
531
|
+
concat!(
|
|
532
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n",
|
|
533
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"p_value\":\"0.01\"}\n"
|
|
534
|
+
),
|
|
535
|
+
);
|
|
536
|
+
let fields = Some(vec!["subject".to_string(), "object".to_string()]);
|
|
537
|
+
dedup_ndjson(input, output.clone(), true, None, fields).expect("dedup edges");
|
|
538
|
+
assert_eq!(edge_ids(&output).len(), 1);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
#[test]
|
|
542
|
+
fn key_order_alone_no_longer_ships_a_duplicate_id() {
|
|
543
|
+
// WHY: `uuid_for_json_object` sorts keys but the emitted bytes preserve insertion
|
|
544
|
+
// order, so byte-keyed dedup used to keep BOTH of these -- same id, two lines.
|
|
545
|
+
// Keying on the id collapses them.
|
|
546
|
+
let dir = tempdir().expect("tempdir");
|
|
547
|
+
let (input, output) = write_edges(
|
|
548
|
+
dir.path(),
|
|
549
|
+
concat!(
|
|
550
|
+
"{\"subject\":\"A\",\"object\":\"B\"}\n",
|
|
551
|
+
"{\"object\":\"B\",\"subject\":\"A\"}\n"
|
|
552
|
+
),
|
|
553
|
+
);
|
|
554
|
+
dedup_ndjson(input, output.clone(), true, None, None).expect("dedup edges");
|
|
555
|
+
assert_eq!(edge_ids(&output).len(), 1);
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
#[test]
|
|
559
|
+
fn the_domain_separates_identical_edges_across_graphs() {
|
|
560
|
+
// WHY: two graphs asserting the same triple must not mint the same id, which is
|
|
561
|
+
// what makes a narrow `uuid_fields` safe across KGs.
|
|
562
|
+
let left_dir = tempdir().expect("tempdir");
|
|
563
|
+
let (left_in, left_out) =
|
|
564
|
+
write_edges(left_dir.path(), "{\"subject\":\"A\",\"object\":\"B\"}\n");
|
|
565
|
+
dedup_ndjson(
|
|
566
|
+
left_in,
|
|
567
|
+
left_out.clone(),
|
|
568
|
+
true,
|
|
569
|
+
Some("infores:left".to_string()),
|
|
570
|
+
None,
|
|
571
|
+
)
|
|
572
|
+
.expect("dedup");
|
|
573
|
+
|
|
574
|
+
let right_dir = tempdir().expect("tempdir");
|
|
575
|
+
let (right_in, right_out) =
|
|
576
|
+
write_edges(right_dir.path(), "{\"subject\":\"A\",\"object\":\"B\"}\n");
|
|
577
|
+
dedup_ndjson(
|
|
578
|
+
right_in,
|
|
579
|
+
right_out.clone(),
|
|
580
|
+
true,
|
|
581
|
+
Some("infores:right".to_string()),
|
|
582
|
+
None,
|
|
583
|
+
)
|
|
584
|
+
.expect("dedup");
|
|
585
|
+
|
|
586
|
+
assert_ne!(edge_ids(&left_out), edge_ids(&right_out));
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
#[test]
|
|
590
|
+
fn differing_keys_reports_only_real_differences_and_never_the_id() {
|
|
591
|
+
let left = serde_json::json!({"id": "x", "subject": "A", "p_value": "0.01"});
|
|
592
|
+
let right =
|
|
593
|
+
serde_json::json!({"id": "y", "subject": "A", "p_value": "0.99", "effect_size": 1.0});
|
|
594
|
+
assert_eq!(
|
|
595
|
+
differing_keys(&left, &right),
|
|
596
|
+
vec!["effect_size", "p_value"]
|
|
597
|
+
);
|
|
598
|
+
}
|
|
599
|
+
}
|