tablassert 16.3.1__tar.gz → 16.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-16.3.1 → tablassert-16.6.0}/PKG-INFO +4 -1
- {tablassert-16.3.1 → tablassert-16.6.0}/README.md +1 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/pyproject.toml +4 -1
- {tablassert-16.3.1 → tablassert-16.6.0}/rust/src/ndjson.rs +220 -5
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/agent.py +791 -86
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/biolink.py +8 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/cli.py +70 -2
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/coerce.py +411 -165
- tablassert-16.6.0/src/tablassert/distill.py +108 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/extras.py +2 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/lib.py +37 -35
- {tablassert-16.3.1 → tablassert-16.6.0}/LICENSE +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/rust/Cargo.lock +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/rust/Cargo.toml +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/rust/src/fullmap.rs +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/rust/src/json.rs +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/rust/src/lib.rs +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/rust/src/uuid.rs +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/__init__.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/enums.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/errors.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/fullmap.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/ingests.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/log.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/models.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/nlp.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/progress.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/qc.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/rig.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/study.py +0 -0
- {tablassert-16.3.1 → tablassert-16.6.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version: 16.
|
|
3
|
+
Version: 16.6.0
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -33,6 +33,7 @@ Requires-Dist: fastexcel>=0.20.2
|
|
|
33
33
|
Requires-Dist: smolagents>=1.26.0 ; extra == 'agent'
|
|
34
34
|
Requires-Dist: litellm>=1.93.0 ; extra == 'agent'
|
|
35
35
|
Requires-Dist: aria2==0.0.1b0 ; extra == 'aria2'
|
|
36
|
+
Requires-Dist: datasets>=3.0.0 ; extra == 'distill'
|
|
36
37
|
Requires-Dist: loguru>=0.7.3 ; extra == 'log'
|
|
37
38
|
Requires-Dist: dspy>=3.2.1 ; extra == 'optimize'
|
|
38
39
|
Requires-Dist: scikit-learn>=1.8.0 ; extra == 'qc'
|
|
@@ -40,6 +41,7 @@ Requires-Dist: sentence-transformers>=5.3.0 ; extra == 'qc'
|
|
|
40
41
|
Requires-Dist: polars[rtcompat]>=1.40.1 ; extra == 'rt'
|
|
41
42
|
Provides-Extra: agent
|
|
42
43
|
Provides-Extra: aria2
|
|
44
|
+
Provides-Extra: distill
|
|
43
45
|
Provides-Extra: log
|
|
44
46
|
Provides-Extra: optimize
|
|
45
47
|
Provides-Extra: qc
|
|
@@ -58,6 +60,7 @@ Project-URL: Source, https://github.com/SkyeAv/Tablassert
|
|
|
58
60
|
# Tablassert
|
|
59
61
|
|
|
60
62
|
[](https://pypi.org/project/tablassert/)
|
|
63
|
+
[](https://pypi.org/project/tablassert/)
|
|
61
64
|
[](https://pypi.org/project/tablassert/)
|
|
62
65
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
63
66
|
[](https://github.com/SkyeAv/Tablassert/blob/main/LICENSE)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# Tablassert
|
|
2
2
|
|
|
3
3
|
[](https://pypi.org/project/tablassert/)
|
|
4
|
+
[](https://pypi.org/project/tablassert/)
|
|
4
5
|
[](https://pypi.org/project/tablassert/)
|
|
5
6
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
6
7
|
[](https://github.com/SkyeAv/Tablassert/blob/main/LICENSE)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "16.
|
|
3
|
+
version = "16.6.0"
|
|
4
4
|
description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -95,6 +95,9 @@ agent = [
|
|
|
95
95
|
optimize = [
|
|
96
96
|
"dspy>=3.2.1",
|
|
97
97
|
]
|
|
98
|
+
distill = [
|
|
99
|
+
"datasets>=3.0.0",
|
|
100
|
+
]
|
|
98
101
|
log = [
|
|
99
102
|
"loguru>=0.7.3",
|
|
100
103
|
]
|
|
@@ -216,11 +216,24 @@ struct MergeIndex {
|
|
|
216
216
|
/// - scalar fields: first-wins on conflict, counted;
|
|
217
217
|
/// - fields only on `incoming`: copied over (only a conflict when both sides disagree);
|
|
218
218
|
/// - `id` is never touched: both sides carry the same one by construction.
|
|
219
|
+
///
|
|
220
|
+
/// `number_of_cases` has one hardcoded exception to first-wins: when the MERGED record
|
|
221
|
+
/// carries `supporting_case_ids` (a build-internal `list[str]` of the case IDs behind
|
|
222
|
+
/// the count -- allowed onto edge frames so it reaches this pass, then stripped before
|
|
223
|
+
/// write) and either side carried a count, the count is recomputed as the length of the
|
|
224
|
+
/// unioned ID list. A case ID shared by both records is one case, so first-wins and
|
|
225
|
+
/// summing both over- and under-report; the union length is the exact count. The
|
|
226
|
+
/// superseded divergence is NOT reported as a scalar conflict. When neither side
|
|
227
|
+
/// carries the list, `number_of_cases` stays an ordinary first-wins scalar.
|
|
219
228
|
fn merge_records(stored: &mut Value, incoming: &Value) -> PyResult<u64> {
|
|
220
229
|
let Some(incoming_map) = incoming.as_object() else {
|
|
221
230
|
return Err(runtime_error("expected JSON object"));
|
|
222
231
|
};
|
|
223
232
|
let mut conflicts: u64 = 0;
|
|
233
|
+
// Read both counts BEFORE the fold: the fold may copy incoming's over a stored side
|
|
234
|
+
// that lacks it, and the recompute rule below needs to know each side contributed one.
|
|
235
|
+
let stored_cases: Option<Value> = stored.get("number_of_cases").cloned();
|
|
236
|
+
let incoming_cases: Option<Value> = incoming_map.get("number_of_cases").cloned();
|
|
224
237
|
let Some(stored_map) = stored.as_object_mut() else {
|
|
225
238
|
return Err(runtime_error("expected JSON object"));
|
|
226
239
|
};
|
|
@@ -259,9 +272,48 @@ fn merge_records(stored: &mut Value, incoming: &Value) -> PyResult<u64> {
|
|
|
259
272
|
}
|
|
260
273
|
}
|
|
261
274
|
}
|
|
275
|
+
// WHY: exact-unique `number_of_cases` semantics (see the docstring). Guarded on the
|
|
276
|
+
// merged record actually carrying the ID list: a one-sided carrier is fine (the union
|
|
277
|
+
// is just that side's list), while a carrier-less merge never recomputes.
|
|
278
|
+
if let Some(union_len) = stored
|
|
279
|
+
.get("supporting_case_ids")
|
|
280
|
+
.and_then(Value::as_array)
|
|
281
|
+
.map(Vec::len)
|
|
282
|
+
{
|
|
283
|
+
if stored_cases.is_some() || incoming_cases.is_some() {
|
|
284
|
+
// Undo the loop's conflict count when it fired on this very field: the
|
|
285
|
+
// recompute supersedes first-wins, so the divergence is not a conflict.
|
|
286
|
+
// Mirrors the loop's condition exactly -- both sides present, unequal, and
|
|
287
|
+
// not both arrays (two arrays took the union path and were never counted).
|
|
288
|
+
if let (Some(left), Some(right)) = (&stored_cases, &incoming_cases) {
|
|
289
|
+
if left != right && !(left.is_array() && right.is_array()) {
|
|
290
|
+
conflicts -= 1;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
let Some(map) = stored.as_object_mut() else {
|
|
294
|
+
return Err(runtime_error("expected JSON object"));
|
|
295
|
+
};
|
|
296
|
+
map.insert(
|
|
297
|
+
"number_of_cases".to_string(),
|
|
298
|
+
Value::Number(serde_json::Number::from(union_len)),
|
|
299
|
+
);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
262
302
|
Ok(conflicts)
|
|
263
303
|
}
|
|
264
304
|
|
|
305
|
+
/// Remove build-internal carrier fields from an edge record before it is written.
|
|
306
|
+
///
|
|
307
|
+
/// `supporting_case_ids` exists only so merge mode can recompute `number_of_cases`
|
|
308
|
+
/// (see `merge_records`); it must never ship in the final NDJSON, so EVERY edge write
|
|
309
|
+
/// path drops it -- the buffering merge pass and the default streaming path alike.
|
|
310
|
+
/// Nodes never carry it and are untouched.
|
|
311
|
+
fn strip_internal_edge_fields(value: &mut Value) {
|
|
312
|
+
if let Some(map) = value.as_object_mut() {
|
|
313
|
+
map.remove("supporting_case_ids");
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
265
317
|
impl MergeIndex {
|
|
266
318
|
fn absorb(&mut self, id: [u8; 16], value: Value, content: u64) -> PyResult<()> {
|
|
267
319
|
match self.records.entry(id) {
|
|
@@ -290,7 +342,8 @@ impl MergeIndex {
|
|
|
290
342
|
/// Merge-mode edge pass: buffer every unique edge, fold divergent same-id records into the
|
|
291
343
|
/// first, then write in first-seen order. Returns (divergent records merged, conflicting
|
|
292
344
|
/// scalar fields) for the summary log. Runs ONLY under `uuid_on_collision: merge`; the
|
|
293
|
-
/// default path stays streaming and never buffers a record.
|
|
345
|
+
/// default path stays streaming and never buffers a record. The build-internal
|
|
346
|
+
/// `supporting_case_ids` carrier is stripped from each record right before the write.
|
|
294
347
|
fn dedup_edges_merge(
|
|
295
348
|
reader: BufReader<File>,
|
|
296
349
|
mut writer: BufWriter<File>,
|
|
@@ -310,10 +363,13 @@ fn dedup_edges_merge(
|
|
|
310
363
|
};
|
|
311
364
|
index.absorb(edge_id_bytes(&value)?, value, content)?;
|
|
312
365
|
}
|
|
313
|
-
for id in &index.order {
|
|
314
|
-
let (_, value) =
|
|
366
|
+
for id in std::mem::take(&mut index.order) {
|
|
367
|
+
let Some((_, mut value)) = index.records.remove(&id) else {
|
|
368
|
+
continue;
|
|
369
|
+
};
|
|
370
|
+
strip_internal_edge_fields(&mut value);
|
|
315
371
|
writer
|
|
316
|
-
.write_all(&emitted_json_bytes(value).map_err(runtime_error)?)
|
|
372
|
+
.write_all(&emitted_json_bytes(&value).map_err(runtime_error)?)
|
|
317
373
|
.map_err(runtime_error)?;
|
|
318
374
|
writer.write_all(b"\n").map_err(runtime_error)?;
|
|
319
375
|
}
|
|
@@ -359,7 +415,7 @@ pub fn dedup_ndjson(
|
|
|
359
415
|
})
|
|
360
416
|
.map(|value| value.and_then(|value| finalize_record(value, is_edges, &domain, fields)))
|
|
361
417
|
.try_for_each(|record| -> PyResult<()> {
|
|
362
|
-
let Some(Finalized { value, content }) = record? else {
|
|
418
|
+
let Some(Finalized { mut value, content }) = record? else {
|
|
363
419
|
return Ok(());
|
|
364
420
|
};
|
|
365
421
|
let write: bool = if is_edges {
|
|
@@ -382,6 +438,9 @@ pub fn dedup_ndjson(
|
|
|
382
438
|
)
|
|
383
439
|
};
|
|
384
440
|
if write {
|
|
441
|
+
if is_edges {
|
|
442
|
+
strip_internal_edge_fields(&mut value);
|
|
443
|
+
}
|
|
385
444
|
writer
|
|
386
445
|
.write_all(&emitted_json_bytes(&value).map_err(runtime_error)?)
|
|
387
446
|
.map_err(runtime_error)?;
|
|
@@ -877,6 +936,162 @@ mod tests {
|
|
|
877
936
|
);
|
|
878
937
|
}
|
|
879
938
|
|
|
939
|
+
#[test]
|
|
940
|
+
fn merge_mode_recomputes_number_of_cases_from_case_id_union() {
|
|
941
|
+
// WHY: the hardcoded merge rule. Two builds of one edge (e.g. different FAERS
|
|
942
|
+
// quarters) each know their own case count and case IDs; first-wins would keep
|
|
943
|
+
// the left count and summing would double-count the shared case. The merged
|
|
944
|
+
// count is the size of the UNION of both `supporting_case_ids` lists -- exact.
|
|
945
|
+
let dir = tempdir().expect("tempdir");
|
|
946
|
+
let (input, output) = write_edges(
|
|
947
|
+
dir.path(),
|
|
948
|
+
concat!(
|
|
949
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":2,\"supporting_case_ids\":[\"case:1\",\"case:2\"]}\n",
|
|
950
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":5,\"supporting_case_ids\":[\"case:2\",\"case:3\"]}\n"
|
|
951
|
+
),
|
|
952
|
+
);
|
|
953
|
+
let (merged, conflicts) = dedup_ndjson(
|
|
954
|
+
input,
|
|
955
|
+
output.clone(),
|
|
956
|
+
true,
|
|
957
|
+
None,
|
|
958
|
+
spo_fields(),
|
|
959
|
+
Some("merge".to_string()),
|
|
960
|
+
)
|
|
961
|
+
.expect("merge dedup");
|
|
962
|
+
|
|
963
|
+
// The recomputed count supersedes the 2-vs-5 divergence: no scalar conflict.
|
|
964
|
+
assert_eq!((merged, conflicts), (1, 0));
|
|
965
|
+
let edge = merged_edge(&output);
|
|
966
|
+
// |{case:1, case:2} u {case:2, case:3}| = 3 -- the shared ID counts once.
|
|
967
|
+
assert_eq!(edge["number_of_cases"], serde_json::json!(3));
|
|
968
|
+
// The carrier is build-internal and must never ship.
|
|
969
|
+
assert!(edge.get("supporting_case_ids").is_none());
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
#[test]
|
|
973
|
+
fn merge_mode_one_sided_case_ids_still_recompute_the_count() {
|
|
974
|
+
// WHY: only one side carries `supporting_case_ids`. The union is that side's
|
|
975
|
+
// list, but the other side still contributed a `number_of_cases`, so the count
|
|
976
|
+
// is recomputed to the union length rather than kept from either record.
|
|
977
|
+
let dir = tempdir().expect("tempdir");
|
|
978
|
+
let (input, output) = write_edges(
|
|
979
|
+
dir.path(),
|
|
980
|
+
concat!(
|
|
981
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":7}\n",
|
|
982
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"supporting_case_ids\":[\"case:1\",\"case:2\"]}\n"
|
|
983
|
+
),
|
|
984
|
+
);
|
|
985
|
+
let (merged, conflicts) = dedup_ndjson(
|
|
986
|
+
input,
|
|
987
|
+
output.clone(),
|
|
988
|
+
true,
|
|
989
|
+
None,
|
|
990
|
+
spo_fields(),
|
|
991
|
+
Some("merge".to_string()),
|
|
992
|
+
)
|
|
993
|
+
.expect("merge dedup");
|
|
994
|
+
|
|
995
|
+
assert_eq!((merged, conflicts), (1, 0));
|
|
996
|
+
let edge = merged_edge(&output);
|
|
997
|
+
assert_eq!(edge["number_of_cases"], serde_json::json!(2));
|
|
998
|
+
assert!(edge.get("supporting_case_ids").is_none());
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
#[test]
|
|
1002
|
+
fn merge_mode_without_case_ids_keeps_first_wins_number_of_cases() {
|
|
1003
|
+
// WHY: the rule only fires when the carrier is present. With no
|
|
1004
|
+
// `supporting_case_ids` on either record, `number_of_cases` is an ordinary
|
|
1005
|
+
// first-wins scalar and the divergence is still counted as a conflict.
|
|
1006
|
+
let dir = tempdir().expect("tempdir");
|
|
1007
|
+
let (input, output) = write_edges(
|
|
1008
|
+
dir.path(),
|
|
1009
|
+
concat!(
|
|
1010
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":2}\n",
|
|
1011
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":5}\n"
|
|
1012
|
+
),
|
|
1013
|
+
);
|
|
1014
|
+
let (merged, conflicts) = dedup_ndjson(
|
|
1015
|
+
input,
|
|
1016
|
+
output.clone(),
|
|
1017
|
+
true,
|
|
1018
|
+
None,
|
|
1019
|
+
spo_fields(),
|
|
1020
|
+
Some("merge".to_string()),
|
|
1021
|
+
)
|
|
1022
|
+
.expect("merge dedup");
|
|
1023
|
+
|
|
1024
|
+
assert_eq!((merged, conflicts), (1, 1));
|
|
1025
|
+
assert_eq!(
|
|
1026
|
+
merged_edge(&output)["number_of_cases"],
|
|
1027
|
+
serde_json::json!(2)
|
|
1028
|
+
);
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
#[test]
|
|
1032
|
+
fn merge_mode_case_count_is_order_independent() {
|
|
1033
|
+
// WHY: the recomputed count derives from the sorted canonical-bytes list union,
|
|
1034
|
+
// so both arrival orders must produce byte-identical output, count included.
|
|
1035
|
+
let left_dir = tempdir().expect("tempdir");
|
|
1036
|
+
let (left_in, left_out) = write_edges(
|
|
1037
|
+
left_dir.path(),
|
|
1038
|
+
concat!(
|
|
1039
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":1,\"supporting_case_ids\":[\"case:2\"]}\n",
|
|
1040
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":2,\"supporting_case_ids\":[\"case:1\",\"case:3\"]}\n"
|
|
1041
|
+
),
|
|
1042
|
+
);
|
|
1043
|
+
dedup_ndjson(
|
|
1044
|
+
left_in,
|
|
1045
|
+
left_out.clone(),
|
|
1046
|
+
true,
|
|
1047
|
+
None,
|
|
1048
|
+
spo_fields(),
|
|
1049
|
+
Some("merge".to_string()),
|
|
1050
|
+
)
|
|
1051
|
+
.expect("merge dedup");
|
|
1052
|
+
|
|
1053
|
+
let right_dir = tempdir().expect("tempdir");
|
|
1054
|
+
let (right_in, right_out) = write_edges(
|
|
1055
|
+
right_dir.path(),
|
|
1056
|
+
concat!(
|
|
1057
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":2,\"supporting_case_ids\":[\"case:1\",\"case:3\"]}\n",
|
|
1058
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":1,\"supporting_case_ids\":[\"case:2\"]}\n"
|
|
1059
|
+
),
|
|
1060
|
+
);
|
|
1061
|
+
dedup_ndjson(
|
|
1062
|
+
right_in,
|
|
1063
|
+
right_out.clone(),
|
|
1064
|
+
true,
|
|
1065
|
+
None,
|
|
1066
|
+
spo_fields(),
|
|
1067
|
+
Some("merge".to_string()),
|
|
1068
|
+
)
|
|
1069
|
+
.expect("merge dedup");
|
|
1070
|
+
|
|
1071
|
+
let count = merged_edge(&right_out)["number_of_cases"].clone();
|
|
1072
|
+
assert_eq!(
|
|
1073
|
+
fs::read_to_string(left_out).expect("read left"),
|
|
1074
|
+
fs::read_to_string(right_out).expect("read right")
|
|
1075
|
+
);
|
|
1076
|
+
assert_eq!(count, serde_json::json!(3));
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
#[test]
|
|
1080
|
+
fn streaming_edges_never_ship_supporting_case_ids() {
|
|
1081
|
+
// WHY: the carrier is stripped on the default path too, so a graph that never
|
|
1082
|
+
// opts into merge mode still cannot leak the build-internal field.
|
|
1083
|
+
let dir = tempdir().expect("tempdir");
|
|
1084
|
+
let (input, output) = write_edges(
|
|
1085
|
+
dir.path(),
|
|
1086
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":2,\"supporting_case_ids\":[\"case:1\",\"case:2\"]}\n",
|
|
1087
|
+
);
|
|
1088
|
+
dedup_ndjson(input, output.clone(), true, None, spo_fields(), None).expect("dedup");
|
|
1089
|
+
|
|
1090
|
+
let edge = merged_edge(&output);
|
|
1091
|
+
assert_eq!(edge["number_of_cases"], serde_json::json!(2));
|
|
1092
|
+
assert!(edge.get("supporting_case_ids").is_none());
|
|
1093
|
+
}
|
|
1094
|
+
|
|
880
1095
|
#[test]
|
|
881
1096
|
fn merge_mode_writes_edges_in_first_seen_order() {
|
|
882
1097
|
// WHY: buffering must not reorder the graph -- the first record to claim each id
|