tablassert 16.4.0__tar.gz → 16.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {tablassert-16.4.0 → tablassert-16.6.0}/PKG-INFO +3 -1
  2. {tablassert-16.4.0 → tablassert-16.6.0}/pyproject.toml +4 -1
  3. {tablassert-16.4.0 → tablassert-16.6.0}/rust/src/ndjson.rs +220 -5
  4. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/agent.py +606 -63
  5. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/biolink.py +8 -0
  6. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/cli.py +62 -2
  7. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/coerce.py +411 -165
  8. tablassert-16.6.0/src/tablassert/distill.py +108 -0
  9. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/extras.py +2 -0
  10. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/lib.py +21 -19
  11. {tablassert-16.4.0 → tablassert-16.6.0}/LICENSE +0 -0
  12. {tablassert-16.4.0 → tablassert-16.6.0}/README.md +0 -0
  13. {tablassert-16.4.0 → tablassert-16.6.0}/rust/Cargo.lock +0 -0
  14. {tablassert-16.4.0 → tablassert-16.6.0}/rust/Cargo.toml +0 -0
  15. {tablassert-16.4.0 → tablassert-16.6.0}/rust/examples/count_tables.rs +0 -0
  16. {tablassert-16.4.0 → tablassert-16.6.0}/rust/src/fullmap.rs +0 -0
  17. {tablassert-16.4.0 → tablassert-16.6.0}/rust/src/json.rs +0 -0
  18. {tablassert-16.4.0 → tablassert-16.6.0}/rust/src/lib.rs +0 -0
  19. {tablassert-16.4.0 → tablassert-16.6.0}/rust/src/uuid.rs +0 -0
  20. {tablassert-16.4.0 → tablassert-16.6.0}/rust/tests/build_golden.rs +0 -0
  21. {tablassert-16.4.0 → tablassert-16.6.0}/rust/tests/common/mod.rs +0 -0
  22. {tablassert-16.4.0 → tablassert-16.6.0}/rust/tests/extract_prebuilt.rs +0 -0
  23. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/__init__.py +0 -0
  24. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/_lazy.py +0 -0
  25. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/enums.py +0 -0
  26. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/errors.py +0 -0
  27. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/fullmap.py +0 -0
  28. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/graph_target.py +0 -0
  29. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/ingests.py +0 -0
  30. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/log.py +0 -0
  31. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/models.py +0 -0
  32. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/nlp.py +0 -0
  33. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/progress.py +0 -0
  34. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/qc.py +0 -0
  35. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/rig.py +0 -0
  36. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/rs.pyi +0 -0
  37. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/study.py +0 -0
  38. {tablassert-16.4.0 → tablassert-16.6.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 16.4.0
3
+ Version: 16.6.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -33,6 +33,7 @@ Requires-Dist: fastexcel>=0.20.2
33
33
  Requires-Dist: smolagents>=1.26.0 ; extra == 'agent'
34
34
  Requires-Dist: litellm>=1.93.0 ; extra == 'agent'
35
35
  Requires-Dist: aria2==0.0.1b0 ; extra == 'aria2'
36
+ Requires-Dist: datasets>=3.0.0 ; extra == 'distill'
36
37
  Requires-Dist: loguru>=0.7.3 ; extra == 'log'
37
38
  Requires-Dist: dspy>=3.2.1 ; extra == 'optimize'
38
39
  Requires-Dist: scikit-learn>=1.8.0 ; extra == 'qc'
@@ -40,6 +41,7 @@ Requires-Dist: sentence-transformers>=5.3.0 ; extra == 'qc'
40
41
  Requires-Dist: polars[rtcompat]>=1.40.1 ; extra == 'rt'
41
42
  Provides-Extra: agent
42
43
  Provides-Extra: aria2
44
+ Provides-Extra: distill
43
45
  Provides-Extra: log
44
46
  Provides-Extra: optimize
45
47
  Provides-Extra: qc
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "16.4.0"
3
+ version = "16.6.0"
4
4
  description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -95,6 +95,9 @@ agent = [
95
95
  optimize = [
96
96
  "dspy>=3.2.1",
97
97
  ]
98
+ distill = [
99
+ "datasets>=3.0.0",
100
+ ]
98
101
  log = [
99
102
  "loguru>=0.7.3",
100
103
  ]
@@ -216,11 +216,24 @@ struct MergeIndex {
216
216
  /// - scalar fields: first-wins on conflict, counted;
217
217
  /// - fields only on `incoming`: copied over (only a conflict when both sides disagree);
218
218
  /// - `id` is never touched: both sides carry the same one by construction.
219
+ ///
220
+ /// `number_of_cases` has one hardcoded exception to first-wins: when the MERGED record
221
+ /// carries `supporting_case_ids` (a build-internal `list[str]` of the case IDs behind
222
+ /// the count -- allowed onto edge frames so it reaches this pass, then stripped before
223
+ /// write) and either side carried a count, the count is recomputed as the length of the
224
+ /// unioned ID list. A case ID shared by both records is one case, so first-wins and
225
+ /// summing both over- and under-report; the union length is the exact count. The
226
+ /// superseded divergence is NOT reported as a scalar conflict. When neither side
227
+ /// carries the list, `number_of_cases` stays an ordinary first-wins scalar.
219
228
  fn merge_records(stored: &mut Value, incoming: &Value) -> PyResult<u64> {
220
229
  let Some(incoming_map) = incoming.as_object() else {
221
230
  return Err(runtime_error("expected JSON object"));
222
231
  };
223
232
  let mut conflicts: u64 = 0;
233
+ // Read both counts BEFORE the fold: the fold may copy incoming's over a stored side
234
+ // that lacks it, and the recompute rule below needs to know each side contributed one.
235
+ let stored_cases: Option<Value> = stored.get("number_of_cases").cloned();
236
+ let incoming_cases: Option<Value> = incoming_map.get("number_of_cases").cloned();
224
237
  let Some(stored_map) = stored.as_object_mut() else {
225
238
  return Err(runtime_error("expected JSON object"));
226
239
  };
@@ -259,9 +272,48 @@ fn merge_records(stored: &mut Value, incoming: &Value) -> PyResult<u64> {
259
272
  }
260
273
  }
261
274
  }
275
+ // WHY: exact-unique `number_of_cases` semantics (see the docstring). Guarded on the
276
+ // merged record actually carrying the ID list: a one-sided carrier is fine (the union
277
+ // is just that side's list), while a carrier-less merge never recomputes.
278
+ if let Some(union_len) = stored
279
+ .get("supporting_case_ids")
280
+ .and_then(Value::as_array)
281
+ .map(Vec::len)
282
+ {
283
+ if stored_cases.is_some() || incoming_cases.is_some() {
284
+ // Undo the loop's conflict count when it fired on this very field: the
285
+ // recompute supersedes first-wins, so the divergence is not a conflict.
286
+ // Mirrors the loop's condition exactly -- both sides present, unequal, and
287
+ // not both arrays (two arrays took the union path and were never counted).
288
+ if let (Some(left), Some(right)) = (&stored_cases, &incoming_cases) {
289
+ if left != right && !(left.is_array() && right.is_array()) {
290
+ conflicts -= 1;
291
+ }
292
+ }
293
+ let Some(map) = stored.as_object_mut() else {
294
+ return Err(runtime_error("expected JSON object"));
295
+ };
296
+ map.insert(
297
+ "number_of_cases".to_string(),
298
+ Value::Number(serde_json::Number::from(union_len)),
299
+ );
300
+ }
301
+ }
262
302
  Ok(conflicts)
263
303
  }
264
304
 
305
+ /// Remove build-internal carrier fields from an edge record before it is written.
306
+ ///
307
+ /// `supporting_case_ids` exists only so merge mode can recompute `number_of_cases`
308
+ /// (see `merge_records`); it must never ship in the final NDJSON, so EVERY edge write
309
+ /// path drops it -- the buffering merge pass and the default streaming path alike.
310
+ /// Nodes never carry it and are untouched.
311
+ fn strip_internal_edge_fields(value: &mut Value) {
312
+ if let Some(map) = value.as_object_mut() {
313
+ map.remove("supporting_case_ids");
314
+ }
315
+ }
316
+
265
317
  impl MergeIndex {
266
318
  fn absorb(&mut self, id: [u8; 16], value: Value, content: u64) -> PyResult<()> {
267
319
  match self.records.entry(id) {
@@ -290,7 +342,8 @@ impl MergeIndex {
290
342
  /// Merge-mode edge pass: buffer every unique edge, fold divergent same-id records into the
291
343
  /// first, then write in first-seen order. Returns (divergent records merged, conflicting
292
344
  /// scalar fields) for the summary log. Runs ONLY under `uuid_on_collision: merge`; the
293
- /// default path stays streaming and never buffers a record.
345
+ /// default path stays streaming and never buffers a record. The build-internal
346
+ /// `supporting_case_ids` carrier is stripped from each record right before the write.
294
347
  fn dedup_edges_merge(
295
348
  reader: BufReader<File>,
296
349
  mut writer: BufWriter<File>,
@@ -310,10 +363,13 @@ fn dedup_edges_merge(
310
363
  };
311
364
  index.absorb(edge_id_bytes(&value)?, value, content)?;
312
365
  }
313
- for id in &index.order {
314
- let (_, value) = &index.records[id];
366
+ for id in std::mem::take(&mut index.order) {
367
+ let Some((_, mut value)) = index.records.remove(&id) else {
368
+ continue;
369
+ };
370
+ strip_internal_edge_fields(&mut value);
315
371
  writer
316
- .write_all(&emitted_json_bytes(value).map_err(runtime_error)?)
372
+ .write_all(&emitted_json_bytes(&value).map_err(runtime_error)?)
317
373
  .map_err(runtime_error)?;
318
374
  writer.write_all(b"\n").map_err(runtime_error)?;
319
375
  }
@@ -359,7 +415,7 @@ pub fn dedup_ndjson(
359
415
  })
360
416
  .map(|value| value.and_then(|value| finalize_record(value, is_edges, &domain, fields)))
361
417
  .try_for_each(|record| -> PyResult<()> {
362
- let Some(Finalized { value, content }) = record? else {
418
+ let Some(Finalized { mut value, content }) = record? else {
363
419
  return Ok(());
364
420
  };
365
421
  let write: bool = if is_edges {
@@ -382,6 +438,9 @@ pub fn dedup_ndjson(
382
438
  )
383
439
  };
384
440
  if write {
441
+ if is_edges {
442
+ strip_internal_edge_fields(&mut value);
443
+ }
385
444
  writer
386
445
  .write_all(&emitted_json_bytes(&value).map_err(runtime_error)?)
387
446
  .map_err(runtime_error)?;
@@ -877,6 +936,162 @@ mod tests {
877
936
  );
878
937
  }
879
938
 
939
+ #[test]
940
+ fn merge_mode_recomputes_number_of_cases_from_case_id_union() {
941
+ // WHY: the hardcoded merge rule. Two builds of one edge (e.g. different FAERS
942
+ // quarters) each know their own case count and case IDs; first-wins would keep
943
+ // the left count and summing would double-count the shared case. The merged
944
+ // count is the size of the UNION of both `supporting_case_ids` lists -- exact.
945
+ let dir = tempdir().expect("tempdir");
946
+ let (input, output) = write_edges(
947
+ dir.path(),
948
+ concat!(
949
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":2,\"supporting_case_ids\":[\"case:1\",\"case:2\"]}\n",
950
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":5,\"supporting_case_ids\":[\"case:2\",\"case:3\"]}\n"
951
+ ),
952
+ );
953
+ let (merged, conflicts) = dedup_ndjson(
954
+ input,
955
+ output.clone(),
956
+ true,
957
+ None,
958
+ spo_fields(),
959
+ Some("merge".to_string()),
960
+ )
961
+ .expect("merge dedup");
962
+
963
+ // The recomputed count supersedes the 2-vs-5 divergence: no scalar conflict.
964
+ assert_eq!((merged, conflicts), (1, 0));
965
+ let edge = merged_edge(&output);
966
+ // |{case:1, case:2} u {case:2, case:3}| = 3 -- the shared ID counts once.
967
+ assert_eq!(edge["number_of_cases"], serde_json::json!(3));
968
+ // The carrier is build-internal and must never ship.
969
+ assert!(edge.get("supporting_case_ids").is_none());
970
+ }
971
+
972
+ #[test]
973
+ fn merge_mode_one_sided_case_ids_still_recompute_the_count() {
974
+ // WHY: only one side carries `supporting_case_ids`. The union is that side's
975
+ // list, but the other side still contributed a `number_of_cases`, so the count
976
+ // is recomputed to the union length rather than kept from either record.
977
+ let dir = tempdir().expect("tempdir");
978
+ let (input, output) = write_edges(
979
+ dir.path(),
980
+ concat!(
981
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":7}\n",
982
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"supporting_case_ids\":[\"case:1\",\"case:2\"]}\n"
983
+ ),
984
+ );
985
+ let (merged, conflicts) = dedup_ndjson(
986
+ input,
987
+ output.clone(),
988
+ true,
989
+ None,
990
+ spo_fields(),
991
+ Some("merge".to_string()),
992
+ )
993
+ .expect("merge dedup");
994
+
995
+ assert_eq!((merged, conflicts), (1, 0));
996
+ let edge = merged_edge(&output);
997
+ assert_eq!(edge["number_of_cases"], serde_json::json!(2));
998
+ assert!(edge.get("supporting_case_ids").is_none());
999
+ }
1000
+
1001
+ #[test]
1002
+ fn merge_mode_without_case_ids_keeps_first_wins_number_of_cases() {
1003
+ // WHY: the rule only fires when the carrier is present. With no
1004
+ // `supporting_case_ids` on either record, `number_of_cases` is an ordinary
1005
+ // first-wins scalar and the divergence is still counted as a conflict.
1006
+ let dir = tempdir().expect("tempdir");
1007
+ let (input, output) = write_edges(
1008
+ dir.path(),
1009
+ concat!(
1010
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":2}\n",
1011
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":5}\n"
1012
+ ),
1013
+ );
1014
+ let (merged, conflicts) = dedup_ndjson(
1015
+ input,
1016
+ output.clone(),
1017
+ true,
1018
+ None,
1019
+ spo_fields(),
1020
+ Some("merge".to_string()),
1021
+ )
1022
+ .expect("merge dedup");
1023
+
1024
+ assert_eq!((merged, conflicts), (1, 1));
1025
+ assert_eq!(
1026
+ merged_edge(&output)["number_of_cases"],
1027
+ serde_json::json!(2)
1028
+ );
1029
+ }
1030
+
1031
+ #[test]
1032
+ fn merge_mode_case_count_is_order_independent() {
1033
+ // WHY: the recomputed count derives from the sorted canonical-bytes list union,
1034
+ // so both arrival orders must produce byte-identical output, count included.
1035
+ let left_dir = tempdir().expect("tempdir");
1036
+ let (left_in, left_out) = write_edges(
1037
+ left_dir.path(),
1038
+ concat!(
1039
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":1,\"supporting_case_ids\":[\"case:2\"]}\n",
1040
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":2,\"supporting_case_ids\":[\"case:1\",\"case:3\"]}\n"
1041
+ ),
1042
+ );
1043
+ dedup_ndjson(
1044
+ left_in,
1045
+ left_out.clone(),
1046
+ true,
1047
+ None,
1048
+ spo_fields(),
1049
+ Some("merge".to_string()),
1050
+ )
1051
+ .expect("merge dedup");
1052
+
1053
+ let right_dir = tempdir().expect("tempdir");
1054
+ let (right_in, right_out) = write_edges(
1055
+ right_dir.path(),
1056
+ concat!(
1057
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":2,\"supporting_case_ids\":[\"case:1\",\"case:3\"]}\n",
1058
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":1,\"supporting_case_ids\":[\"case:2\"]}\n"
1059
+ ),
1060
+ );
1061
+ dedup_ndjson(
1062
+ right_in,
1063
+ right_out.clone(),
1064
+ true,
1065
+ None,
1066
+ spo_fields(),
1067
+ Some("merge".to_string()),
1068
+ )
1069
+ .expect("merge dedup");
1070
+
1071
+ let count = merged_edge(&right_out)["number_of_cases"].clone();
1072
+ assert_eq!(
1073
+ fs::read_to_string(left_out).expect("read left"),
1074
+ fs::read_to_string(right_out).expect("read right")
1075
+ );
1076
+ assert_eq!(count, serde_json::json!(3));
1077
+ }
1078
+
1079
+ #[test]
1080
+ fn streaming_edges_never_ship_supporting_case_ids() {
1081
+ // WHY: the carrier is stripped on the default path too, so a graph that never
1082
+ // opts into merge mode still cannot leak the build-internal field.
1083
+ let dir = tempdir().expect("tempdir");
1084
+ let (input, output) = write_edges(
1085
+ dir.path(),
1086
+ "{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"number_of_cases\":2,\"supporting_case_ids\":[\"case:1\",\"case:2\"]}\n",
1087
+ );
1088
+ dedup_ndjson(input, output.clone(), true, None, spo_fields(), None).expect("dedup");
1089
+
1090
+ let edge = merged_edge(&output);
1091
+ assert_eq!(edge["number_of_cases"], serde_json::json!(2));
1092
+ assert!(edge.get("supporting_case_ids").is_none());
1093
+ }
1094
+
880
1095
  #[test]
881
1096
  fn merge_mode_writes_edges_in_first_seen_order() {
882
1097
  // WHY: buffering must not reorder the graph -- the first record to claim each id