tablassert 19.0.1__tar.gz → 19.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {tablassert-19.0.1 → tablassert-19.2.0}/PKG-INFO +5 -4
  2. {tablassert-19.0.1 → tablassert-19.2.0}/README.md +4 -3
  3. {tablassert-19.0.1 → tablassert-19.2.0}/pyproject.toml +1 -1
  4. {tablassert-19.0.1 → tablassert-19.2.0}/rust/src/ndjson.rs +132 -32
  5. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/biolink.py +83 -12
  6. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/cli.py +127 -1
  7. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/fullmap.py +104 -1
  8. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/lib.py +28 -1
  9. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/models.py +4 -0
  10. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/qc.py +37 -8
  11. {tablassert-19.0.1 → tablassert-19.2.0}/LICENSE +0 -0
  12. {tablassert-19.0.1 → tablassert-19.2.0}/rust/Cargo.lock +0 -0
  13. {tablassert-19.0.1 → tablassert-19.2.0}/rust/Cargo.toml +0 -0
  14. {tablassert-19.0.1 → tablassert-19.2.0}/rust/examples/count_tables.rs +0 -0
  15. {tablassert-19.0.1 → tablassert-19.2.0}/rust/src/fullmap.rs +0 -0
  16. {tablassert-19.0.1 → tablassert-19.2.0}/rust/src/json.rs +0 -0
  17. {tablassert-19.0.1 → tablassert-19.2.0}/rust/src/lib.rs +0 -0
  18. {tablassert-19.0.1 → tablassert-19.2.0}/rust/src/nlp.rs +0 -0
  19. {tablassert-19.0.1 → tablassert-19.2.0}/rust/src/uuid.rs +0 -0
  20. {tablassert-19.0.1 → tablassert-19.2.0}/rust/tests/build_golden.rs +0 -0
  21. {tablassert-19.0.1 → tablassert-19.2.0}/rust/tests/common/mod.rs +0 -0
  22. {tablassert-19.0.1 → tablassert-19.2.0}/rust/tests/extract_prebuilt.rs +0 -0
  23. {tablassert-19.0.1 → tablassert-19.2.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
  24. {tablassert-19.0.1 → tablassert-19.2.0}/rust/tests/nlp_golden.rs +0 -0
  25. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/__init__.py +0 -0
  26. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/_lazy.py +0 -0
  27. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/agent.py +0 -0
  28. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/cli_entry.py +0 -0
  29. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/coerce.py +0 -0
  30. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
  31. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/distill.py +0 -0
  32. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/distill_reward.py +0 -0
  33. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/enums.py +0 -0
  34. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/errors.py +0 -0
  35. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/extras.py +0 -0
  36. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/graph_target.py +0 -0
  37. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/ingests.py +0 -0
  38. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/log.py +0 -0
  39. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/net.py +0 -0
  40. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/nlp.py +0 -0
  41. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/progress.py +0 -0
  42. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/rig.py +0 -0
  43. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/rs.pyi +0 -0
  44. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/runcache.py +0 -0
  45. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/study.py +0 -0
  46. {tablassert-19.0.1 → tablassert-19.2.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 19.0.1
3
+ Version: 19.2.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -60,10 +60,11 @@ Project-URL: Source, https://github.com/SkyeAv/Tablassert
60
60
 
61
61
  # Tablassert
62
62
 
63
- [![PyPI](https://img.shields.io/pypi/v/tablassert.svg)](https://pypi.org/project/tablassert/)
64
- [![Python](https://img.shields.io/pypi/pyversions/tablassert.svg)](https://pypi.org/project/tablassert/)
63
+ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
64
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
65
+ [![PyPI version](https://badge.fury.io/py/tablassert.svg)](https://badge.fury.io/py/tablassert)
66
+ [![Downloads](https://static.pepy.tech/badge/tablassert/month)](https://pepy.tech/project/tablassert)
65
67
  [![CI](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml/badge.svg)](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
66
- [![License](https://img.shields.io/pypi/l/tablassert.svg)](https://github.com/SkyeAv/Tablassert/blob/main/LICENSE)
67
68
  [![Docs](https://img.shields.io/github/deployments/SkyeAv/Tablassert/github-pages?label=docs)](https://skyeav.github.io/Tablassert/)
68
69
  [![GitHub stars](https://img.shields.io/github/stars/SkyeAv/Tablassert.svg)](https://github.com/SkyeAv/Tablassert/stargazers)
69
70
 
@@ -1,9 +1,10 @@
1
1
  # Tablassert
2
2
 
3
- [![PyPI](https://img.shields.io/pypi/v/tablassert.svg)](https://pypi.org/project/tablassert/)
4
- [![Python](https://img.shields.io/pypi/pyversions/tablassert.svg)](https://pypi.org/project/tablassert/)
3
+ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
4
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
5
+ [![PyPI version](https://badge.fury.io/py/tablassert.svg)](https://badge.fury.io/py/tablassert)
6
+ [![Downloads](https://static.pepy.tech/badge/tablassert/month)](https://pepy.tech/project/tablassert)
5
7
  [![CI](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml/badge.svg)](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
6
- [![License](https://img.shields.io/pypi/l/tablassert.svg)](https://github.com/SkyeAv/Tablassert/blob/main/LICENSE)
7
8
  [![Docs](https://img.shields.io/github/deployments/SkyeAv/Tablassert/github-pages?label=docs)](https://skyeav.github.io/Tablassert/)
8
9
  [![GitHub stars](https://img.shields.io/github/stars/SkyeAv/Tablassert.svg)](https://github.com/SkyeAv/Tablassert/stargazers)
9
10
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "19.0.1"
3
+ version = "19.2.0"
4
4
  description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -5,7 +5,7 @@ use pyo3::prelude::*;
5
5
  use rustc_hash::{FxHashMap, FxHashSet};
6
6
  use serde_json::Value;
7
7
  use std::collections::hash_map::Entry;
8
- use std::fs::File;
8
+ use std::fs::{self, File};
9
9
  use std::io::{BufRead, BufReader, BufWriter, Write};
10
10
  use std::path::{Path, PathBuf};
11
11
  use std::rc::Rc;
@@ -299,8 +299,9 @@ impl ListState {
299
299
  /// membership are hash sets, and the per-fold re-canonicalization + re-sort of every
300
300
  /// stored list item is gone -- lists sort ONCE, at write-out, and only if a union ran.
301
301
  struct MergedRecord {
302
- /// The live record: scalars first-wins, lists in original/append order until the
303
- /// deferred write-out sort.
302
+ /// The live record: conflicting scalars keep the lexicographically smallest canonical
303
+ /// value (see `merge_records`), lists in original/append order until the deferred
304
+ /// write-out sort.
304
305
  value: Value,
305
306
  /// Content hashes of every record absorbed into this id: O(1) exact-repeat
306
307
  /// suppression. Bare hashes keep the pre-US-002 `Vec<u64>` semantics exactly (a false
@@ -311,6 +312,9 @@ struct MergedRecord {
311
312
  lists: FxHashMap<String, ListState>,
312
313
  /// Distinct non-empty scalar source values for each `original_*` field.
313
314
  original_values: OriginalValues,
315
+ /// Conflicting scalar field names, one entry per fold conflict (with multiplicity).
316
+ /// Consumed at write-out by the per-predicate fold report.
317
+ conflict_fields: Vec<String>,
314
318
  }
315
319
 
316
320
  impl MergedRecord {
@@ -332,6 +336,7 @@ impl MergedRecord {
332
336
  value,
333
337
  hashes,
334
338
  lists,
339
+ conflict_fields: Vec::new(),
335
340
  })
336
341
  }
337
342
 
@@ -387,8 +392,9 @@ impl MergedRecord {
387
392
  }
388
393
  }
389
394
 
390
- /// Fold `incoming` into `stored`, field-wise. Returns the number of conflicting scalar
391
- /// fields (kept first-wins) so the caller can report them.
395
+ /// Fold `incoming` into `stored`, field-wise. Returns the conflicting scalar field names
396
+ /// (one entry per conflict, reported for the base record) so the caller can build the fold
397
+ /// report.
392
398
  ///
393
399
  /// Near-linear implementation of the semantics frozen by `merge_records_reference` and
394
400
  /// policed by `merge_fold_matches_reference_on_fuzz`:
@@ -398,25 +404,32 @@ impl MergedRecord {
398
404
  /// checked against the field's `ListState` set in O(1); stored items are never
399
405
  /// re-canonicalized, and the sort by canonical bytes is deferred to write-out
400
406
  /// (`MergedRecord::finish`), where it runs only for fields that saw a real union;
401
- /// - scalar fields: first-wins on conflict, counted, except `original_*` fields;
407
+ /// - scalar fields: a conflicting scalar keeps the LEXICOGRAPHICALLY SMALLEST canonical
408
+ /// value -- a commutative, associative winner, so the merged record is identical under
409
+ /// any arrival order -- and each conflict is reported by field name for the fold report,
410
+ /// except `original_*` fields;
402
411
  /// - `original_*` fields collect distinct non-empty strings and render sorted values as
403
412
  /// `A`, `A|B`, or `A|B|C`;
404
413
  /// - fields only on `incoming`: copied over (only a conflict when both sides disagree);
405
414
  /// - `id` is never touched: both sides carry the same one by construction.
406
415
  ///
407
- /// `number_of_cases` has one hardcoded exception to first-wins: when the MERGED record
408
- /// carries `supporting_case_ids` (a build-internal `list[str]` of the case IDs behind
409
- /// the count -- allowed onto edge frames so it reaches this pass, then stripped before
410
- /// write) and either side carried a count, the count is recomputed as the length of the
411
- /// unioned ID list. A case ID shared by both records is one case, so first-wins and
412
- /// summing both over- and under-report; the union length is the exact count. The
413
- /// superseded divergence is NOT reported as a scalar conflict. When neither side
414
- /// carries the list, `number_of_cases` stays an ordinary first-wins scalar.
415
- fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<u64> {
416
+ /// `number_of_cases` has one hardcoded exception to the smallest-value rule: when the
417
+ /// MERGED record carries `supporting_case_ids` (a build-internal `list[str]` of the case
418
+ /// IDs behind the count -- allowed onto edge frames so it reaches this pass, then stripped
419
+ /// before write) and either side carried a count, the count is recomputed as the length of
420
+ /// the unioned ID list. A case ID shared by both records is one case, so smallest-value
421
+ /// and summing both over- and under-report; the union length is the exact count. The
422
+ /// superseded divergence is NOT reported as a scalar conflict. When neither side carries
423
+ /// the list, `number_of_cases` stays an ordinary smallest-value scalar.
424
+ fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<Vec<String>> {
416
425
  let Some(incoming_map) = incoming.as_object() else {
417
426
  return Err(runtime_error("expected JSON object"));
418
427
  };
419
- let mut conflicts: u64 = 0;
428
+ let mut conflict_fields: Vec<String> = Vec::new();
429
+ // Winner overrides are collected during the loop and applied after it: the stored
430
+ // value's mutable borrow is live while a conflict is detected, and the winner needs
431
+ // the canonical bytes of BOTH sides compared.
432
+ let mut winner_overrides: Vec<(String, Value)> = Vec::new();
420
433
  // Read both counts BEFORE the fold: the fold may copy incoming's over a stored side
421
434
  // that lacks it, and the recompute rule below needs to know each side contributed one.
422
435
  let stored_cases: Option<Value> = stored.value.get("number_of_cases").cloned();
@@ -487,14 +500,37 @@ fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<u64> {
487
500
  .map(str::to_owned),
488
501
  );
489
502
  } else if stored_value != incoming_value {
490
- conflicts += 1;
503
+ conflict_fields.push(key.clone());
504
+ let stored_wins = {
505
+ let stored_bytes =
506
+ canonical_json_bytes(stored_value).map_err(runtime_error)?;
507
+ let incoming_bytes =
508
+ canonical_json_bytes(incoming_value).map_err(runtime_error)?;
509
+ stored_bytes <= incoming_bytes
510
+ };
511
+ if !stored_wins {
512
+ winner_overrides.push((key.clone(), incoming_value.clone()));
513
+ }
491
514
  }
492
515
  } else if stored_value != incoming_value {
493
- conflicts += 1;
516
+ conflict_fields.push(key.clone());
517
+ let stored_wins = {
518
+ let stored_bytes =
519
+ canonical_json_bytes(stored_value).map_err(runtime_error)?;
520
+ let incoming_bytes =
521
+ canonical_json_bytes(incoming_value).map_err(runtime_error)?;
522
+ stored_bytes <= incoming_bytes
523
+ };
524
+ if !stored_wins {
525
+ winner_overrides.push((key.clone(), incoming_value.clone()));
526
+ }
494
527
  }
495
528
  }
496
529
  }
497
530
  }
531
+ for (key, value) in winner_overrides {
532
+ stored_map.insert(key, value);
533
+ }
498
534
  // WHY: exact-unique `number_of_cases` semantics (see the docstring). Guarded on the
499
535
  // merged record actually carrying the ID list: a one-sided carrier is fine (the union
500
536
  // is just that side's list), while a carrier-less merge never recomputes.
@@ -511,7 +547,17 @@ fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<u64> {
511
547
  // not both arrays (two arrays took the union path and were never counted).
512
548
  if let (Some(left), Some(right)) = (&stored_cases, &incoming_cases) {
513
549
  if left != right && !(left.is_array() && right.is_array()) {
514
- conflicts -= 1;
550
+ // The superseded divergence was counted by an EARLIER fold when the
551
+ // counts first collided (not necessarily this one), so the pop is
552
+ // unconditional on this fold's pushes: at most one live
553
+ // number_of_cases conflict can exist per merged record, and the
554
+ // recompute retires it.
555
+ if let Some(pos) = conflict_fields
556
+ .iter()
557
+ .rposition(|field| field == "number_of_cases")
558
+ {
559
+ conflict_fields.remove(pos);
560
+ }
515
561
  }
516
562
  }
517
563
  let Some(map) = stored.value.as_object_mut() else {
@@ -523,7 +569,7 @@ fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<u64> {
523
569
  );
524
570
  }
525
571
  }
526
- Ok(conflicts)
572
+ Ok(conflict_fields)
527
573
  }
528
574
 
529
575
  /// Remove build-internal carrier fields from an edge record before it is written.
@@ -555,8 +601,10 @@ impl MergeIndex {
555
601
  return Ok(());
556
602
  }
557
603
  self.merged += 1;
558
- self.scalar_conflicts += merge_records(record, &value)?;
604
+ let fields: Vec<String> = merge_records(record, &value)?;
605
+ self.scalar_conflicts += fields.len() as u64;
559
606
  record.hashes.insert(content);
607
+ record.conflict_fields.extend(fields);
560
608
  Ok(())
561
609
  }
562
610
  }
@@ -573,7 +621,8 @@ impl MergeIndex {
573
621
  /// Original semantics doc:
574
622
  ///
575
623
  /// Fold `incoming` into `stored`, field-wise. Returns the number of conflicting scalar
576
- /// fields (kept first-wins) so the caller can report them.
624
+ /// fields (the base record's value wins; see `merge_records`) so the caller can report
625
+ /// them.
577
626
  ///
578
627
  /// - list fields: union, deduped by canonical JSON bytes (so two `sources` objects that
579
628
  /// differ only in key order collapse), then sorted by canonical bytes so the merged
@@ -582,20 +631,21 @@ impl MergeIndex {
582
631
  /// - fields only on `incoming`: copied over (only a conflict when both sides disagree);
583
632
  /// - `id` is never touched: both sides carry the same one by construction.
584
633
  ///
585
- /// `number_of_cases` has one hardcoded exception to first-wins: when the MERGED record
634
+ /// `number_of_cases` has one hardcoded exception to base-wins: when the MERGED record
586
635
  /// carries `supporting_case_ids` (a build-internal `list[str]` of the case IDs behind
587
636
  /// the count -- allowed onto edge frames so it reaches this pass, then stripped before
588
637
  /// write) and either side carried a count, the count is recomputed as the length of the
589
- /// unioned ID list. A case ID shared by both records is one case, so first-wins and
638
+ /// unioned ID list. A case ID shared by both records is one case, so base-wins and
590
639
  /// summing both over- and under-report; the union length is the exact count. The
591
640
  /// superseded divergence is NOT reported as a scalar conflict. When neither side
592
- /// carries the list, `number_of_cases` stays an ordinary first-wins scalar.
641
+ /// carries the list, `number_of_cases` stays an ordinary base-wins scalar.
593
642
  #[cfg(test)]
594
643
  fn merge_records_reference(stored: &mut Value, incoming: &Value) -> PyResult<u64> {
595
644
  let Some(incoming_map) = incoming.as_object() else {
596
645
  return Err(runtime_error("expected JSON object"));
597
646
  };
598
647
  let mut conflicts: u64 = 0;
648
+ let mut winner_overrides: Vec<(String, Value)> = Vec::new();
599
649
  let mut original_values: OriginalValues = collect_original_values(stored);
600
650
  for (key, values) in collect_original_values(incoming) {
601
651
  original_values.entry(key).or_default().extend(values);
@@ -640,10 +690,19 @@ fn merge_records_reference(stored: &mut Value, incoming: &Value) -> PyResult<u64
640
690
  // Aggregated and rendered after this fold, once all values are known.
641
691
  } else if stored_value != incoming_value {
642
692
  conflicts += 1;
693
+ let stored_bytes = canonical_json_bytes(stored_value).map_err(runtime_error)?;
694
+ let incoming_bytes =
695
+ canonical_json_bytes(incoming_value).map_err(runtime_error)?;
696
+ if incoming_bytes < stored_bytes {
697
+ winner_overrides.push((key.clone(), incoming_value.clone()));
698
+ }
643
699
  }
644
700
  }
645
701
  }
646
702
  }
703
+ for (key, value) in winner_overrides {
704
+ stored_map.insert(key, value);
705
+ }
647
706
  if let Some(map) = stored.as_object_mut() {
648
707
  for (key, values) in &original_values {
649
708
  if !values.is_empty() {
@@ -684,6 +743,12 @@ fn merge_records_reference(stored: &mut Value, incoming: &Value) -> PyResult<u64
684
743
  /// FROZEN EQUIVALENCE ORACLE -- BYTE-VERBATIM copy of the pre-US-002 `MergeIndex` absorb
685
744
  /// path (state shape included), compiled ONLY under `cfg(test)`; drives
686
745
  /// `merge_records_reference`. Treat as read-only -- see that fn's docs.
746
+ ///
747
+ /// AMENDED once, deliberately (fold determinism): the oracle's base election now keeps the
748
+ /// record with the smallest canonical bytes as the fold base, exactly mirroring the
749
+ /// production `MergeIndex::absorb`, so both sides still demand byte-identical merged
750
+ /// output under any arrival order. This is a semantic amendment agreed for the fold
751
+ /// report work, not an optimization -- the oracle remains the algorithm of record.
687
752
  #[cfg(test)]
688
753
  #[derive(Default)]
689
754
  struct MergeIndexReference {
@@ -719,17 +784,24 @@ impl MergeIndexReference {
719
784
  }
720
785
  }
721
786
 
722
- /// Merge-mode edge pass: buffer every unique edge, fold divergent same-id records into the
723
- /// first, then write in first-seen order. Returns (divergent records merged, conflicting
724
- /// scalar fields) for the summary log. Runs ONLY under `uuid_on_collision: merge`; the
725
- /// default path stays streaming and never buffers a record. The build-internal
726
- /// `supporting_case_ids` carrier is stripped from each record right before the write.
787
+ /// Merge-mode edge pass: buffer every unique edge, fold divergent same-id records into
788
+ /// their smallest-canonical-bytes base, then write in first-seen id order. Returns
789
+ /// (divergent records merged, conflicting scalar fields) for the summary log, and writes a
790
+ /// fold report (`<output>.foldreport.json`: per-predicate, per-field conflict counts) when
791
+ /// any record folded. Runs ONLY under `uuid_on_collision: merge`; the default path stays
792
+ /// streaming and never buffers a record. The build-internal `supporting_case_ids` carrier
793
+ /// is stripped from each record right before the write.
727
794
  fn dedup_edges_merge(
728
795
  reader: BufReader<File>,
729
796
  mut writer: BufWriter<File>,
730
797
  domain: &str,
731
798
  fields: Option<&[String]>,
799
+ output: &Path,
732
800
  ) -> PyResult<(u64, u64)> {
801
+ let mut fold_report: std::collections::BTreeMap<
802
+ String,
803
+ std::collections::BTreeMap<String, u64>,
804
+ > = std::collections::BTreeMap::new();
733
805
  let mut index: MergeIndex = MergeIndex::default();
734
806
  for line in reader.lines() {
735
807
  let line: String = line.map_err(runtime_error)?;
@@ -744,9 +816,23 @@ fn dedup_edges_merge(
744
816
  index.absorb(edge_id_bytes(&value)?, value, content)?;
745
817
  }
746
818
  for id in std::mem::take(&mut index.order) {
747
- let Some(record) = index.records.remove(&id) else {
819
+ let Some(mut record) = index.records.remove(&id) else {
748
820
  continue;
749
821
  };
822
+ // Fold report: every conflict this id accumulated is attributed to the base
823
+ // record's predicate, so consumers can see WHERE divergent evidence merged.
824
+ if !record.conflict_fields.is_empty() {
825
+ let predicate = record
826
+ .value
827
+ .get("predicate")
828
+ .and_then(Value::as_str)
829
+ .unwrap_or("_unknown_predicate_")
830
+ .to_string();
831
+ let fields_report = fold_report.entry(predicate).or_default();
832
+ for field in std::mem::take(&mut record.conflict_fields) {
833
+ *fields_report.entry(field).or_insert(0) += 1;
834
+ }
835
+ }
750
836
  // The one deferred sort: unioned lists sort here, everything else ships as-is.
751
837
  let mut value: Value = record.finish()?;
752
838
  strip_internal_edge_fields(&mut value);
@@ -756,6 +842,20 @@ fn dedup_edges_merge(
756
842
  writer.write_all(b"\n").map_err(runtime_error)?;
757
843
  }
758
844
  writer.flush().map_err(runtime_error)?;
845
+ if !fold_report.is_empty() {
846
+ let report_path = output.with_file_name(format!(
847
+ "{}.foldreport.json",
848
+ output
849
+ .file_name()
850
+ .and_then(std::ffi::OsStr::to_str)
851
+ .unwrap_or_default()
852
+ ));
853
+ fs::write(
854
+ &report_path,
855
+ serde_json::to_vec_pretty(&fold_report).map_err(runtime_error)?,
856
+ )
857
+ .map_err(runtime_error)?;
858
+ }
759
859
  Ok((index.merged, index.scalar_conflicts))
760
860
  }
761
861
 
@@ -783,7 +883,7 @@ pub fn dedup_ndjson(
783
883
  let reader: BufReader<File> = BufReader::new(File::open(input).map_err(runtime_error)?);
784
884
  let mut writer: BufWriter<File> = BufWriter::new(File::create(&output).map_err(runtime_error)?);
785
885
  if is_edges && merge {
786
- return dedup_edges_merge(reader, writer, &domain, fields);
886
+ return dedup_edges_merge(reader, writer, &domain, fields, &output);
787
887
  }
788
888
  let mut nodes: FxHashMap<u64, Vec<Vec<u8>>> = FxHashMap::default();
789
889
  let mut edges: EdgeIndex = EdgeIndex::default();
@@ -11,7 +11,10 @@ Deriving these values from the model -- instead of hand-maintaining copies that
11
11
  can drift -- guarantees Tablassert's KGX output uses only terms that are valid
12
12
  for the pinned Biolink version. Downstream translator-ingests can therefore
13
13
  consume Tablassert output without running Koza or re-validating against the
14
- Biolink Model.
14
+ Biolink Model. A tiny curated set of named local extensions
15
+ (:data:`PREDICATE_OVERRIDES`, the DAKP prevention family) rides on top of that
16
+ model-derived vocabulary; each is a deliberate step ahead of the pinned model
17
+ and is dropped by a drift tripwire the moment the model adopts it.
15
18
 
16
19
  Source mapping
17
20
  --------------
@@ -22,7 +25,8 @@ Source mapping
22
25
  - ``Predicates`` / ``Qualifiers``: walked from the bundled
23
26
  ``biolink_model.yaml`` slot hierarchy via ``linkml-runtime`` (a pinned
24
27
  dependency of ``biolink-model``), because predicates and qualifiers are LinkML
25
- *slots* rather than Pydantic classes.
28
+ *slots* rather than Pydantic classes. ``Predicates`` then unions
29
+ :data:`PREDICATE_OVERRIDES`, the named local extensions above.
26
30
  - ``ALLOWED_EDGE_FIELDS``: the Biolink ``Association`` model fields (walked over
27
31
  the MRO) unioned with the qualifier slot names and a curated set of KGX /
28
32
  Tablassert edge columns that are not Biolink Association fields.
@@ -65,6 +69,7 @@ __all__ = [
65
69
  "EFFECT_TYPE_VALUES",
66
70
  "ENUM_RANGED_QUALIFIERS",
67
71
  "KNOWN_PENDING_EDGE_FIELDS",
72
+ "PREDICATE_OVERRIDES",
68
73
  "STUDY_METADATA_FIELDS",
69
74
  "UNSATISFIABLE_EDGE_FIELDS",
70
75
  "AgentTypes",
@@ -188,6 +193,22 @@ scan otherwise picks it up), the suite fails and the stale name is removed from
188
193
  ``CATEGORY_OVERRIDES`` -- the same philosophy as ``CLASS_FIELD_OVERRIDES``.
189
194
  """
190
195
 
196
+ PREDICATE_OVERRIDES: frozenset[str] = frozenset({"applied_to_prevent", "contraindicated_in_the_prevention_of", "prevents"})
197
+ """Biolink predicate names Tablassert deliberately supports ahead of pinned model.
198
+
199
+ ``Predicates`` normally comes directly from installed Biolink ``related to`` slots.
200
+ DAKP needs this prevention family before upstream adds it:
201
+ ``applied_to_prevent`` expresses use of an intervention for prevention,
202
+ ``prevents`` expresses preventive effect, and
203
+ ``contraindicated_in_the_prevention_of`` expresses a prevention-specific
204
+ contraindication. Each emits as its ordinary ``biolink:`` CURIE, but remains a
205
+ local extension until Biolink adopts it.
206
+
207
+ A tripwire test asserts every member remains absent from installed ``related to``
208
+ slots. When Biolink adds one, that test fails and the stale member is removed from
209
+ this set, restoring fully model-derived vocabulary for that predicate.
210
+ """
211
+
191
212
 
192
213
  def _entity_category_names() -> list[str]:
193
214
  """Collect all Biolink entity category names from the Pydantic model.
@@ -265,8 +286,8 @@ def _slot_descendants(root: str) -> frozenset[str]:
265
286
 
266
287
 
267
288
  def _predicate_values() -> list[str]:
268
- """All Biolink predicates: the ``related to`` slot hierarchy, snake_cased (root included)."""
269
- return sorted({_snake(p) for p in (_slot_descendants("related to") | {"related to"})})
289
+ """All Biolink predicates plus deliberate pending-upstream local extensions."""
290
+ return sorted({_snake(p) for p in (_slot_descendants("related to") | {"related to"})} | PREDICATE_OVERRIDES)
270
291
 
271
292
 
272
293
  def _qualifier_values() -> list[str]:
@@ -478,6 +499,9 @@ if TYPE_CHECKING:
478
499
  CHEMICAL_GENE_INTERACTION_ASSOCIATION: EdgeCategories
479
500
 
480
501
  class Predicates(str, Enum):
502
+ APPLIED_TO_PREVENT: Predicates
503
+ CONTRAINDICATED_IN_THE_PREVENTION_OF: Predicates
504
+ PREVENTS: Predicates
481
505
  RELATED_TO: Predicates
482
506
  TREATS: Predicates
483
507
 
@@ -856,17 +880,62 @@ def numeric_slot_kind(field: str) -> str | None:
856
880
  return None
857
881
 
858
882
 
883
+ def _is_pending_edge_problem(problem: str, record: dict[str, Any]) -> bool:
884
+ """Whether a single edge failure on ``record`` is a known, intentional model gap.
885
+
886
+ Two deliberate-gap kinds:
887
+
888
+ * ``extra_forbidden`` on a field in :data:`KNOWN_PENDING_EDGE_FIELDS` -- a curated
889
+ KGX / Tablassert column emitted ahead of the pinned model (same contract as
890
+ :func:`is_pending_problem`).
891
+ * ``literal_error`` / enum mismatch on the ``predicate`` slot when the record's
892
+ own predicate value is a member of :data:`PREDICATE_OVERRIDES` -- a local
893
+ predicate Tablassert emits ahead of the installed model. Most association
894
+ classes (``Association`` itself, the DAKP-pinned
895
+ ``EntityToDisease`` / ``EntityToPhenotypicFeature`` classes) declare ``predicate``
896
+ as open ``str`` and accept it natively; the constrained ones -- e.g.
897
+ ``ChemicalEntityToBiologicalProcessAssociation``'s exhaustive ``Literal`` or the
898
+ ``GeneToDiseasePredicateEnum`` family -- reject it with exactly these error
899
+ types, which this arm forgives.
900
+
901
+ Every other failure is a real defect.
902
+ """
903
+ field, _, error_type = problem.rpartition(": ")
904
+ if field == "predicate" and error_type in {"literal_error", "enum"}:
905
+ predicate: Any = record.get("predicate")
906
+ normalized: str = predicate if isinstance(predicate, str) else ""
907
+ if not normalized.startswith("biolink:"):
908
+ normalized = f"biolink:{normalized}"
909
+ return normalized.removeprefix("biolink:") in PREDICATE_OVERRIDES
910
+ return error_type == "extra_forbidden" and field in KNOWN_PENDING_EDGE_FIELDS
911
+
912
+
859
913
  def is_pending_problem(problem: str) -> bool:
860
- """Whether a ``"field: error-type"`` problem is a known, intentional model gap.
914
+ """Whether a ``"field: error-type"`` problem string is a known, intentional model gap.
861
915
 
862
- True only for an ``extra_forbidden`` rejection of a field in
863
- :data:`KNOWN_PENDING_EDGE_FIELDS` -- i.e. Tablassert emitted a column on purpose that
864
- the pinned Biolink Model has not declared yet. Every other failure is a real defect.
916
+ Kept field-only for the existing extra-forbidden vocabulary. For predicate
917
+ failures the record's own predicate value is needed, so prefer
918
+ :func:`_is_pending_edge_problem` (or :func:`_record_has_only_pending_problems` for
919
+ a record's full error list).
865
920
  """
866
921
  field, _, error_type = problem.rpartition(": ")
867
922
  return error_type == "extra_forbidden" and field in KNOWN_PENDING_EDGE_FIELDS
868
923
 
869
924
 
925
+ def _record_has_only_pending_problems(record: dict[str, Any], problems: list[str]) -> bool:
926
+ """True when every failure of ``record`` is a known, intentional model gap.
927
+
928
+ Applies the strict :func:`is_pending_problem` contract to edge records (where the
929
+ override-predicate exemption is meaningful) and keeps the field-only contract
930
+ everywhere else, so a predicate mismatch on a node record can never be forgiven.
931
+ """
932
+ if not problems:
933
+ return False
934
+ if "predicate" in record:
935
+ return all(_is_pending_edge_problem(problem, record) for problem in problems)
936
+ return all(is_pending_problem(problem) for problem in problems)
937
+
938
+
870
939
  def validate_kgx(nodes_path: Path, edges_path: Path, limit: int = 20) -> dict[str, Any]:
871
940
  """Validate emitted KGX NDJSON files against the Biolink Pydantic model.
872
941
 
@@ -877,9 +946,11 @@ def validate_kgx(nodes_path: Path, edges_path: Path, limit: int = 20) -> dict[st
877
946
 
878
947
  Two pass rates are reported. ``valid`` is strict and drives ``ok`` (the CLI's
879
948
  non-zero exit). ``valid_excluding_pending`` additionally counts records whose *every*
880
- failure is a :func:`is_pending_problem` -- the score to optimize against, so a
881
- deliberate gap like the KGX denormalized carryovers is not mistaken for a
882
- malformed record. The two converge as future model releases absorb curated extras.
949
+ failure is a known, intentional model gap -- an :func:`is_pending_problem` field
950
+ rejection, or a record-scoped :data:`PREDICATE_OVERRIDES` predicate rejection (see
951
+ :func:`_is_pending_edge_problem`) -- the score to optimize against, so a deliberate
952
+ gap like the KGX denormalized carryovers is not mistaken for a malformed record. The
953
+ two converge as future model releases absorb curated extras.
883
954
 
884
955
  Args:
885
956
  nodes_path: Path to ``<name>_<version>.nodes.ndjson``.
@@ -914,7 +985,7 @@ def validate_kgx(nodes_path: Path, edges_path: Path, limit: int = 20) -> dict[st
914
985
  valid += 1
915
986
  valid_excluding_pending += 1
916
987
  continue
917
- if all(is_pending_problem(problem) for problem in errors):
988
+ if _record_has_only_pending_problems(record, errors):
918
989
  valid_excluding_pending += 1
919
990
  problems.update(errors)
920
991
  if len(examples) < limit:
@@ -17,7 +17,7 @@ from importlib.metadata import version as get_version
17
17
  from itertools import chain, pairwise
18
18
  from multiprocessing import Pool
19
19
  from pathlib import Path
20
- from typing import TYPE_CHECKING, Annotated, Any, BinaryIO, Literal, NoReturn
20
+ from typing import TYPE_CHECKING, Annotated, Any, BinaryIO, Literal, NoReturn, cast
21
21
  from urllib.error import HTTPError, URLError
22
22
  from urllib.request import Request, urlopen
23
23
 
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
32
32
  import polars as pl
33
33
  import pydantic
34
34
 
35
+ from tablassert.biolink import Categories
35
36
  from tablassert.lib import Tcode
36
37
  from tablassert.models import Graph
37
38
  from tablassert.progress import PipelineProgress
@@ -367,6 +368,10 @@ def build_graph_pipeline(
367
368
  "head": head,
368
369
  "name": g.name,
369
370
  "infores": g.rig.source_info.infores_id,
371
+ "qc_similarity_threshold": g.qc_similarity_threshold,
372
+ "qc_fuzzy_ratio_threshold": g.qc_fuzzy_ratio_threshold,
373
+ "qc_fuzzy_partial_threshold": g.qc_fuzzy_partial_threshold,
374
+ "qc_aliases": g.qc_aliases,
370
375
  }
371
376
  )
372
377
  )
@@ -968,6 +973,127 @@ def validate_kgx_command(
968
973
  print("KGX output is Biolink-compliant.", file=sys.stderr)
969
974
 
970
975
 
976
+ @APP.command(name="quick-map")
977
+ def quick_map_command(
978
+ terms: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False)],
979
+ *,
980
+ fullmap: Annotated[Path, cyclopts.Parameter(name=["--fullmap", "-f"])],
981
+ taxon: Annotated[int | None, cyclopts.Parameter(name=["--taxon", "-t"])] = 9606,
982
+ prioritize: Annotated[list[str] | None, cyclopts.Parameter(name=["--prioritize", "-p"])] = None,
983
+ avoid: Annotated[list[str] | None, cyclopts.Parameter(name=["--avoid", "-a"])] = None,
984
+ exclude_prefixes: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-prefixes", "-ep"])] = None,
985
+ exclude_regex: Annotated[list[str] | None, cyclopts.Parameter(name=["--exclude-regex", "-er"])] = None,
986
+ ) -> None:
987
+ """Show what fullmap entity resolution does with one or more terms, as a build would see it.
988
+
989
+ Each term runs through the exact op chain a build runs per node column -- level-one/level-two
990
+ normalization, probe-key extraction, the redb fetch, then ``filter_and_rank``'s taxon filter,
991
+ category prioritize/avoid, prefix/regex exclusion, ``PR`` scoring, and best-tier dedup -- so the
992
+ printed rows are the rows ``build-kg`` would emit for a cell holding that term under a
993
+ ``NodeEncoding`` carrying the same settings. Use it to debug a mapping ("why did this cell
994
+ resolve to that CURIE?") or to smoke-test a freshly built fullmap. The flags mirror the
995
+ ``NodeEncoding`` config fields one-to-one; ``--taxon`` defaults to 9606 like the config does,
996
+ and ``--taxon 0`` disables the filter like ``taxon: null`` does.
997
+
998
+ ``--prioritize`` / ``--avoid`` accept the live Biolink entity category names -- the same
999
+ vocabulary those keys accept in a table config -- validated at run time rather than enumerated
1000
+ here (the enum is built dynamically from the installed biolink-model), so an invalid value
1001
+ exits 2 naming the nearest valid names. A term with no matches prints an explicit "no matches"
1002
+ line (with the normalized probe keys it became, so the miss is diagnosable) and still exits 0:
1003
+ a miss is a finding, not a failure. Only usage errors and an unreadable fullmap exit 2.
1004
+
1005
+ Args:
1006
+ terms: One or more terms to resolve (positional); any casing or whitespace, and a CURIE
1007
+ string works too because the fullmap indexes CURIEs and their equivalent identifiers
1008
+ as terms.
1009
+ fullmap: Path to the fullmap redb file, or a directory holding one (resolved like the
1010
+ ``Graph.fullmap`` config field: ``<dir>/fullmap.redb`` then ``<dir>/data/fullmap.redb``).
1011
+ taxon: NCBI taxon id constraining taxon-bearing matches; ``0`` disables the filter.
1012
+ prioritize: Biolink categories ranked higher (see the vocabulary note above).
1013
+ avoid: Biolink categories dropped entirely (see the vocabulary note above).
1014
+ exclude_prefixes: CURIE namespace prefixes (text before the first ':') to drop.
1015
+ exclude_regex: Regex patterns; any resolved CURIE matching one is dropped.
1016
+ """
1017
+ from difflib import get_close_matches
1018
+
1019
+ from rich.console import Console
1020
+ from rich.table import Table
1021
+
1022
+ from tablassert import rs
1023
+ from tablassert.fullmap import _KNOWN_CATEGORIES, _probe_keys, fullmap_db_path, quick_map
1024
+
1025
+ def fail(message: str) -> NoReturn:
1026
+ """Print one actionable user error and use the CLI's documented exit status."""
1027
+ print(f"tablassert quick-map: {message}", file=sys.stderr)
1028
+ raise SystemExit(2)
1029
+
1030
+ # Flag validation precedes any fullmap access: a typo'd category or regex is the user's fastest
1031
+ # loop to close, and these checks are pure so they fire before the redb is even resolved.
1032
+ for label, categories in (("prioritize", prioritize), ("avoid", avoid)):
1033
+ for category in categories or []:
1034
+ if category not in _KNOWN_CATEGORIES:
1035
+ nearest: list[str] = get_close_matches(category, _KNOWN_CATEGORIES, n=3, cutoff=0.5)
1036
+ hint: str = f"; did you mean {', '.join(repr(x) for x in nearest)}" if nearest else ""
1037
+ fail(
1038
+ f"--{label} value {category!r} is not a known Biolink entity category{hint}; accepted values are the same names the `prioritize`/`avoid` config keys accept"
1039
+ )
1040
+ for pattern in exclude_regex or []:
1041
+ # Mirrors the NodeEncoding.exclude_regex validator: an empty pattern matches EVERY CURIE
1042
+ # and would silently drop all candidates, and a non-polars regex would die mid-filter.
1043
+ if not str(pattern).strip():
1044
+ fail(f"--exclude-regex entries must be non-empty patterns (an empty pattern matches every CURIE), got {pattern!r}")
1045
+ try:
1046
+ pl.Series([""]).str.contains(str(pattern))
1047
+ except Exception as e:
1048
+ fail(f"--exclude-regex entries must be polars-compatible regular expressions, got {pattern!r}: {e}")
1049
+
1050
+ db: Path = fullmap_db_path(fullmap)
1051
+ if not db.is_file():
1052
+ fail(f"no fullmap database at {db} (resolved from {fullmap}) — build one with tablassert build-fullmap")
1053
+
1054
+ results: dict[str, pl.DataFrame] = quick_map(
1055
+ terms,
1056
+ db,
1057
+ taxon=str(taxon) if taxon else None,
1058
+ prioritize=cast("list[Categories] | None", prioritize),
1059
+ avoid=cast("list[Categories] | None", avoid),
1060
+ exclude_prefixes=exclude_prefixes,
1061
+ exclude_regex=exclude_regex,
1062
+ )
1063
+
1064
+ # One run-level header: the resolved database, its source version, the term count, and every
1065
+ # active filter, printed once so per-term output stays pure result. Joined as ONE string so a
1066
+ # wrapped line never dangles a separator at the break.
1067
+ console: Console = Console()
1068
+ active: list[str] = []
1069
+ if taxon:
1070
+ active.append(f"taxon={taxon}")
1071
+ for label, values in (("prioritize", prioritize), ("avoid", avoid), ("exclude-prefixes", exclude_prefixes), ("exclude-regex", exclude_regex)):
1072
+ if values:
1073
+ active.append(f"{label}={','.join(values)}")
1074
+ header: str = " · ".join(["[bold]quick-map[/bold]", str(db), f"source {rs.fullmap_source_version()}", f"{len(results)} term(s)", *active])
1075
+ console.print(header)
1076
+
1077
+ # Probe keys come from the SAME normalization quick_map probed with, so a title can never
1078
+ # diagnose a miss against a key the lookup never used.
1079
+ keys: pl.DataFrame = _probe_keys(terms)
1080
+ columns: tuple[str, ...] = ("CURIE", "PREFERRED_NAME", "CATEGORY_NAME", "TAXON_ID", "SOURCE_NAME", "NLP_LEVEL", "PR")
1081
+ for term, matches in results.items():
1082
+ position: int = terms.index(term)
1083
+ level_one_key: str = str(keys.get_column("term")[position])
1084
+ level_two_key: str = str(keys.get_column("term_two")[position])
1085
+ probe: str = level_one_key if level_one_key == level_two_key else f"{level_one_key} | {level_two_key}"
1086
+ if matches.height == 0:
1087
+ console.print(f"{term!r} → {probe} · no matches")
1088
+ continue
1089
+ table: Table = Table(title=f"{term!r} → {probe}", title_justify="left")
1090
+ for column in columns:
1091
+ table.add_column(column)
1092
+ for row in matches.iter_rows(named=True):
1093
+ table.add_row(*(str(row[column]) for column in columns))
1094
+ console.print(table)
1095
+
1096
+
971
1097
  @APP.command(name="agent")
972
1098
  def agent(
973
1099
  pmc_ids: Annotated[list[str], cyclopts.Parameter(allow_leading_hyphen=False)],
@@ -12,7 +12,7 @@ from tablassert._lazy import LazyModule
12
12
  from tablassert.biolink import Categories
13
13
  from tablassert.errors import TablassertError
14
14
  from tablassert.log import cat
15
- from tablassert.nlp import _normalize_terms_series
15
+ from tablassert.nlp import _normalize_terms_series, level_one, level_two
16
16
 
17
17
  logger = cat("FULLMAP")
18
18
 
@@ -754,3 +754,106 @@ def resolve(
754
754
  column_context=column_context,
755
755
  tag=tag,
756
756
  )
757
+
758
+
759
+ def _probe_keys(terms: list[str]) -> pl.DataFrame:
760
+ """Normalize raw terms through the exact level-one/level-two steps a build applies.
761
+
762
+ This is ``Tcode.node_prep``'s normalization pair (``lib.py`` ``node_prep``) with no
763
+ encodings in front of it: ``level_one`` then ``level_two`` on one ``term`` column.
764
+ ``quick_map`` builds its probe frame from the result, and callers use it to show a
765
+ user what their raw input became -- both need the SAME normalization the build ran,
766
+ never a Python-side reimplementation, or a miss would be diagnosed against the wrong
767
+ key.
768
+
769
+ Args:
770
+ terms: Raw input terms, in input order.
771
+
772
+ Returns:
773
+ DataFrame with ``term`` (level-one) and ``term_two`` (level-two) columns,
774
+ position-aligned with ``terms``.
775
+ """
776
+ return level_two(level_one(pl.LazyFrame({"term": terms}), "term"), "term").collect()
777
+
778
+
779
+ def quick_map(
780
+ terms: list[str],
781
+ db: Path,
782
+ *,
783
+ taxon: str | None = "9606",
784
+ prioritize: list[Categories] | None = None,
785
+ avoid: list[Categories] | None = None,
786
+ exclude_prefixes: list[str] | None = None,
787
+ exclude_regex: list[str] | None = None,
788
+ ) -> dict[str, pl.DataFrame]:
789
+ """Resolve raw terms against a fullmap exactly as a build would, one entry per input.
790
+
791
+ This is the ``tablassert quick-map`` core: it answers "what will entity resolution do
792
+ with THIS term under THESE filters?" by running the same op chain a build runs per
793
+ node column -- normalize (``level_one``/``level_two``) -> extract probe keys
794
+ (``distinct``) -> fetch (``lookup_rows``) -> filter, rank, and dedup
795
+ (``filter_and_rank``) -- over a one-column frame instead of a table. Nothing is
796
+ reimplemented, so the returned rows are the rows a build would emit for a cell holding
797
+ that term under a ``NodeEncoding`` carrying the same settings.
798
+
799
+ Differences from a build, both deliberate:
800
+
801
+ - ``column_context`` is fixed ``False``. The frequency tiebreaker counts category
802
+ occurrences within one resolved column of a real table; a handful of probe terms is
803
+ not that population, so enabling it would invent a ranking signal no build would
804
+ have.
805
+ - The whole input is ONE batch through ONE ``lookup_rows`` call (the same pooling
806
+ ``resolve_batch`` does across columns), not one round trip per term.
807
+
808
+ Grouping back to inputs: a ranked row's ``term`` is a normalized probe key, so each
809
+ input is reported under every probe key it produced that survived ``distinct``'s
810
+ junk-term filter. Two inputs that normalize to the same key both report that key's
811
+ matches -- the fullmap genuinely cannot distinguish them. An input whose keys were
812
+ all dropped (a purely numeric or sentinel term) maps to a zero-row frame, because a
813
+ build would never probe it either. Repeated identical inputs collapse to one entry.
814
+
815
+ Args:
816
+ terms: Raw input terms, in input order (any casing/whitespace; a CURIE string
817
+ works too, because the builder indexes CURIEs and their equivalent
818
+ identifiers as terms).
819
+ db: Path to the fullmap redb file (already resolved via ``fullmap_db_path``).
820
+ taxon: Optional NCBI taxon id constraining taxon-bearing matches; rows with
821
+ ``TAXON_ID`` 0 are retained. ``None`` disables the filter.
822
+ prioritize: Categories to boost in ranking.
823
+ avoid: Categories to drop entirely (see :func:`filter_and_rank`, which also drops
824
+ categories the ``Categories`` enum cannot name).
825
+ exclude_prefixes: CURIE namespace prefixes (text before the first ':') to drop.
826
+ exclude_regex: Regex patterns; any CURIE matching one is dropped.
827
+
828
+ Returns:
829
+ Insertion-ordered mapping of each distinct input term to its ranked matches
830
+ DataFrame (``filter_and_rank``'s schema: ``term``, ``CURIE``, ``PREFERRED_NAME``,
831
+ ``CATEGORY_NAME``, ``TAXON_ID``, ``SOURCE_NAME``, ``SOURCE_VERSION``,
832
+ ``NLP_LEVEL``, ``PR``). A term with no matches maps to ``empty_matches(False)``,
833
+ never to a missing key, so callers never branch on presence.
834
+ """
835
+ if not terms:
836
+ return {}
837
+ # Position-aligned (level-one, level-two) keys per input; ALSO the display truth for
838
+ # callers, so probe keys and titles can never drift apart.
839
+ keys: pl.DataFrame = _probe_keys(terms)
840
+ # distinct() dedups the probe keys across inputs, tags each with its NLP level, and
841
+ # drops numeric/sentinel/generic junk exactly as a build's term extraction does.
842
+ terms_df: pl.DataFrame = distinct(keys.lazy(), "term", "term_two").collect()
843
+ probe_keys: list[str] = terms_df.get_column("term").to_list()
844
+ probed: set[str] = set(probe_keys)
845
+ # ONE redb round trip for the whole batch (term cache, dimension maps, and lock retry
846
+ # all live inside lookup_rows), then ONE filter/rank pass whose per-term best-tier
847
+ # dedup already partitions results by probe key.
848
+ rows: list[dict[str, object]] = lookup_rows(db, probe_keys) if probe_keys else []
849
+ matches: pl.DataFrame = filter_and_rank(pl.DataFrame(rows), terms_df, taxon, prioritize, avoid, False, exclude_prefixes, exclude_regex)
850
+ level_one_col: pl.Series = keys.get_column("term")
851
+ level_two_col: pl.Series = keys.get_column("term_two")
852
+ out: dict[str, pl.DataFrame] = {}
853
+ for position, term in enumerate(terms):
854
+ candidates: list[str] = []
855
+ for key in (level_one_col[position], level_two_col[position]):
856
+ if key and key in probed and key not in candidates:
857
+ candidates.append(key)
858
+ out[term] = matches.filter(pl.col("term").is_in(candidates)) if candidates else empty_matches(False)
859
+ return out
@@ -1150,6 +1150,10 @@ class Tcode(Section):
1150
1150
  head: bool = Field(False)
1151
1151
  name: str | None = Field(None)
1152
1152
  infores: str | None = Field(None)
1153
+ qc_similarity_threshold: float = Field(0.5)
1154
+ qc_fuzzy_ratio_threshold: float = Field(70.0)
1155
+ qc_fuzzy_partial_threshold: float = Field(80.0)
1156
+ qc_aliases: dict[str, str] = Field(default_factory=dict)
1153
1157
 
1154
1158
  def encoding(self: Self, x: Encoding, col: str, table_literal: bool = False) -> list[Any]:
1155
1159
  """Collect helper for Encoding classes.
@@ -1302,7 +1306,30 @@ class Tcode(Section):
1302
1306
  # QC audits only the strict columns: a nullable qualifier's nulls are expected
1303
1307
  # (blank cell / no match), not resolution errors for the audit to delete.
1304
1308
  [
1305
- (fullmap_audit, (col, self.store.stem, self.config.name, "passed", True))
1309
+ (
1310
+ fullmap_audit,
1311
+ (
1312
+ (col, self.store.stem, self.config.name, "passed", True)
1313
+ if (
1314
+ self.qc_similarity_threshold == 0.5
1315
+ and self.qc_fuzzy_ratio_threshold == 70.0
1316
+ and self.qc_fuzzy_partial_threshold == 80.0
1317
+ and not self.qc_aliases
1318
+ )
1319
+ else (
1320
+ col,
1321
+ self.store.stem,
1322
+ self.config.name,
1323
+ "passed",
1324
+ True,
1325
+ None,
1326
+ self.qc_similarity_threshold,
1327
+ self.qc_fuzzy_ratio_threshold,
1328
+ self.qc_fuzzy_partial_threshold,
1329
+ self.qc_aliases,
1330
+ )
1331
+ ),
1332
+ )
1306
1333
  for x, col in node_columns
1307
1334
  if not (isinstance(x, Qualifier) and x.nullable)
1308
1335
  ]
@@ -1138,6 +1138,10 @@ class Graph(TablaBase):
1138
1138
  description="Explicit UUID namespace. Defaults to `rig.source_info.infores_id` when `uuid_fields` is set, `TABLASSERT` otherwise. Set it only when graphs must deliberately share an id space.",
1139
1139
  examples=["infores:multiomicskg"],
1140
1140
  )
1141
+ qc_similarity_threshold: float = Field(default=0.5, ge=0.0, le=1.0, description="SapBERT cosine threshold for QC.")
1142
+ qc_fuzzy_ratio_threshold: float = Field(default=70.0, ge=0.0, le=100.0, description="RapidFuzz ratio threshold for QC.")
1143
+ qc_fuzzy_partial_threshold: float = Field(default=80.0, ge=0.0, le=100.0, description="RapidFuzz token threshold for QC.")
1144
+ qc_aliases: dict[str, str] = Field(default_factory=dict, description="QC source-text aliases applied before similarity scoring.")
1141
1145
  uuid_on_collision: Literal["error", "merge"] = Field(
1142
1146
  default="error",
1143
1147
  description="What to do when two different edges derive one id under `uuid_fields`: `error` aborts the build with `uuid-fields-not-a-key`; `merge` folds the records into one edge (list fields unioned and sorted, scalars first-wins). Requires `uuid_fields`, and buffers one full record per unique id, which is why it is opt-in.",
@@ -1,5 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import re
3
4
  from collections.abc import Callable
4
5
  from functools import cache
5
6
  from pathlib import Path
@@ -27,6 +28,18 @@ MODEL: Path = BASE / "sapbert"
27
28
  # BioBERT gate; SapBERT's UMLS synonym-alignment training shifts true synonym
28
29
  # pairs upward, so a tuning pass against real reject logs is the natural follow-up.
29
30
  SIMILARITY_THRESHOLD: float = 0.5
31
+ FUZZ_RATIO_THRESHOLD: float = 70.0
32
+ FUZZ_PARTIAL_THRESHOLD: float = 80.0
33
+
34
+
35
+ def _qc_text(value: str) -> str:
36
+ """Normalize separator noise before lexical QC scoring."""
37
+ return re.sub(r"[^\w]+", " ", value.replace("?", "-")).strip().casefold()
38
+
39
+
40
+ def _qc_alias(value: str, aliases: dict[str, str]) -> str:
41
+ """Resolve configured source-specific aliases before generic similarity stages."""
42
+ return aliases.get(value.casefold(), value)
30
43
 
31
44
 
32
45
  def _cascade(passed: pl.DataFrame, scored: pl.DataFrame, out: str) -> tuple[pl.DataFrame, pl.DataFrame]:
@@ -121,6 +134,10 @@ def fullmap_audit(
121
134
  out: str = "passed",
122
135
  log: bool = True,
123
136
  on_phase: Callable[[str], None] | None = None,
137
+ similarity_threshold: float = SIMILARITY_THRESHOLD,
138
+ fuzzy_ratio_threshold: float = FUZZ_RATIO_THRESHOLD,
139
+ fuzzy_partial_threshold: float = FUZZ_PARTIAL_THRESHOLD,
140
+ aliases: dict[str, str] | None = None,
124
141
  ) -> pl.LazyFrame:
125
142
  """Audit that fullmap correctly processed source strings into CURIEs.
126
143
 
@@ -182,8 +199,11 @@ def fullmap_audit(
182
199
  on_phase("qc:exact")
183
200
  # Collection point: pending pairs require eager.
184
201
  df: pl.DataFrame = lf.collect()
202
+ aliases = {str(key).casefold(): str(value) for key, value in (aliases or {}).items()}
185
203
  pairs: pl.DataFrame = df.select(cols).unique()
186
- pairs = pairs.with_columns((pl.col(cols[1]) == pl.col(cols[2])).alias(out))
204
+ pairs = pairs.with_columns(
205
+ pl.struct(cols[1:]).map_elements(lambda row: _qc_alias(str(row[cols[1]]), aliases) == str(row[cols[2]]), return_dtype=pl.Boolean).alias(out)
206
+ )
187
207
 
188
208
  passed: pl.DataFrame
189
209
  pending: pl.DataFrame
@@ -203,15 +223,24 @@ def fullmap_audit(
203
223
  # Stage 2: fuzzy matching via RapidFuzz (batched).
204
224
  if on_phase is not None:
205
225
  on_phase("qc:fuzzy")
206
- originals: list[str] = pending.get_column(cols[1]).to_list()
207
- preferreds: list[str] = pending.get_column(cols[2]).to_list()
226
+ originals: list[str] = [_qc_alias(str(value), aliases) for value in pending.get_column(cols[1]).to_list()]
227
+ preferreds: list[str] = [str(value) for value in pending.get_column(cols[2]).to_list()]
228
+ normalized_originals: list[str] = [_qc_text(value) for value in originals]
229
+ normalized_preferreds: list[str] = [_qc_text(value) for value in preferreds]
208
230
 
209
- ratio_scores: object = cpdist(originals, preferreds, scorer=fuzz.ratio)
210
- partial_scores: object = cpdist(originals, preferreds, scorer=fuzz.partial_token_sort_ratio)
231
+ ratio_scores: object = cpdist(normalized_originals, normalized_preferreds, scorer=fuzz.ratio)
232
+ partial_scores: object = cpdist(normalized_originals, normalized_preferreds, scorer=fuzz.partial_token_sort_ratio)
233
+ token_scores: object = cpdist(normalized_originals, normalized_preferreds, scorer=fuzz.token_set_ratio)
211
234
 
212
- pending = pending.with_columns([pl.Series("fuzz_ratio", ratio_scores), pl.Series("fuzz_partial", partial_scores)])
235
+ pending = pending.with_columns(
236
+ [pl.Series("fuzz_ratio", ratio_scores), pl.Series("fuzz_partial", partial_scores), pl.Series("fuzz_token_set", token_scores)]
237
+ )
213
238
 
214
- fuzz_mask: pl.Series = pl.Series(out, (ratio_scores >= 70) | (partial_scores >= 80), dtype=pl.Boolean)
239
+ fuzz_mask: pl.Series = pl.Series(
240
+ out,
241
+ (ratio_scores >= fuzzy_ratio_threshold) | (partial_scores >= fuzzy_partial_threshold) | (token_scores >= fuzzy_partial_threshold),
242
+ dtype=pl.Boolean,
243
+ )
215
244
  masked_fuzz: pl.DataFrame = pending.with_columns(fuzz_mask)
216
245
  pairs = pl.concat((passed, masked_fuzz), how="diagonal")
217
246
 
@@ -253,7 +282,7 @@ def fullmap_audit(
253
282
 
254
283
  pending = pending.with_columns(pl.Series("sapbert_similarity", similarity)) # pyright: ignore
255
284
 
256
- sapbert_mask: pl.Series = pl.Series(out, similarity >= SIMILARITY_THRESHOLD, dtype=pl.Boolean) # pyright: ignore
285
+ sapbert_mask: pl.Series = pl.Series(out, similarity >= similarity_threshold, dtype=pl.Boolean) # pyright: ignore
257
286
  masked_sapbert: pl.DataFrame = pending.with_columns(sapbert_mask)
258
287
  pairs = pl.concat((passed, masked_sapbert), how="diagonal")
259
288
 
File without changes
File without changes
File without changes
File without changes
File without changes