tablassert 22.2.1__tar.gz → 22.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {tablassert-22.2.1 → tablassert-22.3.0}/PKG-INFO +2 -1
  2. {tablassert-22.2.1 → tablassert-22.3.0}/README.md +1 -0
  3. {tablassert-22.2.1 → tablassert-22.3.0}/pyproject.toml +1 -1
  4. {tablassert-22.2.1 → tablassert-22.3.0}/rust/Cargo.lock +6 -6
  5. {tablassert-22.2.1 → tablassert-22.3.0}/rust/Cargo.toml +2 -2
  6. {tablassert-22.2.1 → tablassert-22.3.0}/rust/src/fullmap.rs +46 -14
  7. {tablassert-22.2.1 → tablassert-22.3.0}/rust/src/lib.rs +5 -3
  8. {tablassert-22.2.1 → tablassert-22.3.0}/rust/src/nlp.rs +52 -1
  9. {tablassert-22.2.1 → tablassert-22.3.0}/rust/tests/build_golden.rs +2 -3
  10. {tablassert-22.2.1 → tablassert-22.3.0}/rust/tests/common/mod.rs +3 -2
  11. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/nlp.py +39 -3
  12. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/rs.pyi +1 -0
  13. {tablassert-22.2.1 → tablassert-22.3.0}/LICENSE +0 -0
  14. {tablassert-22.2.1 → tablassert-22.3.0}/rust/examples/count_tables.rs +0 -0
  15. {tablassert-22.2.1 → tablassert-22.3.0}/rust/src/json.rs +0 -0
  16. {tablassert-22.2.1 → tablassert-22.3.0}/rust/src/ndjson.rs +0 -0
  17. {tablassert-22.2.1 → tablassert-22.3.0}/rust/src/uuid.rs +0 -0
  18. {tablassert-22.2.1 → tablassert-22.3.0}/rust/tests/extract_prebuilt.rs +0 -0
  19. {tablassert-22.2.1 → tablassert-22.3.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
  20. {tablassert-22.2.1 → tablassert-22.3.0}/rust/tests/nlp_golden.rs +0 -0
  21. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/__init__.py +0 -0
  22. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/_lazy.py +0 -0
  23. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/biolink.py +0 -0
  24. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/cli.py +0 -0
  25. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/cli_entry.py +0 -0
  26. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/coerce.py +0 -0
  27. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
  28. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/data/infores_catalog.yaml +0 -0
  29. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/enums.py +0 -0
  30. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/errors.py +0 -0
  31. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/extras.py +0 -0
  32. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/fullmap.py +0 -0
  33. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/infores_registry.py +0 -0
  34. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/ingests.py +0 -0
  35. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/lib.py +0 -0
  36. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/log.py +0 -0
  37. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/models.py +0 -0
  38. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/net.py +0 -0
  39. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/progress.py +0 -0
  40. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/qc.py +0 -0
  41. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/rig.py +0 -0
  42. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/runcache.py +0 -0
  43. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/study.py +0 -0
  44. {tablassert-22.2.1 → tablassert-22.3.0}/src/tablassert/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tablassert
3
- Version: 22.2.1
3
+ Version: 22.3.0
4
4
  Classifier: License :: OSI Approved :: Apache Software License
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Intended Audience :: Science/Research
@@ -55,6 +55,7 @@ Project-URL: Source, https://github.com/SkyeAv/Tablassert
55
55
  # Tablassert
56
56
 
57
57
  [![PyPI version](https://badge.fury.io/py/tablassert.svg)](https://badge.fury.io/py/tablassert)
58
+ [![PyPI Downloads](https://static.pepy.tech/personalized-badge/tablassert?period=total&units=INTERNATIONAL_SYSTEM&left_color=GREY&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/tablassert)
58
59
  [![CI](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml/badge.svg)](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
59
60
  [![Docs](https://img.shields.io/github/deployments/SkyeAv/Tablassert/github-pages?label=docs)](https://skyeav.github.io/Tablassert/)
60
61
  [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
@@ -1,6 +1,7 @@
1
1
  # Tablassert
2
2
 
3
3
  [![PyPI version](https://badge.fury.io/py/tablassert.svg)](https://badge.fury.io/py/tablassert)
4
+ [![PyPI Downloads](https://static.pepy.tech/personalized-badge/tablassert?period=total&units=INTERNATIONAL_SYSTEM&left_color=GREY&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/tablassert)
4
5
  [![CI](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml/badge.svg)](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
5
6
  [![Docs](https://img.shields.io/github/deployments/SkyeAv/Tablassert/github-pages?label=docs)](https://skyeav.github.io/Tablassert/)
6
7
  [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "tablassert"
3
- version = "22.2.1"
3
+ version = "22.3.0"
4
4
  description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
5
5
  authors = [
6
6
  { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
@@ -731,27 +731,27 @@ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
731
731
 
732
732
  [[package]]
733
733
  name = "zstd"
734
- version = "0.13.3"
734
+ version = "0.14.3"
735
735
  source = "registry+https://github.com/rust-lang/crates.io-index"
736
- checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
736
+ checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70"
737
737
  dependencies = [
738
738
  "zstd-safe",
739
739
  ]
740
740
 
741
741
  [[package]]
742
742
  name = "zstd-safe"
743
- version = "7.2.4"
743
+ version = "8.2.0"
744
744
  source = "registry+https://github.com/rust-lang/crates.io-index"
745
- checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
745
+ checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b"
746
746
  dependencies = [
747
747
  "zstd-sys",
748
748
  ]
749
749
 
750
750
  [[package]]
751
751
  name = "zstd-sys"
752
- version = "2.0.16+zstd.1.5.7"
752
+ version = "2.1.2+zstd.1.5.7"
753
753
  source = "registry+https://github.com/rust-lang/crates.io-index"
754
- checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
754
+ checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3"
755
755
  dependencies = [
756
756
  "cc",
757
757
  "pkg-config",
@@ -48,7 +48,7 @@ xxhash-rust = { version = "0.8", features = ["xxh64", "xxh3"] }
48
48
  # Streaming zstd decompression for the prebuilt `fullmap.tar.zst` archive
49
49
  # (`extract_prebuilt_fullmap`); the C-backed decoder is the fastest option and
50
50
  # `cc` is available in the build environment.
51
- zstd = "0.13"
51
+ zstd = "0.14"
52
52
 
53
53
  [dev-dependencies]
54
54
  bincode = "1"
@@ -58,7 +58,7 @@ serde = { version = "1", features = ["derive"] }
58
58
  serde_json = "1"
59
59
  tar = "0.4"
60
60
  tempfile = "3"
61
- zstd = "0.13"
61
+ zstd = "0.14"
62
62
 
63
63
  [lints.rust]
64
64
  unused_imports = "deny"
@@ -327,11 +327,19 @@ fn level_one(value: &str) -> Cow<'_, str> {
327
327
  }
328
328
  }
329
329
 
330
- fn level_two(value: &str) -> Cow<'_, str> {
331
- // Fast path: level-two keeps [A-Za-z0-9_]; for an already-lowercase ASCII
332
- // input that is exactly [a-z0-9_], so when every byte qualifies the filter is
333
- // a no-op and we borrow unchanged (the common case post-level_one). Any
334
- // uppercase or non-ASCII byte falls through to the filtering slow path.
330
+ /// Derive the level-two key of a (usually level-one) term: keep every Unicode
331
+ /// alphanumeric character or underscore, drop everything else. This is THE
332
+ /// single level-two key definition: `emit_term` stores build-side keys through
333
+ /// it and `nlp::normalize_terms_l2` derives query-side keys through it, so the
334
+ /// two sides cannot drift apart (the query side used a Unicode-aware polars
335
+ /// regex while the build stripped all non-ASCII, which made non-ASCII terms
336
+ /// unmatchable at level two).
337
+ pub(crate) fn level_two(value: &str) -> Cow<'_, str> {
338
+ // Fast path: for an already-lowercase ASCII input, keeping Unicode word
339
+ // characters is exactly keeping [a-z0-9_], so when every byte qualifies the
340
+ // filter is a no-op and we borrow unchanged (the common case
341
+ // post-level_one). Any uppercase or non-ASCII byte falls through to the
342
+ // filtering slow path.
335
343
  if value
336
344
  .bytes()
337
345
  .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
@@ -341,7 +349,7 @@ fn level_two(value: &str) -> Cow<'_, str> {
341
349
  Cow::Owned(
342
350
  value
343
351
  .chars()
344
- .filter(|c| c.is_ascii_alphanumeric() || *c == '_')
352
+ .filter(|c| c.is_alphanumeric() || *c == '_')
345
353
  .collect(),
346
354
  )
347
355
  }
@@ -3902,6 +3910,26 @@ mod tests {
3902
3910
  assert_eq!(level_two("brca-1"), "brca1");
3903
3911
  }
3904
3912
 
3913
+ #[test]
3914
+ fn level_two_keeps_non_ascii_word_characters() {
3915
+ // WHY: the build side and the Python query side must derive the SAME
3916
+ // level-two key for every level-one string (issue #251). The filter
3917
+ // keeps Unicode alphanumeric characters (accents, Greek letters,
3918
+ // non-ASCII digits) plus underscore, and drops punctuation, symbols,
3919
+ // combining marks, and whitespace -- matching the query side because it
3920
+ // IS the implementation the query side calls.
3921
+ assert_eq!(level_two("au café lait"), "aucafélait");
3922
+ assert_eq!(level_two("β-catenin"), "βcatenin");
3923
+ assert_eq!(level_two("naïve"), "naïve");
3924
+ assert_eq!(level_two("p_value"), "p_value");
3925
+ // Combining marks are neither alphanumeric nor underscore: a decomposed
3926
+ // form must collapse exactly like the query side does.
3927
+ assert_eq!(level_two("cafe\u{301}"), "cafe");
3928
+ // Unchanged ASCII behavior: punctuation still splits, empty stays empty.
3929
+ assert_eq!(level_two("hello-world"), "helloworld");
3930
+ assert_eq!(level_two(""), "");
3931
+ }
3932
+
3905
3933
  /// `term_shard` is the single routing oracle shared by the writer and the
3906
3934
  /// reader, so it must be deterministic (same term -> same shard every call,
3907
3935
  /// or reads would look in the wrong file), agree with the raw xxh64-mask
@@ -6259,9 +6287,9 @@ mod tests {
6259
6287
  Some([(1, 0), (2, 0)].as_slice()),
6260
6288
  "both spellings must land on one normalized level-one key"
6261
6289
  );
6262
- // Level-two still composes AFTER level-one: it is the `[a-z0-9_]` strip
6263
- // of the NORMALIZED key ("aspirin oral" -> "aspirinoral"), not of the
6264
- // raw term (which would have been "oralaspirin").
6290
+ // Level-two still composes AFTER level-one: it is the Unicode-word-
6291
+ // character strip of the NORMALIZED key ("aspirin oral" -> "aspirinoral"),
6292
+ // not of the raw term (which would have been "oralaspirin").
6265
6293
  assert_eq!(
6266
6294
  terms.get("aspirinoral").map(Vec::as_slice),
6267
6295
  Some([(1, 0), (2, 0)].as_slice())
@@ -6275,16 +6303,20 @@ mod tests {
6275
6303
  assert_eq!(terms.len(), 1, "expected only the key `gene`: {terms:?}");
6276
6304
  assert!(terms.contains_key("gene"));
6277
6305
 
6278
- // Digit- and punctuation-bearing tokens survive verbatim (never stemmed)
6279
- // and non-ASCII folds case but skips stemming, so the level-two strip is
6280
- // the only place those characters disappear.
6306
+ // Digit- and punctuation-bearing tokens survive verbatim (never stemmed).
6307
+ // Level-two keeps Unicode word characters (é, β) and drops punctuation,
6308
+ // so "café" derives NO distinct level-two key (it equals its level-one
6309
+ // form) while "β-catenin" derives the level-two key "βcatenin" -- the
6310
+ // same key the Python query side derives through
6311
+ // `nlp::normalize_terms_l2` (issue #251).
6281
6312
  let mut terms: HashMap<String, Vec<(u32, u8)>> = HashMap::new();
6282
6313
  emit_term("TNF-alpha", (4, 0), &mut terms);
6283
6314
  emit_term("café", (5, 0), &mut terms);
6284
- for key in ["tnf-alpha", "tnfalpha", "café", "caf"] {
6315
+ emit_term("β-catenin", (5, 0), &mut terms);
6316
+ for key in ["tnf-alpha", "tnfalpha", "café", "β-catenin", "βcatenin"] {
6285
6317
  assert!(terms.contains_key(key), "missing key {key:?}: {terms:?}");
6286
6318
  }
6287
- assert_eq!(terms.len(), 4);
6319
+ assert_eq!(terms.len(), 5);
6288
6320
 
6289
6321
  // The gates run BEFORE stemming: "Hypothetical Protein" is rejected on
6290
6322
  // its cleaned spelling (post-normalization it reads "hypothet protein"
@@ -63,9 +63,10 @@ pub use fullmap::{
63
63
  };
64
64
  // Level-one normalization. `normalize_l1` is a plain Rust function consumed
65
65
  // by `fullmap::emit_term` (US-003) and by integration tests; `normalize_terms`
66
- // is the batch pyo3 surface the Python side calls (US-002). Both are
67
- // intentional public exports, not dead code.
68
- pub use nlp::{normalize_l1, normalize_terms};
66
+ // is the batch pyo3 surface the Python side calls (US-002), and
67
+ // `normalize_terms_l2` is the batch pyo3 surface for the shared level-two key
68
+ // filter (issue #251). All are intentional public exports, not dead code.
69
+ pub use nlp::{normalize_l1, normalize_terms, normalize_terms_l2};
69
70
 
70
71
  #[pymodule]
71
72
  fn rs(module: &Bound<'_, PyModule>) -> PyResult<()> {
@@ -84,6 +85,7 @@ fn rs(module: &Bound<'_, PyModule>) -> PyResult<()> {
84
85
  module.add_function(wrap_pyfunction!(fullmap::taxon_allowlist_identity, module)?)?;
85
86
  module.add_function(wrap_pyfunction!(ndjson::dedup_ndjson, module)?)?;
86
87
  module.add_function(wrap_pyfunction!(nlp::normalize_terms, module)?)?;
88
+ module.add_function(wrap_pyfunction!(nlp::normalize_terms_l2, module)?)?;
87
89
  module.add_function(wrap_pyfunction!(uuid::namespace_uuid, module)?)?;
88
90
  module.add_function(wrap_pyfunction!(xxh64, module)?)?;
89
91
  module.add_function(wrap_pyfunction!(xxh64_file, module)?)?;
@@ -116,9 +116,32 @@ pub fn normalize_terms(py: Python<'_>, terms: Vec<String>) -> Vec<String> {
116
116
  })
117
117
  }
118
118
 
119
+ /// Batch pyo3 surface for the fullmap `level_two` key filter: derive the
120
+ /// level-two form of a list of (usually level-one) terms, returning an
121
+ /// INDEX-ALIGNED list of strings (rayon's `into_par_iter` with `collect`
122
+ /// preserves input order).
123
+ ///
124
+ /// This is the query-side twin of the build-side key derivation: Python's
125
+ /// `nlp.level_two` default path calls this, and this calls the exact
126
+ /// `fullmap::level_two` filter `emit_term` stores build-side keys through, so
127
+ /// query and build level-two keys are byte-identical by construction. Before
128
+ /// issue #251 the query side ran a Unicode-aware polars `\\W+` regex while the
129
+ /// build stripped every non-ASCII character, so non-ASCII terms could never
130
+ /// match at level two. The GIL is released for the whole batch via `py.detach`
131
+ /// (the same pattern as `normalize_terms`).
132
+ #[pyfunction]
133
+ pub fn normalize_terms_l2(py: Python<'_>, terms: Vec<String>) -> Vec<String> {
134
+ py.detach(move || {
135
+ terms
136
+ .into_par_iter()
137
+ .map(|term| crate::fullmap::level_two(&term).into_owned())
138
+ .collect()
139
+ })
140
+ }
141
+
119
142
  #[cfg(test)]
120
143
  mod tests {
121
- use super::{normalize_l1, normalize_terms};
144
+ use super::{normalize_l1, normalize_terms, normalize_terms_l2};
122
145
  use pyo3::prelude::*;
123
146
  use std::borrow::Cow;
124
147
  use std::time::Instant;
@@ -259,6 +282,34 @@ mod tests {
259
282
  });
260
283
  }
261
284
 
285
+ #[test]
286
+ fn normalize_terms_l2_batch_is_index_aligned_and_handles_empty() {
287
+ // WHY: the Python level-two path pairs input and output positionally;
288
+ // output[i] must be fullmap::level_two(input[i]) -- including "" and an
289
+ // empty batch -- and non-ASCII terms must survive with the SAME keys the
290
+ // build side stores (issue #251: query and build L2 keys must never
291
+ // diverge again, which this pins against literal spellings).
292
+ pyo3::Python::initialize();
293
+ pyo3::Python::attach(|py| {
294
+ let func = pyo3::wrap_pyfunction!(normalize_terms_l2, py).unwrap();
295
+ let input = vec![
296
+ "au café lait".to_string(),
297
+ "β-catenin".to_string(),
298
+ "hello-world".to_string(),
299
+ String::new(),
300
+ ];
301
+ let output: Vec<String> = func.call1((input,)).unwrap().extract().unwrap();
302
+ assert_eq!(output, vec!["aucafélait", "βcatenin", "helloworld", ""]);
303
+
304
+ let empty: Vec<String> = func
305
+ .call1((Vec::<String>::new(),))
306
+ .unwrap()
307
+ .extract()
308
+ .unwrap();
309
+ assert!(empty.is_empty());
310
+ });
311
+ }
312
+
262
313
  #[test]
263
314
  fn normalize_terms_performance_one_million_terms() {
264
315
  // WHY: level-one normalization is a hot path for large tabular inputs;
@@ -61,7 +61,6 @@ alpha gene|HGNC:1
61
61
  alphagene|HGNC:1
62
62
  back\slash|HGNC:3
63
63
  backslash|HGNC:3
64
- caf|HGNC:2
65
64
  café|HGNC:2
66
65
  doid999|MONDO:1
67
66
  doid:999|MONDO:1
@@ -95,7 +94,6 @@ mondo:3|MONDO:3
95
94
  multi|HGNC:10,MONDO:3
96
95
  name quot|HGNC:3
97
96
  namequot|HGNC:3
98
- nave|HGNC:2
99
97
  naïve|HGNC:2
100
98
  ncbigene100|HGNC:1
101
99
  ncbigene101|HGNC:1
@@ -104,8 +102,9 @@ ncbigene:101|HGNC:1
104
102
  nullnam|HGNC:7
105
103
  realnam|HGNC:4
106
104
  share|HGNC:6,MONDO:2
107
- t|HGNC:3
108
105
  été|HGNC:3
106
+ β-catenin|HGNC:1
107
+ βcatenin|HGNC:1
109
108
  "#;
110
109
 
111
110
  // ---------------------------------------------------------------------------
@@ -61,7 +61,8 @@ pub struct SourceRow {
61
61
 
62
62
  // ---------------------------------------------------------------------------
63
63
  // Fixed embedded fixture. Deliberately covers: plain ASCII names, unicode names
64
- // (café / naïve), escaped JSON (quotes, backslashes, \uXXXX), alias fields
64
+ // (café / naïve), a Greek-letter name (β-Catenin, the issue #251 level-two pin),
65
+ // escaped JSON (quotes, backslashes, \uXXXX), alias fields
65
66
  // (id/name/categories/taxon), dead terms (12345/none/nan), equivalent
66
67
  // identifiers, multiple names per row, an empty names array, a null
67
68
  // preferred_name, a row with no names array, and a class row with no
@@ -70,7 +71,7 @@ pub struct SourceRow {
70
71
  // ---------------------------------------------------------------------------
71
72
 
72
73
  pub const SYNONYM_LINES: &[&str] = &[
73
- r#"{"curie":"HGNC:1","preferred_name":"Alpha Gene","names":["Alpha Gene","alpha"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#,
74
+ r#"{"curie":"HGNC:1","preferred_name":"Alpha Gene","names":["Alpha Gene","alpha","β-Catenin"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, // includes a Greek-letter name (issue #251 L2 pin)
74
75
  r#"{"curie":"HGNC:2","preferred_name":"café","names":["café","naïve"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#,
75
76
  r#"{"curie":"HGNC:3","preferred_name":"Esc","names":["\"Quoted Name\"","back\\slash","\u00e9t\u00e9"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#,
76
77
  r#"{"id":"MONDO:1","name":"Alias Disease","names":["alias disease"],"categories":["biolink:Disease"],"taxon":["NCBITaxon:0"]}"#,
@@ -71,6 +71,30 @@ def level_one(lf: pl.LazyFrame, col: str) -> pl.LazyFrame:
71
71
  return lf.with_columns(expr.alias(col))
72
72
 
73
73
 
74
+ def _level_two_series(values: pl.Series) -> pl.Series:
75
+ """Level-two normalize one whole string column through the Rust level-two filter.
76
+
77
+ Args:
78
+ values: Raw column batch (already cast to string by the caller).
79
+
80
+ Returns:
81
+ Series with every non-null value replaced by its level-two form,
82
+ nulls kept exactly where they were.
83
+
84
+ Notes:
85
+ Shares the exact batch shape of ``_normalize_terms_series`` (nulls never
86
+ reach Rust; one GIL-released Rust call per column instead of per row,
87
+ ``map_elements`` is forbidden in this repo) but applies the fullmap
88
+ build-side level-two key filter instead of the level-one pipeline, so
89
+ the query-side ``{col}_two`` keys are byte-identical to the keys
90
+ ``emit_term`` stores at build time (issue #251).
91
+ """
92
+ text: pl.Series = values.cast(pl.String)
93
+ indices: pl.Series = text.is_not_null().arg_true()
94
+ normalized: pl.Series = pl.Series(rs.normalize_terms_l2(text.drop_nulls().to_list()), dtype=pl.String)
95
+ return text.scatter(indices, normalized)
96
+
97
+
74
98
  def level_two(
75
99
  lf: pl.LazyFrame,
76
100
  col: str, # pyright: ignore
@@ -82,16 +106,28 @@ def level_two(
82
106
  The cleaned values are written to a new column named ``f"{col}{tag}"`` rather
83
107
  than overwriting the source column.
84
108
 
109
+ With the default pattern the column is cleaned through the compiled Rust
110
+ filter (``rs.normalize_terms_l2``) that derives the fullmap build-side
111
+ level-two keys, so query-side and build-side keys are byte-identical by
112
+ construction: non-ASCII word characters (accents, Greek letters) survive
113
+ while punctuation, symbols, and combining marks are dropped on both sides.
114
+ A custom ``regex`` keeps the vectorized Polars path and is a generic text
115
+ cleaner with no build-side key contract.
116
+
85
117
  Args:
86
118
  lf: Source LazyFrame.
87
119
  col: Name of the source column to clean.
88
- regex: Pattern whose matches are removed. Defaults to runs of non-word
89
- characters (``\\W+``).
120
+ regex: Pattern whose matches are removed. Only the default non-word
121
+ pattern (``\\W+``) routes through the shared Rust level-two filter;
122
+ any other pattern is applied with Polars directly.
90
123
  tag: Suffix appended to ``col`` to form the output column name.
91
124
 
92
125
  Returns:
93
126
  LazyFrame with the new tagged column added.
94
127
  """
95
- expr: pl.Expr = pl.col(col).str.replace_all(regex, "")
128
+ if regex == r"\W+":
129
+ expr: pl.Expr = pl.col(col).cast(pl.String).map_batches(_level_two_series, return_dtype=pl.String)
130
+ else:
131
+ expr = pl.col(col).str.replace_all(regex, "")
96
132
  col: str = col + tag
97
133
  return lf.with_columns(expr.alias(col))
@@ -26,6 +26,7 @@ def hydrate_sources(db: Path) -> list[str]: ...
26
26
  def lookup_fullmap_terms(db: Path, terms: list[str], return_format: str = "rows") -> list[dict[str, Any]]: ...
27
27
  def namespace_uuid(domain: str, values: list[str]) -> str: ...
28
28
  def normalize_terms(terms: list[str]) -> list[str]: ...
29
+ def normalize_terms_l2(terms: list[str]) -> list[str]: ...
29
30
  def taxon_allowlist_identity(taxon_allowlist: list[int]) -> str: ...
30
31
  def xxh64(data: str) -> str: ...
31
32
  def xxh64_file(path: str) -> str: ...
File without changes