tablassert 22.2.2__tar.gz → 22.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-22.2.2 → tablassert-22.3.1}/PKG-INFO +2 -1
- {tablassert-22.2.2 → tablassert-22.3.1}/README.md +1 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/pyproject.toml +1 -1
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/Cargo.lock +4 -4
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/src/fullmap.rs +46 -14
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/src/lib.rs +5 -3
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/src/nlp.rs +52 -1
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/tests/build_golden.rs +2 -3
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/tests/common/mod.rs +3 -2
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/nlp.py +39 -3
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/rs.pyi +1 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/LICENSE +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/Cargo.toml +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/examples/count_tables.rs +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/src/json.rs +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/src/ndjson.rs +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/src/uuid.rs +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/__init__.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/_lazy.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/biolink.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/cli.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/cli_entry.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/coerce.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/data/infores_catalog.yaml +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/enums.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/errors.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/extras.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/fullmap.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/infores_registry.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/ingests.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/lib.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/log.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/models.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/net.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/progress.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/qc.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/rig.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/runcache.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/study.py +0 -0
- {tablassert-22.2.2 → tablassert-22.3.1}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version: 22.
|
|
3
|
+
Version: 22.3.1
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -55,6 +55,7 @@ Project-URL: Source, https://github.com/SkyeAv/Tablassert
|
|
|
55
55
|
# Tablassert
|
|
56
56
|
|
|
57
57
|
[](https://badge.fury.io/py/tablassert)
|
|
58
|
+
[](https://pepy.tech/projects/tablassert)
|
|
58
59
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
59
60
|
[](https://skyeav.github.io/Tablassert/)
|
|
60
61
|
[](https://opensource.org/licenses/Apache-2.0)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# Tablassert
|
|
2
2
|
|
|
3
3
|
[](https://badge.fury.io/py/tablassert)
|
|
4
|
+
[](https://pepy.tech/projects/tablassert)
|
|
4
5
|
[](https://github.com/SkyeAv/Tablassert/actions/workflows/ci.yml)
|
|
5
6
|
[](https://skyeav.github.io/Tablassert/)
|
|
6
7
|
[](https://opensource.org/licenses/Apache-2.0)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "22.
|
|
3
|
+
version = "22.3.1"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -141,9 +141,9 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
|
|
141
141
|
|
|
142
142
|
[[package]]
|
|
143
143
|
name = "flate2"
|
|
144
|
-
version = "1.1.
|
|
144
|
+
version = "1.1.11"
|
|
145
145
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
146
|
-
checksum = "
|
|
146
|
+
checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4"
|
|
147
147
|
dependencies = [
|
|
148
148
|
"zlib-rs",
|
|
149
149
|
]
|
|
@@ -626,9 +626,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
|
|
626
626
|
|
|
627
627
|
[[package]]
|
|
628
628
|
name = "uuid"
|
|
629
|
-
version = "1.29.
|
|
629
|
+
version = "1.29.1"
|
|
630
630
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
631
|
-
checksum = "
|
|
631
|
+
checksum = "4a06f08a856b009bc7bbaf7d3e79808595d032dd19074c0c2ef26e49e03e6bdc"
|
|
632
632
|
dependencies = [
|
|
633
633
|
"js-sys",
|
|
634
634
|
"md-5",
|
|
@@ -327,11 +327,19 @@ fn level_one(value: &str) -> Cow<'_, str> {
|
|
|
327
327
|
}
|
|
328
328
|
}
|
|
329
329
|
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
330
|
+
/// Derive the level-two key of a (usually level-one) term: keep every Unicode
|
|
331
|
+
/// alphanumeric character or underscore, drop everything else. This is THE
|
|
332
|
+
/// single level-two key definition: `emit_term` stores build-side keys through
|
|
333
|
+
/// it and `nlp::normalize_terms_l2` derives query-side keys through it, so the
|
|
334
|
+
/// two sides cannot drift apart (the query side used a Unicode-aware polars
|
|
335
|
+
/// regex while the build stripped all non-ASCII, which made non-ASCII terms
|
|
336
|
+
/// unmatchable at level two).
|
|
337
|
+
pub(crate) fn level_two(value: &str) -> Cow<'_, str> {
|
|
338
|
+
// Fast path: for an already-lowercase ASCII input, keeping Unicode word
|
|
339
|
+
// characters is exactly keeping [a-z0-9_], so when every byte qualifies the
|
|
340
|
+
// filter is a no-op and we borrow unchanged (the common case
|
|
341
|
+
// post-level_one). Any uppercase or non-ASCII byte falls through to the
|
|
342
|
+
// filtering slow path.
|
|
335
343
|
if value
|
|
336
344
|
.bytes()
|
|
337
345
|
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
|
|
@@ -341,7 +349,7 @@ fn level_two(value: &str) -> Cow<'_, str> {
|
|
|
341
349
|
Cow::Owned(
|
|
342
350
|
value
|
|
343
351
|
.chars()
|
|
344
|
-
.filter(|c| c.
|
|
352
|
+
.filter(|c| c.is_alphanumeric() || *c == '_')
|
|
345
353
|
.collect(),
|
|
346
354
|
)
|
|
347
355
|
}
|
|
@@ -3902,6 +3910,26 @@ mod tests {
|
|
|
3902
3910
|
assert_eq!(level_two("brca-1"), "brca1");
|
|
3903
3911
|
}
|
|
3904
3912
|
|
|
3913
|
+
#[test]
|
|
3914
|
+
fn level_two_keeps_non_ascii_word_characters() {
|
|
3915
|
+
// WHY: the build side and the Python query side must derive the SAME
|
|
3916
|
+
// level-two key for every level-one string (issue #251). The filter
|
|
3917
|
+
// keeps Unicode alphanumeric characters (accents, Greek letters,
|
|
3918
|
+
// non-ASCII digits) plus underscore, and drops punctuation, symbols,
|
|
3919
|
+
// combining marks, and whitespace -- matching the query side because it
|
|
3920
|
+
// IS the implementation the query side calls.
|
|
3921
|
+
assert_eq!(level_two("au café lait"), "aucafélait");
|
|
3922
|
+
assert_eq!(level_two("β-catenin"), "βcatenin");
|
|
3923
|
+
assert_eq!(level_two("naïve"), "naïve");
|
|
3924
|
+
assert_eq!(level_two("p_value"), "p_value");
|
|
3925
|
+
// Combining marks are neither alphanumeric nor underscore: a decomposed
|
|
3926
|
+
// form must collapse exactly like the query side does.
|
|
3927
|
+
assert_eq!(level_two("cafe\u{301}"), "cafe");
|
|
3928
|
+
// Unchanged ASCII behavior: punctuation still splits, empty stays empty.
|
|
3929
|
+
assert_eq!(level_two("hello-world"), "helloworld");
|
|
3930
|
+
assert_eq!(level_two(""), "");
|
|
3931
|
+
}
|
|
3932
|
+
|
|
3905
3933
|
/// `term_shard` is the single routing oracle shared by the writer and the
|
|
3906
3934
|
/// reader, so it must be deterministic (same term -> same shard every call,
|
|
3907
3935
|
/// or reads would look in the wrong file), agree with the raw xxh64-mask
|
|
@@ -6259,9 +6287,9 @@ mod tests {
|
|
|
6259
6287
|
Some([(1, 0), (2, 0)].as_slice()),
|
|
6260
6288
|
"both spellings must land on one normalized level-one key"
|
|
6261
6289
|
);
|
|
6262
|
-
// Level-two still composes AFTER level-one: it is the
|
|
6263
|
-
// of the NORMALIZED key ("aspirin oral" -> "aspirinoral"),
|
|
6264
|
-
// raw term (which would have been "oralaspirin").
|
|
6290
|
+
// Level-two still composes AFTER level-one: it is the Unicode-word-
|
|
6291
|
+
// character strip of the NORMALIZED key ("aspirin oral" -> "aspirinoral"),
|
|
6292
|
+
// not of the raw term (which would have been "oralaspirin").
|
|
6265
6293
|
assert_eq!(
|
|
6266
6294
|
terms.get("aspirinoral").map(Vec::as_slice),
|
|
6267
6295
|
Some([(1, 0), (2, 0)].as_slice())
|
|
@@ -6275,16 +6303,20 @@ mod tests {
|
|
|
6275
6303
|
assert_eq!(terms.len(), 1, "expected only the key `gene`: {terms:?}");
|
|
6276
6304
|
assert!(terms.contains_key("gene"));
|
|
6277
6305
|
|
|
6278
|
-
// Digit- and punctuation-bearing tokens survive verbatim (never stemmed)
|
|
6279
|
-
//
|
|
6280
|
-
//
|
|
6306
|
+
// Digit- and punctuation-bearing tokens survive verbatim (never stemmed).
|
|
6307
|
+
// Level-two keeps Unicode word characters (é, β) and drops punctuation,
|
|
6308
|
+
// so "café" derives NO distinct level-two key (it equals its level-one
|
|
6309
|
+
// form) while "β-catenin" derives the level-two key "βcatenin" -- the
|
|
6310
|
+
// same key the Python query side derives through
|
|
6311
|
+
// `nlp::normalize_terms_l2` (issue #251).
|
|
6281
6312
|
let mut terms: HashMap<String, Vec<(u32, u8)>> = HashMap::new();
|
|
6282
6313
|
emit_term("TNF-alpha", (4, 0), &mut terms);
|
|
6283
6314
|
emit_term("café", (5, 0), &mut terms);
|
|
6284
|
-
|
|
6315
|
+
emit_term("β-catenin", (5, 0), &mut terms);
|
|
6316
|
+
for key in ["tnf-alpha", "tnfalpha", "café", "β-catenin", "βcatenin"] {
|
|
6285
6317
|
assert!(terms.contains_key(key), "missing key {key:?}: {terms:?}");
|
|
6286
6318
|
}
|
|
6287
|
-
assert_eq!(terms.len(),
|
|
6319
|
+
assert_eq!(terms.len(), 5);
|
|
6288
6320
|
|
|
6289
6321
|
// The gates run BEFORE stemming: "Hypothetical Protein" is rejected on
|
|
6290
6322
|
// its cleaned spelling (post-normalization it reads "hypothet protein"
|
|
@@ -63,9 +63,10 @@ pub use fullmap::{
|
|
|
63
63
|
};
|
|
64
64
|
// Level-one normalization. `normalize_l1` is a plain Rust function consumed
|
|
65
65
|
// by `fullmap::emit_term` (US-003) and by integration tests; `normalize_terms`
|
|
66
|
-
// is the batch pyo3 surface the Python side calls (US-002)
|
|
67
|
-
//
|
|
68
|
-
|
|
66
|
+
// is the batch pyo3 surface the Python side calls (US-002), and
|
|
67
|
+
// `normalize_terms_l2` is the batch pyo3 surface for the shared level-two key
|
|
68
|
+
// filter (issue #251). All are intentional public exports, not dead code.
|
|
69
|
+
pub use nlp::{normalize_l1, normalize_terms, normalize_terms_l2};
|
|
69
70
|
|
|
70
71
|
#[pymodule]
|
|
71
72
|
fn rs(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
@@ -84,6 +85,7 @@ fn rs(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
|
84
85
|
module.add_function(wrap_pyfunction!(fullmap::taxon_allowlist_identity, module)?)?;
|
|
85
86
|
module.add_function(wrap_pyfunction!(ndjson::dedup_ndjson, module)?)?;
|
|
86
87
|
module.add_function(wrap_pyfunction!(nlp::normalize_terms, module)?)?;
|
|
88
|
+
module.add_function(wrap_pyfunction!(nlp::normalize_terms_l2, module)?)?;
|
|
87
89
|
module.add_function(wrap_pyfunction!(uuid::namespace_uuid, module)?)?;
|
|
88
90
|
module.add_function(wrap_pyfunction!(xxh64, module)?)?;
|
|
89
91
|
module.add_function(wrap_pyfunction!(xxh64_file, module)?)?;
|
|
@@ -116,9 +116,32 @@ pub fn normalize_terms(py: Python<'_>, terms: Vec<String>) -> Vec<String> {
|
|
|
116
116
|
})
|
|
117
117
|
}
|
|
118
118
|
|
|
119
|
+
/// Batch pyo3 surface for the fullmap `level_two` key filter: derive the
|
|
120
|
+
/// level-two form of a list of (usually level-one) terms, returning an
|
|
121
|
+
/// INDEX-ALIGNED list of strings (rayon's `into_par_iter` with `collect`
|
|
122
|
+
/// preserves input order).
|
|
123
|
+
///
|
|
124
|
+
/// This is the query-side twin of the build-side key derivation: Python's
|
|
125
|
+
/// `nlp.level_two` default path calls this, and this calls the exact
|
|
126
|
+
/// `fullmap::level_two` filter `emit_term` stores build-side keys through, so
|
|
127
|
+
/// query and build level-two keys are byte-identical by construction. Before
|
|
128
|
+
/// issue #251 the query side ran a Unicode-aware polars `\\W+` regex while the
|
|
129
|
+
/// build stripped every non-ASCII character, so non-ASCII terms could never
|
|
130
|
+
/// match at level two. The GIL is released for the whole batch via `py.detach`
|
|
131
|
+
/// (the same pattern as `normalize_terms`).
|
|
132
|
+
#[pyfunction]
|
|
133
|
+
pub fn normalize_terms_l2(py: Python<'_>, terms: Vec<String>) -> Vec<String> {
|
|
134
|
+
py.detach(move || {
|
|
135
|
+
terms
|
|
136
|
+
.into_par_iter()
|
|
137
|
+
.map(|term| crate::fullmap::level_two(&term).into_owned())
|
|
138
|
+
.collect()
|
|
139
|
+
})
|
|
140
|
+
}
|
|
141
|
+
|
|
119
142
|
#[cfg(test)]
|
|
120
143
|
mod tests {
|
|
121
|
-
use super::{normalize_l1, normalize_terms};
|
|
144
|
+
use super::{normalize_l1, normalize_terms, normalize_terms_l2};
|
|
122
145
|
use pyo3::prelude::*;
|
|
123
146
|
use std::borrow::Cow;
|
|
124
147
|
use std::time::Instant;
|
|
@@ -259,6 +282,34 @@ mod tests {
|
|
|
259
282
|
});
|
|
260
283
|
}
|
|
261
284
|
|
|
285
|
+
#[test]
|
|
286
|
+
fn normalize_terms_l2_batch_is_index_aligned_and_handles_empty() {
|
|
287
|
+
// WHY: the Python level-two path pairs input and output positionally;
|
|
288
|
+
// output[i] must be fullmap::level_two(input[i]) -- including "" and an
|
|
289
|
+
// empty batch -- and non-ASCII terms must survive with the SAME keys the
|
|
290
|
+
// build side stores (issue #251: query and build L2 keys must never
|
|
291
|
+
// diverge again, which this pins against literal spellings).
|
|
292
|
+
pyo3::Python::initialize();
|
|
293
|
+
pyo3::Python::attach(|py| {
|
|
294
|
+
let func = pyo3::wrap_pyfunction!(normalize_terms_l2, py).unwrap();
|
|
295
|
+
let input = vec![
|
|
296
|
+
"au café lait".to_string(),
|
|
297
|
+
"β-catenin".to_string(),
|
|
298
|
+
"hello-world".to_string(),
|
|
299
|
+
String::new(),
|
|
300
|
+
];
|
|
301
|
+
let output: Vec<String> = func.call1((input,)).unwrap().extract().unwrap();
|
|
302
|
+
assert_eq!(output, vec!["aucafélait", "βcatenin", "helloworld", ""]);
|
|
303
|
+
|
|
304
|
+
let empty: Vec<String> = func
|
|
305
|
+
.call1((Vec::<String>::new(),))
|
|
306
|
+
.unwrap()
|
|
307
|
+
.extract()
|
|
308
|
+
.unwrap();
|
|
309
|
+
assert!(empty.is_empty());
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
|
|
262
313
|
#[test]
|
|
263
314
|
fn normalize_terms_performance_one_million_terms() {
|
|
264
315
|
// WHY: level-one normalization is a hot path for large tabular inputs;
|
|
@@ -61,7 +61,6 @@ alpha gene|HGNC:1
|
|
|
61
61
|
alphagene|HGNC:1
|
|
62
62
|
back\slash|HGNC:3
|
|
63
63
|
backslash|HGNC:3
|
|
64
|
-
caf|HGNC:2
|
|
65
64
|
café|HGNC:2
|
|
66
65
|
doid999|MONDO:1
|
|
67
66
|
doid:999|MONDO:1
|
|
@@ -95,7 +94,6 @@ mondo:3|MONDO:3
|
|
|
95
94
|
multi|HGNC:10,MONDO:3
|
|
96
95
|
name quot|HGNC:3
|
|
97
96
|
namequot|HGNC:3
|
|
98
|
-
nave|HGNC:2
|
|
99
97
|
naïve|HGNC:2
|
|
100
98
|
ncbigene100|HGNC:1
|
|
101
99
|
ncbigene101|HGNC:1
|
|
@@ -104,8 +102,9 @@ ncbigene:101|HGNC:1
|
|
|
104
102
|
nullnam|HGNC:7
|
|
105
103
|
realnam|HGNC:4
|
|
106
104
|
share|HGNC:6,MONDO:2
|
|
107
|
-
t|HGNC:3
|
|
108
105
|
été|HGNC:3
|
|
106
|
+
β-catenin|HGNC:1
|
|
107
|
+
βcatenin|HGNC:1
|
|
109
108
|
"#;
|
|
110
109
|
|
|
111
110
|
// ---------------------------------------------------------------------------
|
|
@@ -61,7 +61,8 @@ pub struct SourceRow {
|
|
|
61
61
|
|
|
62
62
|
// ---------------------------------------------------------------------------
|
|
63
63
|
// Fixed embedded fixture. Deliberately covers: plain ASCII names, unicode names
|
|
64
|
-
// (café / naïve),
|
|
64
|
+
// (café / naïve), a Greek-letter name (β-Catenin, the issue #251 level-two pin),
|
|
65
|
+
// escaped JSON (quotes, backslashes, \uXXXX), alias fields
|
|
65
66
|
// (id/name/categories/taxon), dead terms (12345/none/nan), equivalent
|
|
66
67
|
// identifiers, multiple names per row, an empty names array, a null
|
|
67
68
|
// preferred_name, a row with no names array, and a class row with no
|
|
@@ -70,7 +71,7 @@ pub struct SourceRow {
|
|
|
70
71
|
// ---------------------------------------------------------------------------
|
|
71
72
|
|
|
72
73
|
pub const SYNONYM_LINES: &[&str] = &[
|
|
73
|
-
r#"{"curie":"HGNC:1","preferred_name":"Alpha Gene","names":["Alpha Gene","alpha"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#,
|
|
74
|
+
r#"{"curie":"HGNC:1","preferred_name":"Alpha Gene","names":["Alpha Gene","alpha","β-Catenin"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, // includes a Greek-letter name (issue #251 L2 pin)
|
|
74
75
|
r#"{"curie":"HGNC:2","preferred_name":"café","names":["café","naïve"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#,
|
|
75
76
|
r#"{"curie":"HGNC:3","preferred_name":"Esc","names":["\"Quoted Name\"","back\\slash","\u00e9t\u00e9"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#,
|
|
76
77
|
r#"{"id":"MONDO:1","name":"Alias Disease","names":["alias disease"],"categories":["biolink:Disease"],"taxon":["NCBITaxon:0"]}"#,
|
|
@@ -71,6 +71,30 @@ def level_one(lf: pl.LazyFrame, col: str) -> pl.LazyFrame:
|
|
|
71
71
|
return lf.with_columns(expr.alias(col))
|
|
72
72
|
|
|
73
73
|
|
|
74
|
+
def _level_two_series(values: pl.Series) -> pl.Series:
|
|
75
|
+
"""Level-two normalize one whole string column through the Rust level-two filter.
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
values: Raw column batch (already cast to string by the caller).
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
Series with every non-null value replaced by its level-two form,
|
|
82
|
+
nulls kept exactly where they were.
|
|
83
|
+
|
|
84
|
+
Notes:
|
|
85
|
+
Shares the exact batch shape of ``_normalize_terms_series`` (nulls never
|
|
86
|
+
reach Rust; one GIL-released Rust call per column instead of per row,
|
|
87
|
+
``map_elements`` is forbidden in this repo) but applies the fullmap
|
|
88
|
+
build-side level-two key filter instead of the level-one pipeline, so
|
|
89
|
+
the query-side ``{col}_two`` keys are byte-identical to the keys
|
|
90
|
+
``emit_term`` stores at build time (issue #251).
|
|
91
|
+
"""
|
|
92
|
+
text: pl.Series = values.cast(pl.String)
|
|
93
|
+
indices: pl.Series = text.is_not_null().arg_true()
|
|
94
|
+
normalized: pl.Series = pl.Series(rs.normalize_terms_l2(text.drop_nulls().to_list()), dtype=pl.String)
|
|
95
|
+
return text.scatter(indices, normalized)
|
|
96
|
+
|
|
97
|
+
|
|
74
98
|
def level_two(
|
|
75
99
|
lf: pl.LazyFrame,
|
|
76
100
|
col: str, # pyright: ignore
|
|
@@ -82,16 +106,28 @@ def level_two(
|
|
|
82
106
|
The cleaned values are written to a new column named ``f"{col}{tag}"`` rather
|
|
83
107
|
than overwriting the source column.
|
|
84
108
|
|
|
109
|
+
With the default pattern the column is cleaned through the compiled Rust
|
|
110
|
+
filter (``rs.normalize_terms_l2``) that derives the fullmap build-side
|
|
111
|
+
level-two keys, so query-side and build-side keys are byte-identical by
|
|
112
|
+
construction: non-ASCII word characters (accents, Greek letters) survive
|
|
113
|
+
while punctuation, symbols, and combining marks are dropped on both sides.
|
|
114
|
+
A custom ``regex`` keeps the vectorized Polars path and is a generic text
|
|
115
|
+
cleaner with no build-side key contract.
|
|
116
|
+
|
|
85
117
|
Args:
|
|
86
118
|
lf: Source LazyFrame.
|
|
87
119
|
col: Name of the source column to clean.
|
|
88
|
-
regex: Pattern whose matches are removed.
|
|
89
|
-
|
|
120
|
+
regex: Pattern whose matches are removed. Only the default non-word
|
|
121
|
+
pattern (``\\W+``) routes through the shared Rust level-two filter;
|
|
122
|
+
any other pattern is applied with Polars directly.
|
|
90
123
|
tag: Suffix appended to ``col`` to form the output column name.
|
|
91
124
|
|
|
92
125
|
Returns:
|
|
93
126
|
LazyFrame with the new tagged column added.
|
|
94
127
|
"""
|
|
95
|
-
|
|
128
|
+
if regex == r"\W+":
|
|
129
|
+
expr: pl.Expr = pl.col(col).cast(pl.String).map_batches(_level_two_series, return_dtype=pl.String)
|
|
130
|
+
else:
|
|
131
|
+
expr = pl.col(col).str.replace_all(regex, "")
|
|
96
132
|
col: str = col + tag
|
|
97
133
|
return lf.with_columns(expr.alias(col))
|
|
@@ -26,6 +26,7 @@ def hydrate_sources(db: Path) -> list[str]: ...
|
|
|
26
26
|
def lookup_fullmap_terms(db: Path, terms: list[str], return_format: str = "rows") -> list[dict[str, Any]]: ...
|
|
27
27
|
def namespace_uuid(domain: str, values: list[str]) -> str: ...
|
|
28
28
|
def normalize_terms(terms: list[str]) -> list[str]: ...
|
|
29
|
+
def normalize_terms_l2(terms: list[str]) -> list[str]: ...
|
|
29
30
|
def taxon_allowlist_identity(taxon_allowlist: list[int]) -> str: ...
|
|
30
31
|
def xxh64(data: str) -> str: ...
|
|
31
32
|
def xxh64_file(path: str) -> str: ...
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|