strsimpy 0.4.0__tar.gz → 0.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. {strsimpy-0.4.0 → strsimpy-0.4.2}/Cargo.lock +2 -2
  2. {strsimpy-0.4.0 → strsimpy-0.4.2}/Cargo.toml +1 -1
  3. {strsimpy-0.4.0 → strsimpy-0.4.2}/PKG-INFO +29 -1
  4. {strsimpy-0.4.0 → strsimpy-0.4.2}/README.md +28 -0
  5. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/jaro_winkler.rs +6 -2
  6. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/ngram.rs +18 -11
  7. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-py/src/algorithms.rs +78 -8
  8. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-py/src/sift4.rs +15 -3
  9. strsimpy-0.4.2/crates/strsimpy-py/src/utils.rs +59 -0
  10. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-py/src/weighted.rs +21 -0
  11. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/__init__.py +1 -1
  12. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_api.py +1 -1
  13. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_cosine.py +19 -0
  14. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_jaro_winkler.py +7 -0
  15. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_ngram.py +28 -0
  16. strsimpy-0.4.2/tests/test_pickle.py +108 -0
  17. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_qgram.py +16 -0
  18. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_sift4.py +30 -1
  19. strsimpy-0.4.0/crates/strsimpy-py/src/utils.rs +0 -28
  20. {strsimpy-0.4.0 → strsimpy-0.4.2}/LICENSE +0 -0
  21. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/Cargo.toml +0 -0
  22. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/cosine.rs +0 -0
  23. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/damerau.rs +0 -0
  24. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/jaccard.rs +0 -0
  25. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/levenshtein.rs +0 -0
  26. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/lib.rs +0 -0
  27. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/longest_common_subsequence.rs +0 -0
  28. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/metric_lcs.rs +0 -0
  29. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/normalized_levenshtein.rs +0 -0
  30. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/optimal_string_alignment.rs +0 -0
  31. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/overlap_coefficient.rs +0 -0
  32. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/qgram.rs +0 -0
  33. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/shingle.rs +0 -0
  34. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/sift4.rs +0 -0
  35. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/sorensen_dice.rs +0 -0
  36. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-core/src/weighted_levenshtein.rs +0 -0
  37. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-py/Cargo.toml +0 -0
  38. {strsimpy-0.4.0 → strsimpy-0.4.2}/crates/strsimpy-py/src/lib.rs +0 -0
  39. {strsimpy-0.4.0 → strsimpy-0.4.2}/pyproject.toml +0 -0
  40. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/_base.py +0 -0
  41. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/cosine.py +0 -0
  42. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/damerau.py +0 -0
  43. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/jaccard.py +0 -0
  44. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/jaro_winkler.py +0 -0
  45. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/levenshtein.py +0 -0
  46. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/longest_common_subsequence.py +0 -0
  47. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/metric_lcs.py +0 -0
  48. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/ngram.py +0 -0
  49. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/normalized_levenshtein.py +0 -0
  50. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/optimal_string_alignment.py +0 -0
  51. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/overlap_coefficient.py +0 -0
  52. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/qgram.py +0 -0
  53. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/shingle_based.py +0 -0
  54. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/sift4.py +0 -0
  55. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/sorensen_dice.py +0 -0
  56. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/string_distance.py +0 -0
  57. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/string_similarity.py +0 -0
  58. {strsimpy-0.4.0 → strsimpy-0.4.2}/python/strsimpy/weighted_levenshtein.py +0 -0
  59. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_damerau.py +0 -0
  60. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_jaccard.py +0 -0
  61. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_levenshtein.py +0 -0
  62. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_longest_common_subsequence.py +0 -0
  63. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_metric_lcs.py +0 -0
  64. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_normalized_levenshtein.py +0 -0
  65. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_optimal_string_alignment.py +0 -0
  66. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_overlap_coefficient.py +0 -0
  67. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_sorensen_dice.py +0 -0
  68. {strsimpy-0.4.0 → strsimpy-0.4.2}/tests/test_weighted_levenshtein.py +0 -0
@@ -103,11 +103,11 @@ dependencies = [
103
103
 
104
104
  [[package]]
105
105
  name = "strsimpy-core"
106
- version = "0.4.0"
106
+ version = "0.4.2"
107
107
 
108
108
  [[package]]
109
109
  name = "strsimpy-py"
110
- version = "0.4.0"
110
+ version = "0.4.2"
111
111
  dependencies = [
112
112
  "pyo3",
113
113
  "strsimpy-core",
@@ -3,7 +3,7 @@ members = ["crates/strsimpy-core", "crates/strsimpy-py"]
3
3
  resolver = "2"
4
4
 
5
5
  [workspace.package]
6
- version = "0.4.0"
6
+ version = "0.4.2"
7
7
  edition = "2021"
8
8
  rust-version = "1.85"
9
9
  license = "MIT"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: strsimpy
3
- Version: 0.4.0
3
+ Version: 0.4.2
4
4
  Classifier: Development Status :: 4 - Beta
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: License :: OSI Approved :: MIT License
@@ -478,6 +478,12 @@ s.distance('This is the first string', 'And this is another string') # 11
478
478
  s.distance('Lorem ipsum dolor sit amet, consectetur adipiscing elit.', 'Amet Lorm ispum dolor sit amet, consetetur adixxxpiscing elit.', maxoffset=10)
479
479
  ```
480
480
 
481
+ A tokenizer can be supplied as a callable or as one of the named presets
482
+ `"ngram"`, `"wordsplit"` and `"characterfrequency"`. The `"ngram"` preset
483
+ splits into overlapping character bigrams (`SIFT4Options.ngramtokenizer` with
484
+ its default `n=2`); use `functools.partial(SIFT4Options.ngramtokenizer, n=3)`
485
+ to choose another n-gram size.
486
+
481
487
  ## Development
482
488
 
483
489
  Prerequisites: Python 3.10+, Rust 1.85+ and a C compiler. [maturin](https://www.maturin.rs/) drives the build.
@@ -513,6 +519,28 @@ them to PyPI together with the source distribution.
513
519
 
514
520
  ## Changelog
515
521
 
522
+ ### 0.4.2
523
+ - Restored `JaroWinkler.matches()` as a public static method returning
524
+ `[matches, transpositions, prefix length, longer input length]` (#45).
525
+ - Added pickle support to every calculator, preserving constructor options
526
+ (`JaroWinkler` threshold, `NGram.n`, the shingle `k` and weighted-Levenshtein
527
+ cost functions). This restores multiprocessing workflows that pass bound
528
+ `distance` methods to workers (#44).
529
+ - `Cosine.similarity_profiles()` and `QGram.distance_profile()` accept any
530
+ Python mapping again, including `MappingProxyType`, `UserDict` and `Counter`
531
+ (#46).
532
+ - Fixed `SIFT4Options.ngramtokenizer()` dropping the final two n-grams, which
533
+ made suffix-only differences invisible; it now returns all `len(s) - n + 1`
534
+ windows (#42).
535
+ - Fixed the `tokenizer="ngram"` SIFT4 preset raising `TypeError`; n-gram
536
+ tokenization now defaults to `n=2` (#43).
537
+
538
+ ### 0.4.1
539
+ - Fixed `NGram` treating literal newline characters in the input as its internal
540
+ padding marker, which produced incorrect edit costs and normalized distances
541
+ above 1 (for example, `NGram(1).distance("\na", "\nb")` returned 1.5 instead
542
+ of 0.5) (#41). Thanks to @Narek-Khachikyan.
543
+
516
544
  ### 0.4.0
517
545
  - Reimplemented all algorithms in Rust and exposed them through PyO3. The
518
546
  Python API, package root exports and per-module imports are unchanged.
@@ -451,6 +451,12 @@ s.distance('This is the first string', 'And this is another string') # 11
451
451
  s.distance('Lorem ipsum dolor sit amet, consectetur adipiscing elit.', 'Amet Lorm ispum dolor sit amet, consetetur adixxxpiscing elit.', maxoffset=10)
452
452
  ```
453
453
 
454
+ A tokenizer can be supplied as a callable or as one of the named presets
455
+ `"ngram"`, `"wordsplit"` and `"characterfrequency"`. The `"ngram"` preset
456
+ splits into overlapping character bigrams (`SIFT4Options.ngramtokenizer` with
457
+ its default `n=2`); use `functools.partial(SIFT4Options.ngramtokenizer, n=3)`
458
+ to choose another n-gram size.
459
+
454
460
  ## Development
455
461
 
456
462
  Prerequisites: Python 3.10+, Rust 1.85+ and a C compiler. [maturin](https://www.maturin.rs/) drives the build.
@@ -486,6 +492,28 @@ them to PyPI together with the source distribution.
486
492
 
487
493
  ## Changelog
488
494
 
495
+ ### 0.4.2
496
+ - Restored `JaroWinkler.matches()` as a public static method returning
497
+ `[matches, transpositions, prefix length, longer input length]` (#45).
498
+ - Added pickle support to every calculator, preserving constructor options
499
+ (`JaroWinkler` threshold, `NGram.n`, the shingle `k` and weighted-Levenshtein
500
+ cost functions). This restores multiprocessing workflows that pass bound
501
+ `distance` methods to workers (#44).
502
+ - `Cosine.similarity_profiles()` and `QGram.distance_profile()` accept any
503
+ Python mapping again, including `MappingProxyType`, `UserDict` and `Counter`
504
+ (#46).
505
+ - Fixed `SIFT4Options.ngramtokenizer()` dropping the final two n-grams, which
506
+ made suffix-only differences invisible; it now returns all `len(s) - n + 1`
507
+ windows (#42).
508
+ - Fixed the `tokenizer="ngram"` SIFT4 preset raising `TypeError`; n-gram
509
+ tokenization now defaults to `n=2` (#43).
510
+
511
+ ### 0.4.1
512
+ - Fixed `NGram` treating literal newline characters in the input as its internal
513
+ padding marker, which produced incorrect edit costs and normalized distances
514
+ above 1 (for example, `NGram(1).distance("\na", "\nb")` returned 1.5 instead
515
+ of 0.5) (#41). Thanks to @Narek-Khachikyan.
516
+
489
517
  ### 0.4.0
490
518
  - Reimplemented all algorithms in Rust and exposed them through PyO3. The
491
519
  Python API, package root exports and per-module imports are unchanged.
@@ -4,7 +4,10 @@
4
4
  pub const DEFAULT_THRESHOLD: f64 = 0.7;
5
5
  const PREFIX_SCALE: f64 = 0.1;
6
6
 
7
- pub(crate) struct MatchDetails {
7
+ /// Details collected while matching two strings: match count, transpositions,
8
+ /// common prefix length and the length of the longer input.
9
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
10
+ pub struct MatchDetails {
8
11
  pub matches: usize,
9
12
  pub transpositions: usize,
10
13
  pub prefix: usize,
@@ -51,7 +54,8 @@ pub fn distance_with_threshold(s0: &str, s1: &str, threshold: f64) -> f64 {
51
54
  1.0 - similarity_with_threshold(s0, s1, threshold)
52
55
  }
53
56
 
54
- pub(crate) fn matches(s0: &str, s1: &str) -> MatchDetails {
57
+ /// Matching details for the Jaro-Winkler algorithm.
58
+ pub fn matches(s0: &str, s1: &str) -> MatchDetails {
55
59
  let chars0: Vec<char> = s0.chars().collect();
56
60
  let chars1: Vec<char> = s1.chars().collect();
57
61
 
@@ -4,13 +4,14 @@
4
4
  ///
5
5
  /// The implementation follows Kondrak's algorithm as used by
6
6
  /// `java-string-similarity`: the first string is prefixed with `n - 1`
7
- /// newline padding characters and the edit distance between n-gram profiles
8
- /// is normalized by the length of the longest input string.
7
+ /// padding markers distinct from input characters. The edit distance between
8
+ /// n-gram profiles is normalized by the length of the longest input string.
9
9
  pub fn distance(s0: &str, s1: &str, n: usize) -> f64 {
10
10
  assert!(n > 0, "n must be greater than zero");
11
11
 
12
- let a: Vec<char> = s0.chars().collect();
13
- let b: Vec<char> = s1.chars().collect();
12
+ // None is reserved for padding; every input character is Some(c).
13
+ let a: Vec<Option<char>> = s0.chars().map(Some).collect();
14
+ let b: Vec<Option<char>> = s1.chars().map(Some).collect();
14
15
 
15
16
  if a == b {
16
17
  return 0.0;
@@ -32,20 +33,18 @@ pub fn distance(s0: &str, s1: &str, n: usize) -> f64 {
32
33
  return 1.0 - matched as f64 / sl.max(tl) as f64;
33
34
  }
34
35
 
35
- const SPECIAL: char = '\n';
36
-
37
- let mut sa = vec![SPECIAL; sl + n - 1];
36
+ let mut sa = vec![None; sl + n - 1];
38
37
  sa[n - 1..].copy_from_slice(&a);
39
38
 
40
39
  let mut previous: Vec<f64> = (0..=sl).map(|i| i as f64).collect();
41
40
  let mut current = vec![0.0f64; sl + 1];
42
- let mut t_j = vec![SPECIAL; n];
41
+ let mut t_j = vec![None; n];
43
42
 
44
43
  for j in 1..=tl {
45
44
  if j < n {
46
45
  let padding = n - j;
47
- t_j[..padding].fill(SPECIAL);
48
- t_j[padding..n].copy_from_slice(&b[..n - padding]);
46
+ t_j[..padding].fill(None);
47
+ t_j[padding..n].copy_from_slice(&b[..j]);
49
48
  } else {
50
49
  t_j.copy_from_slice(&b[j - n..j]);
51
50
  }
@@ -58,7 +57,7 @@ pub fn distance(s0: &str, s1: &str, n: usize) -> f64 {
58
57
  let sc = sa[i - 1 + ni];
59
58
  if sc != t_j[ni] {
60
59
  cost += 1;
61
- } else if sc == SPECIAL {
60
+ } else if sc.is_none() {
62
61
  denominator -= 1;
63
62
  }
64
63
  }
@@ -129,6 +128,14 @@ mod tests {
129
128
  assert!((distance("ab", "ac", 3) - 0.5).abs() < 1e-12);
130
129
  }
131
130
 
131
+ #[test]
132
+ fn input_newlines_are_not_padding() {
133
+ for (n, expected) in [(1, 0.5), (2, 0.25)] {
134
+ assert_eq!(distance("\na", "\nb", n), expected);
135
+ assert_eq!(distance("a\n", "b\n", n), distance("ax", "bx", n));
136
+ }
137
+ }
138
+
132
139
  #[test]
133
140
  fn ordering_matches_upstream_java_test() {
134
141
  let ngram = NGram::default();
@@ -28,6 +28,10 @@ impl Levenshtein {
28
28
  Self
29
29
  }
30
30
 
31
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
32
+ (py.get_type::<Levenshtein>(), ()).into_py_any(py)
33
+ }
34
+
31
35
  fn distance(&self, py: Python<'_>, s0: &str, s1: &str) -> PyResult<Py<PyAny>> {
32
36
  utils::zero_or_int(py, s0 == s1, core::levenshtein::distance(s0, s1))
33
37
  }
@@ -44,6 +48,10 @@ impl NormalizedLevenshtein {
44
48
  Self
45
49
  }
46
50
 
51
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
52
+ (py.get_type::<NormalizedLevenshtein>(), ()).into_py_any(py)
53
+ }
54
+
47
55
  fn __getattr__(&self, py: Python<'_>, name: &str) -> PyResult<Py<PyAny>> {
48
56
  if name == "levenshtein" {
49
57
  Ok(Py::new(py, Levenshtein)?.into_any())
@@ -72,6 +80,10 @@ impl Damerau {
72
80
  Self
73
81
  }
74
82
 
83
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
84
+ (py.get_type::<Damerau>(), ()).into_py_any(py)
85
+ }
86
+
75
87
  fn distance(&self, py: Python<'_>, s0: &str, s1: &str) -> PyResult<Py<PyAny>> {
76
88
  utils::zero_or_int(py, s0 == s1, core::damerau::distance(s0, s1))
77
89
  }
@@ -88,6 +100,10 @@ impl OptimalStringAlignment {
88
100
  Self
89
101
  }
90
102
 
103
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
104
+ (py.get_type::<OptimalStringAlignment>(), ()).into_py_any(py)
105
+ }
106
+
91
107
  fn distance(&self, py: Python<'_>, s0: &str, s1: &str) -> PyResult<Py<PyAny>> {
92
108
  utils::zero_or_int(
93
109
  py,
@@ -119,6 +135,22 @@ impl JaroWinkler {
119
135
  Self { threshold }
120
136
  }
121
137
 
138
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
139
+ (py.get_type::<JaroWinkler>(), (self.threshold,)).into_py_any(py)
140
+ }
141
+
142
+ /// Match details as `[matches, transpositions, prefix, max_len]`.
143
+ #[staticmethod]
144
+ fn matches(s0: &str, s1: &str) -> Vec<usize> {
145
+ let details = core::jaro_winkler::matches(s0, s1);
146
+ vec![
147
+ details.matches,
148
+ details.transpositions,
149
+ details.prefix,
150
+ details.max_len,
151
+ ]
152
+ }
153
+
122
154
  fn get_threshold(&self) -> f64 {
123
155
  self.threshold
124
156
  }
@@ -161,6 +193,10 @@ impl LongestCommonSubsequence {
161
193
  Self
162
194
  }
163
195
 
196
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
197
+ (py.get_type::<LongestCommonSubsequence>(), ()).into_py_any(py)
198
+ }
199
+
164
200
  fn distance(&self, py: Python<'_>, s0: &str, s1: &str) -> PyResult<Py<PyAny>> {
165
201
  utils::zero_or_int(
166
202
  py,
@@ -186,6 +222,10 @@ impl MetricLCS {
186
222
  Self
187
223
  }
188
224
 
225
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
226
+ (py.get_type::<MetricLCS>(), ()).into_py_any(py)
227
+ }
228
+
189
229
  fn __getattr__(&self, py: Python<'_>, name: &str) -> PyResult<Py<PyAny>> {
190
230
  if name == "lcs" {
191
231
  Ok(Py::new(py, LongestCommonSubsequence)?.into_any())
@@ -216,6 +256,10 @@ impl NGram {
216
256
  Ok(Self { n })
217
257
  }
218
258
 
259
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
260
+ (py.get_type::<NGram>(), (self.n,)).into_py_any(py)
261
+ }
262
+
219
263
  #[getter]
220
264
  fn n(&self) -> usize {
221
265
  self.n
@@ -249,6 +293,10 @@ impl QGram {
249
293
  Self { k }
250
294
  }
251
295
 
296
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
297
+ (py.get_type::<QGram>(), (self.k,)).into_py_any(py)
298
+ }
299
+
252
300
  fn get_k(&self) -> usize {
253
301
  self.k
254
302
  }
@@ -271,10 +319,13 @@ impl QGram {
271
319
 
272
320
  #[staticmethod]
273
321
  fn distance_profile(
274
- profile0: HashMap<String, usize>,
275
- profile1: HashMap<String, usize>,
276
- ) -> usize {
277
- core::qgram::distance_profile(&profile0, &profile1)
322
+ profile0: &Bound<'_, PyAny>,
323
+ profile1: &Bound<'_, PyAny>,
324
+ ) -> PyResult<usize> {
325
+ Ok(core::qgram::distance_profile(
326
+ &utils::extract_profile_usize(profile0)?,
327
+ &utils::extract_profile_usize(profile1)?,
328
+ ))
278
329
  }
279
330
  }
280
331
 
@@ -291,6 +342,10 @@ impl Cosine {
291
342
  Self { k }
292
343
  }
293
344
 
345
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
346
+ (py.get_type::<Cosine>(), (self.k,)).into_py_any(py)
347
+ }
348
+
294
349
  fn get_k(&self) -> usize {
295
350
  self.k
296
351
  }
@@ -317,10 +372,13 @@ impl Cosine {
317
372
 
318
373
  fn similarity_profiles(
319
374
  &self,
320
- profile0: HashMap<String, f64>,
321
- profile1: HashMap<String, f64>,
322
- ) -> f64 {
323
- core::cosine::similarity_profiles(&profile0, &profile1)
375
+ profile0: &Bound<'_, PyAny>,
376
+ profile1: &Bound<'_, PyAny>,
377
+ ) -> PyResult<f64> {
378
+ Ok(core::cosine::similarity_profiles(
379
+ &utils::extract_profile_f64(profile0)?,
380
+ &utils::extract_profile_f64(profile1)?,
381
+ ))
324
382
  }
325
383
  }
326
384
 
@@ -337,6 +395,10 @@ impl Jaccard {
337
395
  Self { k }
338
396
  }
339
397
 
398
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
399
+ (py.get_type::<Jaccard>(), (self.k,)).into_py_any(py)
400
+ }
401
+
340
402
  fn get_k(&self) -> usize {
341
403
  self.k
342
404
  }
@@ -376,6 +438,10 @@ impl SorensenDice {
376
438
  Self { k }
377
439
  }
378
440
 
441
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
442
+ (py.get_type::<SorensenDice>(), (self.k,)).into_py_any(py)
443
+ }
444
+
379
445
  fn get_k(&self) -> usize {
380
446
  self.k
381
447
  }
@@ -415,6 +481,10 @@ impl OverlapCoefficient {
415
481
  Self { k }
416
482
  }
417
483
 
484
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
485
+ (py.get_type::<OverlapCoefficient>(), (self.k,)).into_py_any(py)
486
+ }
487
+
418
488
  fn get_k(&self) -> usize {
419
489
  self.k
420
490
  }
@@ -173,6 +173,10 @@ impl SIFT4Options {
173
173
  Ok(resolved)
174
174
  }
175
175
 
176
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
177
+ (py.get_type::<SIFT4Options>(), (self.options(py)?,)).into_py_any(py)
178
+ }
179
+
176
180
  #[getter]
177
181
  fn maxdistance(&self) -> i64 {
178
182
  self.maxdistance
@@ -269,14 +273,18 @@ impl SIFT4Options {
269
273
  ))
270
274
  }
271
275
 
276
+ /// Splits `s` into overlapping character n-grams.
277
+ ///
278
+ /// `n` defaults to 2 so the `"ngram"` named tokenizer preset can be used
279
+ /// with `SIFT4`: the algorithm invokes tokenizers with a single argument.
272
280
  #[staticmethod]
281
+ #[pyo3(signature = (s, n=2))]
273
282
  fn ngramtokenizer(s: &str, n: usize) -> Vec<String> {
274
283
  let chars: Vec<char> = s.chars().collect();
275
- if chars.is_empty() {
284
+ if n == 0 || chars.len() < n {
276
285
  return Vec::new();
277
286
  }
278
- let count = chars.len().saturating_sub(n).saturating_sub(1);
279
- (0..count)
287
+ (0..=chars.len() - n)
280
288
  .map(|i| chars[i..i + n].iter().collect())
281
289
  .collect()
282
290
  }
@@ -455,6 +463,10 @@ impl SIFT4 {
455
463
  Self
456
464
  }
457
465
 
466
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
467
+ (py.get_type::<SIFT4>(), ()).into_py_any(py)
468
+ }
469
+
458
470
  #[pyo3(signature = (s1, s2, maxoffset=5, options=None))]
459
471
  fn distance(
460
472
  &self,
@@ -0,0 +1,59 @@
1
+ use std::collections::HashMap;
2
+
3
+ use pyo3::prelude::*;
4
+ use pyo3::IntoPyObjectExt;
5
+
6
+ /// Converts any Python mapping (a `dict`, `Counter`, `MappingProxyType`,
7
+ /// `UserDict`, ...) into a `HashMap`.
8
+ ///
9
+ /// The profile-based methods used to accept arbitrary mappings; extracting a
10
+ /// concrete `HashMap` directly would reject read-only and user-defined mapping
11
+ /// wrappers, so iterate the mapping protocol instead.
12
+ pub(crate) fn extract_profile_usize(
13
+ profile: &Bound<'_, PyAny>,
14
+ ) -> PyResult<HashMap<String, usize>> {
15
+ extract_profile(profile)
16
+ }
17
+
18
+ /// `f64` counterpart of [`extract_profile_usize`].
19
+ pub(crate) fn extract_profile_f64(profile: &Bound<'_, PyAny>) -> PyResult<HashMap<String, f64>> {
20
+ extract_profile(profile)
21
+ }
22
+
23
+ fn extract_profile<T>(profile: &Bound<'_, PyAny>) -> PyResult<HashMap<String, T>>
24
+ where
25
+ for<'a, 'py> T: FromPyObject<'a, 'py>,
26
+ {
27
+ let mut result = HashMap::new();
28
+ for item in profile.call_method0("items")?.try_iter()? {
29
+ let (key, value) = item?.extract::<(String, T)>()?;
30
+ result.insert(key, value);
31
+ }
32
+ Ok(result)
33
+ }
34
+
35
+ /// Converts an `f64` into a Python number, using `int` for integral values.
36
+ ///
37
+ /// The Python implementation accumulates several SIFT4 values as integers, and
38
+ /// some user-supplied evaluators may rely on receiving an `int`.
39
+ pub(crate) fn to_py_number(py: Python<'_>, value: f64) -> PyResult<Py<PyAny>> {
40
+ if value.is_finite()
41
+ && value.fract() == 0.0
42
+ && value >= i64::MIN as f64
43
+ && value <= i64::MAX as f64
44
+ {
45
+ (value as i64).into_py_any(py)
46
+ } else {
47
+ value.into_py_any(py)
48
+ }
49
+ }
50
+
51
+ /// Mirrors the original Python return types for integral edit distances:
52
+ /// identical strings return the float `0.0`, everything else returns an `int`.
53
+ pub(crate) fn zero_or_int(py: Python<'_>, strings_equal: bool, value: f64) -> PyResult<Py<PyAny>> {
54
+ if strings_equal {
55
+ 0.0f64.into_py_any(py)
56
+ } else {
57
+ (value as usize).into_py_any(py)
58
+ }
59
+ }
@@ -1,6 +1,7 @@
1
1
  //! Python wrapper for weighted Levenshtein distance.
2
2
 
3
3
  use pyo3::prelude::*;
4
+ use pyo3::IntoPyObjectExt;
4
5
  use strsimpy_core::weighted_levenshtein::{self as core, WeightedCosts};
5
6
 
6
7
  fn module_function(py: Python<'_>, name: &str) -> PyResult<Py<PyAny>> {
@@ -91,6 +92,26 @@ impl WeightedLevenshtein {
91
92
  }
92
93
  }
93
94
 
95
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
96
+ let substitution = match &self.substitution_cost_fn {
97
+ Some(function) => function.clone_ref(py),
98
+ None => py.None(),
99
+ };
100
+ let insertion = match &self.insertion_cost_fn {
101
+ Some(function) => function.clone_ref(py),
102
+ None => py.None(),
103
+ };
104
+ let deletion = match &self.deletion_cost_fn {
105
+ Some(function) => function.clone_ref(py),
106
+ None => py.None(),
107
+ };
108
+ (
109
+ py.get_type::<WeightedLevenshtein>(),
110
+ (substitution, insertion, deletion),
111
+ )
112
+ .into_py_any(py)
113
+ }
114
+
94
115
  fn distance(&self, py: Python<'_>, s0: &str, s1: &str) -> PyResult<f64> {
95
116
  let costs = PythonCosts {
96
117
  py,
@@ -38,7 +38,7 @@ from ._strsimpy import (
38
38
  try:
39
39
  __version__ = _package_version("strsimpy")
40
40
  except PackageNotFoundError: # pragma: no cover - only when not installed
41
- __version__ = "0.4.0"
41
+ __version__ = "0.4.2"
42
42
 
43
43
  __all__ = [
44
44
  "SIFT4",
@@ -166,7 +166,7 @@ class TestPublicApi(unittest.TestCase):
166
166
  self.assertEqual({"ab": 1, "bc": 1}, shingle_profile("abc", 2))
167
167
 
168
168
  def test_version(self):
169
- self.assertEqual("0.4.0", strsimpy.__version__)
169
+ self.assertEqual("0.4.2", strsimpy.__version__)
170
170
 
171
171
  def test_integral_distance_return_types(self):
172
172
  # The original Python implementation returned an int for non-identical
@@ -1,4 +1,6 @@
1
1
  import unittest
2
+ from collections import UserDict
3
+ from types import MappingProxyType
2
4
 
3
5
  from strsimpy.cosine import Cosine
4
6
 
@@ -62,6 +64,23 @@ class TestCosine(unittest.TestCase):
62
64
  def test_get_k(self):
63
65
  self.assertEqual(2, Cosine(2).get_k())
64
66
 
67
+ def test_similarity_profiles_accept_mapping_wrappers(self):
68
+ cos = Cosine(2)
69
+ left = MappingProxyType({"ab": 2, "bc": 1})
70
+ right = UserDict({"ab": 1, "cd": 1})
71
+ self.assertAlmostEqual(0.6324555320336759, cos.similarity_profiles(left, right))
72
+ self.assertAlmostEqual(
73
+ cos.similarity_profiles({"ab": 2, "bc": 1}, {"ab": 1, "cd": 1}),
74
+ cos.similarity_profiles(left, right),
75
+ )
76
+ self.assertAlmostEqual(
77
+ cos.similarity_profiles(
78
+ MappingProxyType(cos.get_profile("My first string")),
79
+ MappingProxyType(cos.get_profile("My other string...")),
80
+ ),
81
+ cos.similarity("My first string", "My other string..."),
82
+ )
83
+
65
84
 
66
85
  if __name__ == "__main__":
67
86
  unittest.main()
@@ -49,6 +49,13 @@ class TestJaroWinkler(unittest.TestCase):
49
49
  self.assertEqual(0.7, JaroWinkler().get_threshold())
50
50
  self.assertEqual(0.5, JaroWinkler(0.5).get_threshold())
51
51
 
52
+ def test_matches(self):
53
+ # [matches, transpositions, prefix length, longer input length]
54
+ self.assertEqual([6, 1, 3, 6], JaroWinkler.matches("martha", "marhta"))
55
+ self.assertEqual([6, 1, 3, 6], JaroWinkler().matches("martha", "marhta"))
56
+ self.assertEqual([0, 0, 0, 3], JaroWinkler.matches("", "abc"))
57
+ self.assertEqual([3, 0, 3, 3], JaroWinkler.matches("abc", "abc"))
58
+
52
59
 
53
60
  if __name__ == "__main__":
54
61
  unittest.main()
@@ -39,6 +39,34 @@ class TestNGram(unittest.TestCase):
39
39
  ngram.distance("ABABABAB", "POIULKJH"),
40
40
  )
41
41
 
42
+ def test_newlines_are_input_characters(self):
43
+ self.assertEqual(0.5, NGram(1).distance("\na", "\nb"))
44
+ self.assertEqual(0.25, NGram(2).distance("\na", "\nb"))
45
+
46
+ def test_newline_relabeling_preserves_distance(self):
47
+ pairs = [
48
+ ("\na", "\nb"),
49
+ ("a\nb", "a\nc"),
50
+ ("a\n", "b\n"),
51
+ ("\n\n", "\na"),
52
+ ("a\n\nb", "\na\nb"),
53
+ ("\na\0", "\nb\0"),
54
+ ("海\na", "海\nb"),
55
+ ("", "\n"),
56
+ ]
57
+ for n in range(1, 5):
58
+ ngram = NGram(n)
59
+ for a, b in pairs:
60
+ with self.subTest(n=n, a=a, b=b):
61
+ # Relabeling an input character cannot change edit costs.
62
+ expected = ngram.distance(
63
+ a.replace("\n", "x"), b.replace("\n", "x")
64
+ )
65
+ actual = ngram.distance(a, b)
66
+ self.assertAlmostEqual(expected, actual)
67
+ self.assertGreaterEqual(actual, 0.0)
68
+ self.assertLessEqual(actual, 1.0)
69
+
42
70
  def test_invalid_n(self):
43
71
  with self.assertRaises(ValueError):
44
72
  NGram(0)
@@ -0,0 +1,108 @@
1
+ import pickle
2
+ import unittest
3
+ from multiprocessing import get_context
4
+
5
+ from strsimpy import (
6
+ SIFT4,
7
+ SIFT4Options,
8
+ Cosine,
9
+ Damerau,
10
+ Jaccard,
11
+ JaroWinkler,
12
+ Levenshtein,
13
+ LongestCommonSubsequence,
14
+ MetricLCS,
15
+ NGram,
16
+ NormalizedLevenshtein,
17
+ OptimalStringAlignment,
18
+ OverlapCoefficient,
19
+ QGram,
20
+ SorensenDice,
21
+ WeightedLevenshtein,
22
+ )
23
+
24
+
25
+ def scaled_substitution_cost(char_a, char_b):
26
+ return 2.0 if char_a != char_b else 0.0
27
+
28
+
29
+ class TestPickle(unittest.TestCase):
30
+
31
+ def test_stateless_calculators_roundtrip(self):
32
+ calculators = [
33
+ Levenshtein(),
34
+ NormalizedLevenshtein(),
35
+ Damerau(),
36
+ OptimalStringAlignment(),
37
+ JaroWinkler(),
38
+ LongestCommonSubsequence(),
39
+ MetricLCS(),
40
+ NGram(),
41
+ QGram(),
42
+ Cosine(1),
43
+ Jaccard(1),
44
+ SorensenDice(1),
45
+ OverlapCoefficient(1),
46
+ SIFT4(),
47
+ ]
48
+ for calculator in calculators:
49
+ with self.subTest(calculator=type(calculator).__name__):
50
+ restored = pickle.loads(pickle.dumps(calculator))
51
+ self.assertIs(type(restored), type(calculator))
52
+ self.assertEqual(
53
+ calculator.distance("martha", "marhta"),
54
+ restored.distance("martha", "marhta"),
55
+ )
56
+
57
+ def test_constructor_options_are_retained(self):
58
+ jaro_winkler = pickle.loads(pickle.dumps(JaroWinkler(0.5)))
59
+ self.assertEqual(0.5, jaro_winkler.get_threshold())
60
+ self.assertEqual(0.5, jaro_winkler.threshold)
61
+
62
+ ngram = pickle.loads(pickle.dumps(NGram(3)))
63
+ self.assertEqual(3, ngram.n)
64
+
65
+ for calculator_class in (
66
+ QGram,
67
+ Cosine,
68
+ Jaccard,
69
+ SorensenDice,
70
+ OverlapCoefficient,
71
+ ):
72
+ with self.subTest(calculator=calculator_class.__name__):
73
+ restored = pickle.loads(pickle.dumps(calculator_class(4)))
74
+ self.assertEqual(4, restored.get_k())
75
+
76
+ def test_weighted_levenshtein_keeps_cost_functions(self):
77
+ calculator = WeightedLevenshtein(substitution_cost_fn=scaled_substitution_cost)
78
+ restored = pickle.loads(pickle.dumps(calculator))
79
+ self.assertEqual(2.0, restored.distance("abc", "adc"))
80
+ self.assertEqual(
81
+ calculator.distance("abc", "adc"), restored.distance("abc", "adc")
82
+ )
83
+
84
+ default = pickle.loads(pickle.dumps(WeightedLevenshtein()))
85
+ self.assertEqual(1.0, default.distance("abc", "adc"))
86
+
87
+ def test_sift4_options_roundtrip(self):
88
+ options = SIFT4Options({"maxdistance": 3, "tokenizer": "ngram"})
89
+ restored = pickle.loads(pickle.dumps(options))
90
+ self.assertEqual(3, restored.maxdistance)
91
+ self.assertEqual(
92
+ SIFT4Options.ngramtokenizer("abcd"), restored.tokenizer("abcd")
93
+ )
94
+
95
+ def test_bound_distance_method_can_be_pickled(self):
96
+ distance = pickle.loads(pickle.dumps(Levenshtein().distance))
97
+ self.assertEqual(1, distance("abc", "adc"))
98
+
99
+ def test_multiprocessing_pool_can_use_bound_distance(self):
100
+ context = get_context("spawn")
101
+ with context.Pool(1) as pool:
102
+ self.assertEqual(
103
+ [1], pool.starmap(Levenshtein().distance, [("abc", "adc")])
104
+ )
105
+
106
+
107
+ if __name__ == "__main__":
108
+ unittest.main()
@@ -1,4 +1,6 @@
1
1
  import unittest
2
+ from collections import Counter, UserDict
3
+ from types import MappingProxyType
2
4
 
3
5
  from strsimpy.qgram import QGram
4
6
 
@@ -39,6 +41,20 @@ class TestQGram(unittest.TestCase):
39
41
  def test_whitespace_normalization(self):
40
42
  self.assertEqual({"a ": 1, " b": 1}, QGram(2).get_profile("a b"))
41
43
 
44
+ def test_distance_profile_accepts_mapping_wrappers(self):
45
+ qgram = QGram(2)
46
+ left = MappingProxyType(qgram.get_profile("ABCD"))
47
+ right = UserDict(qgram.get_profile("ABCE"))
48
+ self.assertEqual(2, QGram.distance_profile(left, right))
49
+ self.assertEqual(2, qgram.distance_profile(left, right))
50
+ self.assertEqual(
51
+ 2,
52
+ QGram.distance_profile(
53
+ MappingProxyType({"AB": 1, "BC": 1, "CD": 1}),
54
+ Counter({"AB": 1, "BC": 1, "CE": 1}),
55
+ ),
56
+ )
57
+
42
58
 
43
59
  if __name__ == "__main__":
44
60
  unittest.main()
@@ -1,4 +1,5 @@
1
1
  import unittest
2
+ from functools import partial
2
3
 
3
4
  from strsimpy.sift4 import SIFT4, SIFT4Options
4
5
 
@@ -93,8 +94,36 @@ class SIFT4Test(unittest.TestCase):
93
94
  options.tokenizer = list
94
95
  self.assertIs(options.tokenizer, list)
95
96
 
97
+ def test_ngram_tokenizer_windows(self):
98
+ self.assertEqual(["ab", "bc", "cd"], SIFT4Options.ngramtokenizer("abcd", 2))
99
+ self.assertEqual(["ab"], SIFT4Options.ngramtokenizer("ab", 2))
100
+ self.assertEqual(
101
+ ["he", "el", "ll", "lo"], SIFT4Options.ngramtokenizer("hello", 2)
102
+ )
103
+ self.assertEqual(["abcd"], SIFT4Options.ngramtokenizer("abcd", 4))
104
+ self.assertEqual([], SIFT4Options.ngramtokenizer("abc", 5))
105
+ self.assertEqual([], SIFT4Options.ngramtokenizer("abc", 0))
106
+ self.assertEqual([], SIFT4Options.ngramtokenizer("", 2))
107
+ # The default n matches NGram's default of 2.
108
+ self.assertEqual(["ab", "bc", "cd"], SIFT4Options.ngramtokenizer("abcd"))
109
+
110
+ def test_named_ngram_tokenizer(self):
111
+ strings = ("abcd", "abXY")
112
+ options = {"tokenizer": "ngram"}
113
+ expected = SIFT4().distance(
114
+ *strings,
115
+ options={"tokenizer": partial(SIFT4Options.ngramtokenizer, n=2)},
116
+ )
117
+ self.assertEqual(2, expected)
118
+ self.assertEqual(expected, SIFT4().distance(*strings, options=options))
119
+ self.assertIsInstance(
120
+ SIFT4().distance("alpha beta", "alpha gamma", options=options), int
121
+ )
122
+
96
123
  def test_static_helpers(self):
97
- self.assertEqual(["he", "el"], SIFT4Options.ngramtokenizer("hello", 2))
124
+ self.assertEqual(
125
+ ["he", "el", "ll", "lo"], SIFT4Options.ngramtokenizer("hello", 2)
126
+ )
98
127
  self.assertEqual(
99
128
  ["hello", "world"],
100
129
  SIFT4Options.wordsplittokenizer(" hello\tworld\n"),
@@ -1,28 +0,0 @@
1
- use pyo3::prelude::*;
2
- use pyo3::IntoPyObjectExt;
3
-
4
- /// Converts an `f64` into a Python number, using `int` for integral values.
5
- ///
6
- /// The Python implementation accumulates several SIFT4 values as integers, and
7
- /// some user-supplied evaluators may rely on receiving an `int`.
8
- pub(crate) fn to_py_number(py: Python<'_>, value: f64) -> PyResult<Py<PyAny>> {
9
- if value.is_finite()
10
- && value.fract() == 0.0
11
- && value >= i64::MIN as f64
12
- && value <= i64::MAX as f64
13
- {
14
- (value as i64).into_py_any(py)
15
- } else {
16
- value.into_py_any(py)
17
- }
18
- }
19
-
20
- /// Mirrors the original Python return types for integral edit distances:
21
- /// identical strings return the float `0.0`, everything else returns an `int`.
22
- pub(crate) fn zero_or_int(py: Python<'_>, strings_equal: bool, value: f64) -> PyResult<Py<PyAny>> {
23
- if strings_equal {
24
- 0.0f64.into_py_any(py)
25
- } else {
26
- (value as usize).into_py_any(py)
27
- }
28
- }
File without changes
File without changes
File without changes
File without changes