strsimpy 0.4.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {strsimpy-0.4.0 → strsimpy-0.4.1}/Cargo.lock +2 -2
- {strsimpy-0.4.0 → strsimpy-0.4.1}/Cargo.toml +1 -1
- {strsimpy-0.4.0 → strsimpy-0.4.1}/PKG-INFO +7 -1
- {strsimpy-0.4.0 → strsimpy-0.4.1}/README.md +6 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/ngram.rs +18 -11
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/__init__.py +1 -1
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_api.py +1 -1
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_ngram.py +28 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/LICENSE +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/Cargo.toml +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/cosine.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/damerau.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/jaccard.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/jaro_winkler.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/levenshtein.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/lib.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/longest_common_subsequence.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/metric_lcs.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/normalized_levenshtein.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/optimal_string_alignment.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/overlap_coefficient.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/qgram.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/shingle.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/sift4.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/sorensen_dice.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-core/src/weighted_levenshtein.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-py/Cargo.toml +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-py/src/algorithms.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-py/src/lib.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-py/src/sift4.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-py/src/utils.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/crates/strsimpy-py/src/weighted.rs +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/pyproject.toml +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/_base.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/cosine.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/damerau.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/jaccard.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/jaro_winkler.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/levenshtein.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/longest_common_subsequence.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/metric_lcs.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/ngram.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/normalized_levenshtein.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/optimal_string_alignment.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/overlap_coefficient.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/qgram.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/shingle_based.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/sift4.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/sorensen_dice.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/string_distance.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/string_similarity.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/python/strsimpy/weighted_levenshtein.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_cosine.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_damerau.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_jaccard.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_jaro_winkler.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_levenshtein.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_longest_common_subsequence.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_metric_lcs.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_normalized_levenshtein.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_optimal_string_alignment.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_overlap_coefficient.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_qgram.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_sift4.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_sorensen_dice.py +0 -0
- {strsimpy-0.4.0 → strsimpy-0.4.1}/tests/test_weighted_levenshtein.py +0 -0
|
@@ -103,11 +103,11 @@ dependencies = [
|
|
|
103
103
|
|
|
104
104
|
[[package]]
|
|
105
105
|
name = "strsimpy-core"
|
|
106
|
-
version = "0.4.
|
|
106
|
+
version = "0.4.1"
|
|
107
107
|
|
|
108
108
|
[[package]]
|
|
109
109
|
name = "strsimpy-py"
|
|
110
|
-
version = "0.4.
|
|
110
|
+
version = "0.4.1"
|
|
111
111
|
dependencies = [
|
|
112
112
|
"pyo3",
|
|
113
113
|
"strsimpy-core",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: strsimpy
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -513,6 +513,12 @@ them to PyPI together with the source distribution.
|
|
|
513
513
|
|
|
514
514
|
## Changelog
|
|
515
515
|
|
|
516
|
+
### 0.4.1
|
|
517
|
+
- Fixed `NGram` treating literal newline characters in the input as its internal
|
|
518
|
+
padding marker, which produced incorrect edit costs and normalized distances
|
|
519
|
+
above 1 (for example, `NGram(1).distance("\na", "\nb")` returned 1.5 instead
|
|
520
|
+
of 0.5) (#41). Thanks to @Narek-Khachikyan.
|
|
521
|
+
|
|
516
522
|
### 0.4.0
|
|
517
523
|
- Reimplemented all algorithms in Rust and exposed them through PyO3. The
|
|
518
524
|
Python API, package root exports and per-module imports are unchanged.
|
|
@@ -486,6 +486,12 @@ them to PyPI together with the source distribution.
|
|
|
486
486
|
|
|
487
487
|
## Changelog
|
|
488
488
|
|
|
489
|
+
### 0.4.1
|
|
490
|
+
- Fixed `NGram` treating literal newline characters in the input as its internal
|
|
491
|
+
padding marker, which produced incorrect edit costs and normalized distances
|
|
492
|
+
above 1 (for example, `NGram(1).distance("\na", "\nb")` returned 1.5 instead
|
|
493
|
+
of 0.5) (#41). Thanks to @Narek-Khachikyan.
|
|
494
|
+
|
|
489
495
|
### 0.4.0
|
|
490
496
|
- Reimplemented all algorithms in Rust and exposed them through PyO3. The
|
|
491
497
|
Python API, package root exports and per-module imports are unchanged.
|
|
@@ -4,13 +4,14 @@
|
|
|
4
4
|
///
|
|
5
5
|
/// The implementation follows Kondrak's algorithm as used by
|
|
6
6
|
/// `java-string-similarity`: the first string is prefixed with `n - 1`
|
|
7
|
-
///
|
|
8
|
-
/// is normalized by the length of the longest input string.
|
|
7
|
+
/// padding markers distinct from input characters. The edit distance between
|
|
8
|
+
/// n-gram profiles is normalized by the length of the longest input string.
|
|
9
9
|
pub fn distance(s0: &str, s1: &str, n: usize) -> f64 {
|
|
10
10
|
assert!(n > 0, "n must be greater than zero");
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
let
|
|
12
|
+
// None is reserved for padding; every input character is Some(c).
|
|
13
|
+
let a: Vec<Option<char>> = s0.chars().map(Some).collect();
|
|
14
|
+
let b: Vec<Option<char>> = s1.chars().map(Some).collect();
|
|
14
15
|
|
|
15
16
|
if a == b {
|
|
16
17
|
return 0.0;
|
|
@@ -32,20 +33,18 @@ pub fn distance(s0: &str, s1: &str, n: usize) -> f64 {
|
|
|
32
33
|
return 1.0 - matched as f64 / sl.max(tl) as f64;
|
|
33
34
|
}
|
|
34
35
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
let mut sa = vec![SPECIAL; sl + n - 1];
|
|
36
|
+
let mut sa = vec![None; sl + n - 1];
|
|
38
37
|
sa[n - 1..].copy_from_slice(&a);
|
|
39
38
|
|
|
40
39
|
let mut previous: Vec<f64> = (0..=sl).map(|i| i as f64).collect();
|
|
41
40
|
let mut current = vec![0.0f64; sl + 1];
|
|
42
|
-
let mut t_j = vec![
|
|
41
|
+
let mut t_j = vec![None; n];
|
|
43
42
|
|
|
44
43
|
for j in 1..=tl {
|
|
45
44
|
if j < n {
|
|
46
45
|
let padding = n - j;
|
|
47
|
-
t_j[..padding].fill(
|
|
48
|
-
t_j[padding..n].copy_from_slice(&b[..
|
|
46
|
+
t_j[..padding].fill(None);
|
|
47
|
+
t_j[padding..n].copy_from_slice(&b[..j]);
|
|
49
48
|
} else {
|
|
50
49
|
t_j.copy_from_slice(&b[j - n..j]);
|
|
51
50
|
}
|
|
@@ -58,7 +57,7 @@ pub fn distance(s0: &str, s1: &str, n: usize) -> f64 {
|
|
|
58
57
|
let sc = sa[i - 1 + ni];
|
|
59
58
|
if sc != t_j[ni] {
|
|
60
59
|
cost += 1;
|
|
61
|
-
} else if sc
|
|
60
|
+
} else if sc.is_none() {
|
|
62
61
|
denominator -= 1;
|
|
63
62
|
}
|
|
64
63
|
}
|
|
@@ -129,6 +128,14 @@ mod tests {
|
|
|
129
128
|
assert!((distance("ab", "ac", 3) - 0.5).abs() < 1e-12);
|
|
130
129
|
}
|
|
131
130
|
|
|
131
|
+
#[test]
|
|
132
|
+
fn input_newlines_are_not_padding() {
|
|
133
|
+
for (n, expected) in [(1, 0.5), (2, 0.25)] {
|
|
134
|
+
assert_eq!(distance("\na", "\nb", n), expected);
|
|
135
|
+
assert_eq!(distance("a\n", "b\n", n), distance("ax", "bx", n));
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
132
139
|
#[test]
|
|
133
140
|
fn ordering_matches_upstream_java_test() {
|
|
134
141
|
let ngram = NGram::default();
|
|
@@ -166,7 +166,7 @@ class TestPublicApi(unittest.TestCase):
|
|
|
166
166
|
self.assertEqual({"ab": 1, "bc": 1}, shingle_profile("abc", 2))
|
|
167
167
|
|
|
168
168
|
def test_version(self):
|
|
169
|
-
self.assertEqual("0.4.
|
|
169
|
+
self.assertEqual("0.4.1", strsimpy.__version__)
|
|
170
170
|
|
|
171
171
|
def test_integral_distance_return_types(self):
|
|
172
172
|
# The original Python implementation returned an int for non-identical
|
|
@@ -39,6 +39,34 @@ class TestNGram(unittest.TestCase):
|
|
|
39
39
|
ngram.distance("ABABABAB", "POIULKJH"),
|
|
40
40
|
)
|
|
41
41
|
|
|
42
|
+
def test_newlines_are_input_characters(self):
|
|
43
|
+
self.assertEqual(0.5, NGram(1).distance("\na", "\nb"))
|
|
44
|
+
self.assertEqual(0.25, NGram(2).distance("\na", "\nb"))
|
|
45
|
+
|
|
46
|
+
def test_newline_relabeling_preserves_distance(self):
|
|
47
|
+
pairs = [
|
|
48
|
+
("\na", "\nb"),
|
|
49
|
+
("a\nb", "a\nc"),
|
|
50
|
+
("a\n", "b\n"),
|
|
51
|
+
("\n\n", "\na"),
|
|
52
|
+
("a\n\nb", "\na\nb"),
|
|
53
|
+
("\na\0", "\nb\0"),
|
|
54
|
+
("海\na", "海\nb"),
|
|
55
|
+
("", "\n"),
|
|
56
|
+
]
|
|
57
|
+
for n in range(1, 5):
|
|
58
|
+
ngram = NGram(n)
|
|
59
|
+
for a, b in pairs:
|
|
60
|
+
with self.subTest(n=n, a=a, b=b):
|
|
61
|
+
# Relabeling an input character cannot change edit costs.
|
|
62
|
+
expected = ngram.distance(
|
|
63
|
+
a.replace("\n", "x"), b.replace("\n", "x")
|
|
64
|
+
)
|
|
65
|
+
actual = ngram.distance(a, b)
|
|
66
|
+
self.assertAlmostEqual(expected, actual)
|
|
67
|
+
self.assertGreaterEqual(actual, 0.0)
|
|
68
|
+
self.assertLessEqual(actual, 1.0)
|
|
69
|
+
|
|
42
70
|
def test_invalid_n(self):
|
|
43
71
|
with self.assertRaises(ValueError):
|
|
44
72
|
NGram(0)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|