searcharray 0.0.57__tar.gz → 0.0.59__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.57 → searcharray-0.0.59}/PKG-INFO +7 -6
- {searcharray-0.0.57 → searcharray-0.0.59}/README.md +6 -5
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/indexing.py +1 -1
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/phrase/bigram_freqs.py +9 -10
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/phrase/middle_out.py +8 -8
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/phrase/spans.py +71 -21
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/postings.py +14 -1
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/__init__.py +1 -1
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/intersect.c +1724 -1115
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/intersect.pyx +1 -1
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/merge.c +949 -347
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/popcount.c +2285 -987
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/popcount.pyi +18 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/popcount.pyx +43 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/roaringish.py +10 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/roaringish_ops.c +865 -257
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/search.c +970 -373
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/spans.c +8825 -5546
- searcharray-0.0.59/searcharray/roaringish/spans.pyx +174 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/unique.c +947 -342
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/solr.py +12 -6
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/PKG-INFO +7 -6
- {searcharray-0.0.57 → searcharray-0.0.59}/setup.py +1 -1
- {searcharray-0.0.57 → searcharray-0.0.59}/test/test_msmarco.py +33 -1
- {searcharray-0.0.57 → searcharray-0.0.59}/test/test_phrase_matches.py +49 -1
- searcharray-0.0.59/test/test_slop_matches.py +88 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/test/test_tmdb.py +27 -5
- {searcharray-0.0.57 → searcharray-0.0.59}/test/test_utils.py +1 -0
- searcharray-0.0.57/searcharray/roaringish/spans.pyx +0 -77
- searcharray-0.0.57/test/test_slop_matches.py +0 -52
- {searcharray-0.0.57 → searcharray-0.0.59}/pyproject.toml +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/__init__.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/intersect.pyi +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/merge.pyi +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/merge.pyx +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/roaringish_ops.pyx +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/search.pyi +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/search.pyx +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/unique.pyi +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/unique.pyx +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/similarity.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/utils/sort.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/setup.cfg +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/test/test_extension_array.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/test/test_search.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/test/test_similarity.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/test/test_snp_ops.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.59}/test/test_solr.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: searcharray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.59
|
|
4
4
|
Summary: Searchable pandas text extension arrays for prototyping search
|
|
5
5
|
Home-page: https://github.com/softwaredoug/searcharray
|
|
6
6
|
Author: Doug Turnbull
|
|
@@ -36,13 +36,14 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
|
|
|
36
36
|
Think Lucene, but as a Pandas column.
|
|
37
37
|
|
|
38
38
|
```python
|
|
39
|
-
|
|
39
|
+
from searcharray import SearchArray
|
|
40
|
+
import pandas as pd
|
|
40
41
|
|
|
41
|
-
|
|
42
|
-
|
|
42
|
+
df['title_indexed'] = SearchArray.index(df['title'])
|
|
43
|
+
np.sort(df['title_indexed'].array.score('Cat'))
|
|
43
44
|
|
|
44
|
-
|
|
45
|
-
|
|
45
|
+
> BM25 scores:
|
|
46
|
+
> array([ 0. , 0. , 0. , ..., 15.84568033, 15.84568033, 15.84568033])
|
|
46
47
|
```
|
|
47
48
|
|
|
48
49
|
## Docs | Guide
|
|
@@ -7,13 +7,14 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
|
|
|
7
7
|
Think Lucene, but as a Pandas column.
|
|
8
8
|
|
|
9
9
|
```python
|
|
10
|
-
|
|
10
|
+
from searcharray import SearchArray
|
|
11
|
+
import pandas as pd
|
|
11
12
|
|
|
12
|
-
|
|
13
|
-
|
|
13
|
+
df['title_indexed'] = SearchArray.index(df['title'])
|
|
14
|
+
np.sort(df['title_indexed'].array.score('Cat'))
|
|
14
15
|
|
|
15
|
-
|
|
16
|
-
|
|
16
|
+
> BM25 scores:
|
|
17
|
+
> array([ 0. , 0. , 0. , ..., 15.84568033, 15.84568033, 15.84568033])
|
|
17
18
|
```
|
|
18
19
|
|
|
19
20
|
## Docs | Guide
|
|
@@ -121,7 +121,7 @@ def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate
|
|
|
121
121
|
|
|
122
122
|
# Encode posns to bit array
|
|
123
123
|
logger.info("Encoding positions to bit array")
|
|
124
|
-
posns = PosnBitArrayFromFlatBuilder(terms_w_posns)
|
|
124
|
+
posns = PosnBitArrayFromFlatBuilder(terms_w_posns, max_doc_id=(batch_beg + len(array) - 1))
|
|
125
125
|
bit_posns = posns.build()
|
|
126
126
|
|
|
127
127
|
if np.any(doc_lens > MAX_POSN):
|
|
@@ -5,7 +5,7 @@ from searcharray.roaringish import RoaringishEncoder
|
|
|
5
5
|
import logging
|
|
6
6
|
from enum import Enum
|
|
7
7
|
|
|
8
|
-
from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at
|
|
8
|
+
from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at, key_sum_over
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
logger = logging.getLogger(__name__)
|
|
@@ -82,7 +82,8 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
|
|
|
82
82
|
adj_count = encoder.payload_lsb(overlap)
|
|
83
83
|
adjacents = popcount64(adj_count).view(np.int64)
|
|
84
84
|
|
|
85
|
-
|
|
85
|
+
adjusted = _adj_to_phrase_freq(overlap, adjacents).astype(np.uint64)
|
|
86
|
+
key_sum_over(lhs_doc_ids, adjusted, phrase_freqs)
|
|
86
87
|
# Continue with ?? ?? foo foo
|
|
87
88
|
# term_int without lsbs
|
|
88
89
|
term_int_msbs = lhs_int & ~encoder.payload_lsb_mask
|
|
@@ -90,12 +91,13 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
|
|
|
90
91
|
lhs_cont = None
|
|
91
92
|
# rhs_cont = term_int_msbs | encoder.payload_lsb(rhs_shift)
|
|
92
93
|
rhs_cont = encoder.payload_lsb(rhs_shift & rhs_int) | term_int_msbs
|
|
93
|
-
|
|
94
94
|
lhs_cont = term_int_msbs | encoder.payload_lsb(lhs_int & (lhs_int >> _1))
|
|
95
|
+
|
|
95
96
|
if cont not in [Continuation.RHS, Continuation.BOTH]:
|
|
96
97
|
rhs_cont = None
|
|
97
98
|
if cont not in [Continuation.LHS, Continuation.BOTH]:
|
|
98
99
|
lhs_cont = None
|
|
100
|
+
|
|
99
101
|
return phrase_freqs, (lhs_cont, rhs_cont)
|
|
100
102
|
|
|
101
103
|
|
|
@@ -177,14 +179,11 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
177
179
|
if cont in [Continuation.RHS, Continuation.BOTH]:
|
|
178
180
|
rhs_next = rhs_adj[matches]
|
|
179
181
|
assert rhs_next is not None
|
|
180
|
-
rhs_next = encoder.header(
|
|
181
|
-
|
|
182
|
-
rhs_next = rhs_adj[matches]
|
|
183
|
-
assert rhs_next is not None
|
|
184
|
-
rhs_next |= _1
|
|
185
|
-
rhs_next = encoder.header(rhs_adj) | _1
|
|
182
|
+
rhs_next = encoder.header(rhs_next) | _1
|
|
183
|
+
if cont in [Continuation.LHS, Continuation.BOTH]:
|
|
186
184
|
lhs_next = lhs_adj[matches]
|
|
187
|
-
lhs_next
|
|
185
|
+
assert lhs_next is not None
|
|
186
|
+
lhs_next = encoder.header(lhs_next) | _upper_bit
|
|
188
187
|
rhs_next = None
|
|
189
188
|
return phrase_freqs, (lhs_next, rhs_next)
|
|
190
189
|
|
|
@@ -89,12 +89,10 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
|
|
|
89
89
|
phrase_freqs[mask] = 0
|
|
90
90
|
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
|
|
91
91
|
cont=Continuation.RHS)
|
|
92
|
+
|
|
92
93
|
assert conts[1] is not None
|
|
93
94
|
lhs = conts[1]
|
|
94
95
|
mask &= (phrase_freqs > 0)
|
|
95
|
-
# print("-- pf", phrase_freqs)
|
|
96
|
-
# dec = encoder.decode(lhs)
|
|
97
|
-
# print("-- co", dec)
|
|
98
96
|
phrase_freqs[~mask] = 0
|
|
99
97
|
return phrase_freqs
|
|
100
98
|
|
|
@@ -118,12 +116,10 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
|
|
|
118
116
|
phrase_freqs[mask] = 0
|
|
119
117
|
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
|
|
120
118
|
cont=Continuation.LHS)
|
|
119
|
+
|
|
121
120
|
assert conts[0] is not None
|
|
122
121
|
rhs = conts[0]
|
|
123
122
|
mask &= (phrase_freqs > 0)
|
|
124
|
-
# print("-- pf", phrase_freqs)
|
|
125
|
-
# dec = encoder.decode(lhs)
|
|
126
|
-
# print("-- co", dec)
|
|
127
123
|
phrase_freqs[~mask] = 0
|
|
128
124
|
return phrase_freqs
|
|
129
125
|
|
|
@@ -158,8 +154,9 @@ class PosnBitArrayFromFlatBuilder:
|
|
|
158
154
|
|
|
159
155
|
"""
|
|
160
156
|
|
|
161
|
-
def __init__(self, flat_array: np.ndarray):
|
|
157
|
+
def __init__(self, flat_array: np.ndarray, max_doc_id: int) -> None:
|
|
162
158
|
self.flat_array = flat_array
|
|
159
|
+
self.max_doc_id = max_doc_id
|
|
163
160
|
|
|
164
161
|
def build(self):
|
|
165
162
|
"""Slice the flat array into a 2d array of doc ids and posns."""
|
|
@@ -179,7 +176,9 @@ class PosnBitArrayFromFlatBuilder:
|
|
|
179
176
|
sliced = encoded[beg_idx:end_idx]
|
|
180
177
|
encoded_term_posns[term_ids[into_terms]] = sliced
|
|
181
178
|
|
|
182
|
-
|
|
179
|
+
largest_doc_id_with_term = np.max(self.flat_array[1])
|
|
180
|
+
assert self.max_doc_id >= largest_doc_id_with_term
|
|
181
|
+
return PosnBitArray(encoded_term_posns, self.max_doc_id)
|
|
183
182
|
|
|
184
183
|
|
|
185
184
|
class PosnBitArrayBuilder:
|
|
@@ -390,6 +389,7 @@ class PosnBitArray:
|
|
|
390
389
|
min_posn: Optional[int] = None,
|
|
391
390
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
392
391
|
phrase_freqs = self.empty_buffer()
|
|
392
|
+
|
|
393
393
|
if len(term_ids) < 2:
|
|
394
394
|
raise ValueError("Must have at least two terms")
|
|
395
395
|
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
|
|
@@ -25,7 +25,7 @@ _upper_bit = _1 << (encoder.payload_lsb_bits - _1)
|
|
|
25
25
|
# MSB
|
|
26
26
|
# term A |10| | 12| 13| | 18 * Intersect A+B -> 10,12
|
|
27
27
|
# term B |10| | 12| | 14 15 17 | * RHS intersect -> 13 -> 14
|
|
28
|
-
# term C | 11
|
|
28
|
+
# term C | 11 | 14
|
|
29
29
|
# term D 10 | 15
|
|
30
30
|
# term E | 11 12
|
|
31
31
|
# ------
|
|
@@ -37,39 +37,87 @@ _upper_bit = _1 << (encoder.payload_lsb_bits - _1)
|
|
|
37
37
|
#
|
|
38
38
|
# ----
|
|
39
39
|
#
|
|
40
|
+
# Intersecting
|
|
41
|
+
# term A |10| | 12| 13 18
|
|
42
|
+
# term B |10| | 12| 14 15 17
|
|
43
|
+
# Adjacent
|
|
44
|
+
# term A 10 12| 13| 18
|
|
45
|
+
# term B 10 12| | 14 15 17 |
|
|
46
|
+
# Giving 9.5, 10.5 11.5, 12.5, 13.5 , 17.5
|
|
47
|
+
# ------
|
|
48
|
+
# Next pair, intersecting
|
|
49
|
+
# term B 10 12 | 14 | 15 17 * RHS intersect -> 13 -> 14
|
|
50
|
+
# term C 11 | 14 |
|
|
51
|
+
# Adjacent
|
|
52
|
+
# term B 10 12 14 15 17
|
|
53
|
+
# term C |11 14
|
|
54
|
+
# Giving 10.5, 13.5, 14.5
|
|
55
|
+
# w/ above 10.5, 13.5
|
|
56
|
+
# ------
|
|
57
|
+
#
|
|
58
|
+
#
|
|
40
59
|
# Finally: X X X X <- 4 words selected from each to find max span
|
|
41
60
|
#
|
|
61
|
+
# Does it work to compute any intersection, and then attach any adjecent that exist
|
|
62
|
+
#
|
|
42
63
|
#
|
|
64
|
+
|
|
65
|
+
def hdr(arr):
|
|
66
|
+
return encoder.header(arr) >> (_64 - encoder.header_bits)
|
|
67
|
+
|
|
68
|
+
|
|
43
69
|
def _intersect_all(posns_encoded: List[np.ndarray]):
|
|
44
70
|
"""Intersect all encoded positions at roaringish MSBs."""
|
|
45
71
|
if len(posns_encoded) < 2:
|
|
46
72
|
raise ValueError("Need at least two positions to intersect")
|
|
73
|
+
last_lhs_headers = None
|
|
74
|
+
last_rhs_headers = None
|
|
75
|
+
curr = posns_encoded[0]
|
|
76
|
+
for term_idx, posns_next in enumerate(posns_encoded[1:]):
|
|
77
|
+
lhs_int_idx, rhs_int = intersect(curr, posns_next, mask=encoder.header_mask)
|
|
78
|
+
int_headers = encoder.header(curr[lhs_int_idx])
|
|
47
79
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
80
|
+
# What is adjacent on LHS / RHS of interserction
|
|
81
|
+
# Next to left
|
|
82
|
+
# 0 -> 2
|
|
83
|
+
# <- 1 (curr to right)
|
|
84
|
+
# 1 -> (keep
|
|
85
|
+
# 0 <- 2
|
|
86
|
+
curr_to_right, next_to_left = adjacent(curr, posns_next, mask=encoder.header_mask)
|
|
87
|
+
lhs_headers = merge(int_headers, posns_next[next_to_left])
|
|
88
|
+
rhs_headers = merge(int_headers, curr[curr_to_right])
|
|
89
|
+
next_to_right, curr_to_left = adjacent(posns_next, curr, mask=encoder.header_mask)
|
|
90
|
+
lhs_headers = merge(lhs_headers, curr[curr_to_left])
|
|
91
|
+
rhs_headers = merge(rhs_headers, posns_next[next_to_right])
|
|
51
92
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
93
|
+
if last_lhs_headers is not None:
|
|
94
|
+
lhs, _ = intersect(last_lhs_headers, lhs_headers, mask=encoder.header_mask)
|
|
95
|
+
rhs, _ = intersect(last_rhs_headers, rhs_headers, mask=encoder.header_mask)
|
|
96
|
+
last_lhs_headers = last_lhs_headers[lhs]
|
|
97
|
+
last_rhs_headers = last_rhs_headers[rhs]
|
|
98
|
+
else:
|
|
99
|
+
last_lhs_headers = lhs_headers
|
|
100
|
+
last_rhs_headers = rhs_headers
|
|
56
101
|
|
|
57
|
-
# Update
|
|
58
|
-
lhs_int = posns_next[rhs_int]
|
|
59
|
-
lhs_to_left = posns_next[rhs_ls]
|
|
60
|
-
lhs_to_right = posns_next[rhs_rs]
|
|
102
|
+
# Update by intersecting with current working lhs / rhs headers
|
|
61
103
|
|
|
104
|
+
assert last_rhs_headers is not None
|
|
105
|
+
assert last_lhs_headers is not None
|
|
106
|
+
to_rhs = last_rhs_headers + (_1 << (_64 - encoder.header_bits))
|
|
107
|
+
to_lhs = last_lhs_headers - (_1 << (_64 - encoder.header_bits))
|
|
108
|
+
all_headers = merge(to_rhs, to_lhs, drop_duplicates=True)
|
|
109
|
+
all_headers = merge(last_lhs_headers, all_headers, drop_duplicates=True)
|
|
110
|
+
all_headers = merge(last_rhs_headers, all_headers, drop_duplicates=True)
|
|
62
111
|
# Get active MSBs now
|
|
63
112
|
# Merge all the rest to grab them
|
|
64
|
-
int_header = encoder.header(lhs_int)
|
|
65
|
-
to_left_header = encoder.header(lhs_to_left)
|
|
66
|
-
to_right_header = encoder.header(lhs_to_right)
|
|
67
|
-
merged = merge(int_header, to_left_header, drop_duplicates=True)
|
|
68
|
-
merged = merge(merged, to_right_header, drop_duplicates=True)
|
|
69
113
|
|
|
70
|
-
# Slice only matches at MSBs
|
|
114
|
+
# Slice only matches at header MSBs
|
|
115
|
+
new_posns_encoded = posns_encoded.copy()
|
|
71
116
|
for i in range(len(posns_encoded)):
|
|
72
|
-
|
|
117
|
+
new_posns_encoded[i] = encoder.slice(posns_encoded[i], header=all_headers)
|
|
118
|
+
lengths = np.cumsum([0] + [len(posns) for posns in new_posns_encoded], dtype=np.uint64)
|
|
119
|
+
concatted = np.concatenate(new_posns_encoded, dtype=np.uint64)
|
|
120
|
+
return concatted, lengths
|
|
73
121
|
|
|
74
122
|
|
|
75
123
|
# Picking up from intersections:
|
|
@@ -122,11 +170,13 @@ def span_search(posns_encoded: List[np.ndarray],
|
|
|
122
170
|
slop: int) -> np.ndarray:
|
|
123
171
|
"""Find span matches up to PAYLOAD_LSB bits span distance."""
|
|
124
172
|
# Find inner span candidates
|
|
125
|
-
_intersect_all(posns_encoded)
|
|
173
|
+
posns, lengths = _intersect_all(posns_encoded)
|
|
126
174
|
|
|
127
175
|
# Populate phrase freqs with matches of slop
|
|
128
|
-
r_span_search(
|
|
176
|
+
r_span_search(posns, lengths,
|
|
177
|
+
phrase_freqs, slop,
|
|
129
178
|
encoder.key_mask,
|
|
130
179
|
encoder.header_mask,
|
|
180
|
+
encoder.key_bits,
|
|
131
181
|
encoder.payload_lsb_bits)
|
|
132
182
|
return phrase_freqs
|
|
@@ -469,6 +469,15 @@ class SearchArray(ExtensionArray):
|
|
|
469
469
|
logger.warning("Unique called on SearchArray. This is not supported.")
|
|
470
470
|
return self[:]
|
|
471
471
|
|
|
472
|
+
def __iter__(self):
|
|
473
|
+
if len(self) > 10000:
|
|
474
|
+
warning_text = """Iterating over SearchArray is very slow and not reccomended.
|
|
475
|
+
|
|
476
|
+
If you're looping a dataframe, slice out the non-SearchArray columns first."""
|
|
477
|
+
logger.warning(warning_text)
|
|
478
|
+
warnings.warn(warning_text)
|
|
479
|
+
return super().__iter__()
|
|
480
|
+
|
|
472
481
|
def take(self, indices, allow_fill=False, fill_value=None):
|
|
473
482
|
# Want to take rows of term freqs
|
|
474
483
|
row_indices = np.arange(len(self.term_mat.rows))
|
|
@@ -610,6 +619,7 @@ class SearchArray(ExtensionArray):
|
|
|
610
619
|
|
|
611
620
|
def score(self, token: Union[str, List[str]],
|
|
612
621
|
similarity: Similarity = default_bm25,
|
|
622
|
+
slop: int = 0,
|
|
613
623
|
min_posn: Optional[int] = None,
|
|
614
624
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
615
625
|
"""Score each doc using a similarity function.
|
|
@@ -629,7 +639,8 @@ class SearchArray(ExtensionArray):
|
|
|
629
639
|
tokens_l = [token] if isinstance(token, str) else token
|
|
630
640
|
all_dfs = np.asarray([self.docfreq(token) for token in tokens_l])
|
|
631
641
|
|
|
632
|
-
tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn
|
|
642
|
+
tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn,
|
|
643
|
+
slop=slop)
|
|
633
644
|
token = self._check_token_arg(token)
|
|
634
645
|
doc_lens = self.doclengths()
|
|
635
646
|
|
|
@@ -647,6 +658,8 @@ class SearchArray(ExtensionArray):
|
|
|
647
658
|
slop=0,
|
|
648
659
|
min_posn: Optional[int] = None,
|
|
649
660
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
661
|
+
if slop > 0:
|
|
662
|
+
logger.warning("!! Slop is experimental and may be slow, crash, or inaccurate etc")
|
|
650
663
|
try:
|
|
651
664
|
# Decide how/if we need to filter doc ids
|
|
652
665
|
term_ids = [self.term_dict.get_term_id(token) for token in tokens]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
|
|
2
2
|
from searcharray.roaringish.search import count_odds # noqa
|
|
3
|
-
from searcharray.roaringish.popcount import popcount_reduce_at, popcount64 # noqa
|
|
3
|
+
from searcharray.roaringish.popcount import popcount_reduce_at, popcount64, key_sum_over # noqa
|
|
4
4
|
from searcharray.roaringish.intersect import intersect, adjacent # noqa
|
|
5
5
|
from searcharray.roaringish.unique import unique # noqa
|
|
6
6
|
from searcharray.roaringish.merge import merge # noqa
|