searcharray 0.0.53__tar.gz → 0.0.55__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.53 → searcharray-0.0.55}/PKG-INFO +1 -1
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/phrase/bigram_freqs.py +43 -39
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/phrase/middle_out.py +62 -12
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/postings.py +63 -81
- searcharray-0.0.55/searcharray/roaringish/__init__.py +7 -0
- searcharray-0.0.53/searcharray/roaringish/snp_ops.c → searcharray-0.0.55/searcharray/roaringish/intersect.c +12344 -17504
- searcharray-0.0.55/searcharray/roaringish/intersect.pyi +25 -0
- searcharray-0.0.55/searcharray/roaringish/intersect.pyx +395 -0
- searcharray-0.0.55/searcharray/roaringish/merge.c +31524 -0
- searcharray-0.0.55/searcharray/roaringish/merge.pyi +8 -0
- searcharray-0.0.55/searcharray/roaringish/merge.pyx +160 -0
- searcharray-0.0.55/searcharray/roaringish/popcount.c +31835 -0
- searcharray-0.0.55/searcharray/roaringish/popcount.pyi +20 -0
- searcharray-0.0.55/searcharray/roaringish/popcount.pyx +113 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/roaringish.py +34 -3
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/roaringish_ops.c +300 -1076
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/roaringish_ops.pyx +1 -33
- searcharray-0.0.55/searcharray/roaringish/search.c +31761 -0
- searcharray-0.0.55/searcharray/roaringish/search.pyi +25 -0
- searcharray-0.0.55/searcharray/roaringish/search.pyx +141 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/snp_ops.pxd +0 -10
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/spans.c +242 -370
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/spans.pyx +7 -1
- searcharray-0.0.55/searcharray/roaringish/unique.c +31772 -0
- searcharray-0.0.55/searcharray/roaringish/unique.pyi +6 -0
- searcharray-0.0.55/searcharray/roaringish/unique.pyx +147 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/similarity.py +15 -6
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/solr.py +86 -42
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/SOURCES.txt +15 -4
- {searcharray-0.0.53 → searcharray-0.0.55}/setup.py +3 -3
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_minmax_posns.py +1 -5
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_msmarco.py +176 -145
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_phrase_matches.py +76 -19
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_search.py +9 -6
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_snp_ops.py +100 -7
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_tmdb.py +1 -1
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_utils.py +8 -4
- searcharray-0.0.53/searcharray/phrase/posn_diffs.py +0 -147
- searcharray-0.0.53/searcharray/phrase/scan_merge.py +0 -84
- searcharray-0.0.53/searcharray/roaringish/__init__.py +0 -4
- searcharray-0.0.53/searcharray/roaringish/snp_ops.pyx +0 -624
- {searcharray-0.0.53 → searcharray-0.0.55}/README.md +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/pyproject.toml +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/__init__.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/indexing.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/phrase/spans.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/utils/sort.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/setup.cfg +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_extension_array.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_similarity.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_slop_matches.py +0 -0
- {searcharray-0.0.53 → searcharray-0.0.55}/test/test_solr.py +0 -0
|
@@ -5,7 +5,7 @@ from searcharray.roaringish import RoaringishEncoder
|
|
|
5
5
|
import logging
|
|
6
6
|
from enum import Enum
|
|
7
7
|
|
|
8
|
-
from searcharray.roaringish import intersect, popcount64, merge
|
|
8
|
+
from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
logger = logging.getLogger(__name__)
|
|
@@ -99,7 +99,10 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
|
|
|
99
99
|
return phrase_freqs, (lhs_cont, rhs_cont)
|
|
100
100
|
|
|
101
101
|
|
|
102
|
-
def _inner_bigram_freqs(lhs: np.ndarray,
|
|
102
|
+
def _inner_bigram_freqs(lhs: np.ndarray,
|
|
103
|
+
rhs: np.ndarray,
|
|
104
|
+
lhs_int: np.ndarray,
|
|
105
|
+
rhs_int: np.ndarray,
|
|
103
106
|
phrase_freqs: np.ndarray,
|
|
104
107
|
cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
105
108
|
"""Count bigram matches between two encoded arrays, within a 64 bit word with same MSBs.
|
|
@@ -108,6 +111,8 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
108
111
|
-----------
|
|
109
112
|
lhs: roaringish encoded posn array of left term
|
|
110
113
|
rhs: roaringish encoded posn array of right term
|
|
114
|
+
lhs_int: np.ndarray, intersected lhs
|
|
115
|
+
rhs_int: np.ndarray, intersected rhs
|
|
111
116
|
phrase_freqs: np.ndarray, preallocated phrase freqs for output
|
|
112
117
|
cont_rhs: bool, whether to continue matching on the rhs or lhs
|
|
113
118
|
|
|
@@ -117,7 +122,7 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
117
122
|
cont_next: the next (lhs or rhs) array to continue matching
|
|
118
123
|
|
|
119
124
|
"""
|
|
120
|
-
lhs_int, rhs_int = encoder.intersect(lhs, rhs)
|
|
125
|
+
# lhs_int, rhs_int = encoder.intersect(lhs, rhs)
|
|
121
126
|
lhs_doc_ids = encoder.keys(lhs_int)
|
|
122
127
|
if len(lhs_int) != len(rhs_int):
|
|
123
128
|
raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
|
|
@@ -144,18 +149,12 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
144
149
|
lhs_next = overlap_bits.copy()
|
|
145
150
|
lhs_next |= (lhs_int & (encoder.key_mask | encoder.payload_msb_mask))
|
|
146
151
|
|
|
147
|
-
|
|
148
|
-
if np.any(matches2):
|
|
149
|
-
transitions = np.argwhere(np.diff(lhs_doc_ids[matches2]) != 0).flatten() + 1
|
|
150
|
-
transitions = np.insert(transitions, 0, 0)
|
|
151
|
-
counted_bits = popcount64(overlap_bits[matches2])
|
|
152
|
-
reduced = np.add.reduceat(counted_bits,
|
|
153
|
-
transitions)
|
|
154
|
-
phrase_freqs[np.unique(lhs_doc_ids[matches2])] += reduced
|
|
152
|
+
popcount_reduce_at(lhs_doc_ids, overlap_bits, phrase_freqs)
|
|
155
153
|
return phrase_freqs, (lhs_next, rhs_next)
|
|
156
154
|
|
|
157
155
|
|
|
158
156
|
def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
157
|
+
lhs_adj: np.ndarray, rhs_adj: np.ndarray,
|
|
159
158
|
phrase_freqs: np.ndarray,
|
|
160
159
|
cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
161
160
|
"""Count bigram matches between two encoded arrays where they occur in adjacent 64 bit words.
|
|
@@ -166,10 +165,9 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
166
165
|
rhs_next: the next rhs array to continue matching
|
|
167
166
|
|
|
168
167
|
"""
|
|
169
|
-
|
|
170
|
-
lhs_doc_ids = encoder.keys(lhs_int)
|
|
168
|
+
lhs_doc_ids = encoder.keys(lhs_adj)
|
|
171
169
|
# lhs lsb set and rhs lsb's most significant bit set
|
|
172
|
-
matches = ((
|
|
170
|
+
matches = ((lhs_adj & _upper_bit) != 0) & ((rhs_adj & _1) != 0)
|
|
173
171
|
unique, counts = np.unique(lhs_doc_ids[matches], return_counts=True)
|
|
174
172
|
phrase_freqs[unique] += counts
|
|
175
173
|
# Set lsb to 0 where no match, lsb to 1 where match
|
|
@@ -177,40 +175,40 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
177
175
|
lhs_next = None if cont == Continuation.RHS else np.asarray([], dtype=np.uint64)
|
|
178
176
|
if np.any(matches):
|
|
179
177
|
if cont in [Continuation.RHS, Continuation.BOTH]:
|
|
180
|
-
rhs_next =
|
|
178
|
+
rhs_next = rhs_adj[matches]
|
|
181
179
|
assert rhs_next is not None
|
|
182
|
-
rhs_next = encoder.header(
|
|
180
|
+
rhs_next = encoder.header(rhs_adj) | _1
|
|
183
181
|
elif cont in [Continuation.LHS, Continuation.BOTH]:
|
|
184
|
-
rhs_next =
|
|
182
|
+
rhs_next = rhs_adj[matches]
|
|
185
183
|
assert rhs_next is not None
|
|
186
184
|
rhs_next |= _1
|
|
187
|
-
rhs_next = encoder.header(
|
|
188
|
-
lhs_next =
|
|
189
|
-
lhs_next = encoder.header(
|
|
185
|
+
rhs_next = encoder.header(rhs_adj) | _1
|
|
186
|
+
lhs_next = lhs_adj[matches]
|
|
187
|
+
lhs_next = encoder.header(lhs_adj) | _upper_bit
|
|
190
188
|
rhs_next = None
|
|
191
189
|
return phrase_freqs, (lhs_next, rhs_next)
|
|
192
190
|
|
|
193
191
|
|
|
194
|
-
def _set_adjbit_at_header(
|
|
192
|
+
def _set_adjbit_at_header(next_inner: np.ndarray, next_adj: np.ndarray,
|
|
195
193
|
cont: Continuation = Continuation.RHS) -> np.ndarray:
|
|
196
194
|
"""Merge two encoded arrays on their headers."""
|
|
197
|
-
if len(
|
|
198
|
-
return
|
|
199
|
-
if len(
|
|
200
|
-
return
|
|
195
|
+
if len(next_inner) == 0:
|
|
196
|
+
return next_adj
|
|
197
|
+
if len(next_adj) == 0:
|
|
198
|
+
return next_inner
|
|
201
199
|
|
|
202
|
-
|
|
203
|
-
|
|
200
|
+
same_header_inner, same_header_adj = intersect(next_inner, next_adj,
|
|
201
|
+
mask=encoder.header_mask)
|
|
204
202
|
# Set _1 on intersection
|
|
205
|
-
ignore_mask = np.ones(len(
|
|
206
|
-
ignore_mask[
|
|
207
|
-
if len(
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
if len(
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
return merge(
|
|
203
|
+
ignore_mask = np.ones(len(next_adj), dtype=bool)
|
|
204
|
+
ignore_mask[same_header_adj] = False
|
|
205
|
+
if len(same_header_inner) > 0 and cont == Continuation.RHS:
|
|
206
|
+
next_inner[same_header_inner] |= _1
|
|
207
|
+
next_adj = next_adj[ignore_mask]
|
|
208
|
+
if len(same_header_adj) > 0 and cont == Continuation.LHS:
|
|
209
|
+
next_inner[same_header_adj] |= _upper_bit
|
|
210
|
+
next_adj = next_adj[ignore_mask]
|
|
211
|
+
return merge(next_inner, next_adj)
|
|
214
212
|
|
|
215
213
|
|
|
216
214
|
def bigram_freqs(lhs: np.ndarray,
|
|
@@ -249,9 +247,13 @@ def bigram_freqs(lhs: np.ndarray,
|
|
|
249
247
|
# rhs_term: 1234 101 0101000100
|
|
250
248
|
# ^^ ^^ <- inner matches, terms next to each other
|
|
251
249
|
#
|
|
252
|
-
|
|
253
|
-
= _inner_bigram_freqs(lhs, rhs, phrase_freqs, cont)
|
|
250
|
+
lhs_int, rhs_int, lhs_adj, rhs_adj = encoder.intersect_candidates(lhs, rhs)
|
|
254
251
|
|
|
252
|
+
phrase_freqs, (lhs_next_inner, rhs_next_inner)\
|
|
253
|
+
= _inner_bigram_freqs(lhs, rhs,
|
|
254
|
+
lhs_int, rhs_int,
|
|
255
|
+
phrase_freqs,
|
|
256
|
+
cont)
|
|
255
257
|
# ***************************************************************************
|
|
256
258
|
# "Adjacent" matches:
|
|
257
259
|
# Of course we also need to check for matches that span two 64 bit words,
|
|
@@ -266,7 +268,9 @@ def bigram_freqs(lhs: np.ndarray,
|
|
|
266
268
|
# 1 less than the posn_msb of rhs
|
|
267
269
|
# so we detect these differently
|
|
268
270
|
phrase_freqs, (lhs_next_adj, rhs_next_adj)\
|
|
269
|
-
= _adjacent_bigram_freqs(lhs, rhs,
|
|
271
|
+
= _adjacent_bigram_freqs(lhs, rhs,
|
|
272
|
+
lhs_adj, rhs_adj,
|
|
273
|
+
phrase_freqs, cont)
|
|
270
274
|
|
|
271
275
|
# ***************************************************************************
|
|
272
276
|
# rhs_next is where the bigram ends on the rhs side, we can use this
|
|
@@ -13,7 +13,7 @@ from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
|
|
|
13
13
|
from searcharray.phrase.spans import span_search
|
|
14
14
|
import numbers
|
|
15
15
|
import logging
|
|
16
|
-
from collections import defaultdict
|
|
16
|
+
from collections import defaultdict, abc
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
logger = logging.getLogger(__name__)
|
|
@@ -72,6 +72,7 @@ def trim_phrase_search(encoded_posns: List[np.ndarray],
|
|
|
72
72
|
|
|
73
73
|
def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
|
|
74
74
|
phrase_freqs: np.ndarray,
|
|
75
|
+
max_doc_id: np.uint64 = _0,
|
|
75
76
|
trim: bool = True) -> np.ndarray:
|
|
76
77
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
77
78
|
if len(encoded_posns) < 2:
|
|
@@ -86,7 +87,8 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
|
|
|
86
87
|
for rhs in encoded_posns[1:]:
|
|
87
88
|
# Only count the count of the last bigram (ignoring the ones where priors did not match)
|
|
88
89
|
phrase_freqs[mask] = 0
|
|
89
|
-
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
|
|
90
|
+
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
|
|
91
|
+
cont=Continuation.RHS)
|
|
90
92
|
assert conts[1] is not None
|
|
91
93
|
lhs = conts[1]
|
|
92
94
|
mask &= (phrase_freqs > 0)
|
|
@@ -99,6 +101,7 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
|
|
|
99
101
|
|
|
100
102
|
def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
|
|
101
103
|
phrase_freqs: np.ndarray,
|
|
104
|
+
max_doc_id: np.uint64 = _0,
|
|
102
105
|
trim: bool = True) -> np.ndarray:
|
|
103
106
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
104
107
|
if len(encoded_posns) < 2:
|
|
@@ -113,7 +116,8 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
|
|
|
113
116
|
for lhs in encoded_posns[-2::-1]:
|
|
114
117
|
# Only count the count of the last bigram (ignoring the ones where priors did not match)
|
|
115
118
|
phrase_freqs[mask] = 0
|
|
116
|
-
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
|
|
119
|
+
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
|
|
120
|
+
cont=Continuation.LHS)
|
|
117
121
|
assert conts[0] is not None
|
|
118
122
|
rhs = conts[0]
|
|
119
123
|
mask &= (phrase_freqs > 0)
|
|
@@ -126,18 +130,19 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
|
|
|
126
130
|
|
|
127
131
|
def compute_phrase_freqs(encoded_posns: List[np.ndarray],
|
|
128
132
|
phrase_freqs: np.ndarray,
|
|
133
|
+
max_doc_id: np.uint64 = _0,
|
|
129
134
|
trim: bool = False) -> np.ndarray:
|
|
130
135
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
131
136
|
shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
|
|
132
137
|
if shortest_len_index <= 1:
|
|
133
|
-
return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim)
|
|
138
|
+
return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim, max_doc_id=max_doc_id)
|
|
134
139
|
elif shortest_len_index >= len(encoded_posns) - 2:
|
|
135
|
-
return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim)
|
|
140
|
+
return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim, max_doc_id=max_doc_id)
|
|
136
141
|
else:
|
|
137
142
|
# We optimize this case by going middle-out
|
|
138
143
|
# We can take the min of both directions phrase freqs
|
|
139
|
-
lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim)
|
|
140
|
-
rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim)
|
|
144
|
+
lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim, max_doc_id=max_doc_id)
|
|
145
|
+
rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim, max_doc_id=max_doc_id)
|
|
141
146
|
phrase_freqs = np.minimum(lhs, rhs)
|
|
142
147
|
return phrase_freqs
|
|
143
148
|
|
|
@@ -259,6 +264,35 @@ def index_range(rng, key):
|
|
|
259
264
|
return np.asarray(list(rng))[key]
|
|
260
265
|
|
|
261
266
|
|
|
267
|
+
class FilteredPosns(abc.Mapping):
|
|
268
|
+
"""When someone slices, this lets us avoid repeating constantly
|
|
269
|
+
slicing the keys out of the encoded positions."""
|
|
270
|
+
|
|
271
|
+
def __init__(self, base, doc_ids):
|
|
272
|
+
self.base = base
|
|
273
|
+
self.sliced = {}
|
|
274
|
+
self.doc_ids = doc_ids
|
|
275
|
+
|
|
276
|
+
def __getitem__(self, key):
|
|
277
|
+
"""Slice with encoder, cache, and return."""
|
|
278
|
+
if key in self.sliced:
|
|
279
|
+
return self.sliced[key]
|
|
280
|
+
|
|
281
|
+
sliced = encoder.slice(self.base[key],
|
|
282
|
+
keys=self.doc_ids)
|
|
283
|
+
self.sliced[key] = sliced
|
|
284
|
+
return sliced
|
|
285
|
+
|
|
286
|
+
def __setitem__(self, key, value):
|
|
287
|
+
self.base[key] = value
|
|
288
|
+
|
|
289
|
+
def __iter__(self):
|
|
290
|
+
return iter(self.doc_ids)
|
|
291
|
+
|
|
292
|
+
def __len__(self):
|
|
293
|
+
return len(self.doc_ids)
|
|
294
|
+
|
|
295
|
+
|
|
262
296
|
class PosnBitArray:
|
|
263
297
|
|
|
264
298
|
def __init__(self, encoded_term_posns, max_doc_id: int):
|
|
@@ -274,9 +308,21 @@ class PosnBitArray:
|
|
|
274
308
|
self.docfreq(term_id)
|
|
275
309
|
self.termfreqs(term_id)
|
|
276
310
|
|
|
311
|
+
def filter(self, doc_ids) -> "PosnBitArray":
|
|
312
|
+
"""Filter my doc ids to only those in doc_ids."""
|
|
313
|
+
enc_term_posns = self.encoded_term_posns
|
|
314
|
+
if isinstance(enc_term_posns, FilteredPosns):
|
|
315
|
+
self.encoded_term_posns = enc_term_posns.base
|
|
316
|
+
return PosnBitArray(FilteredPosns(self.encoded_term_posns, doc_ids), self.max_doc_id)
|
|
317
|
+
|
|
318
|
+
def _reset_filter(self):
|
|
319
|
+
if isinstance(self.encoded_term_posns, FilteredPosns):
|
|
320
|
+
self.encoded_term_posns = self.encoded_term_posns.base
|
|
321
|
+
|
|
277
322
|
def clear_cache(self):
|
|
278
323
|
self.docfreq_cache = {}
|
|
279
324
|
self.termfreq_cache = {}
|
|
325
|
+
self._reset_filter()
|
|
280
326
|
|
|
281
327
|
def copy(self):
|
|
282
328
|
new = PosnBitArray(deepcopy(self.encoded_term_posns), self.max_doc_id)
|
|
@@ -335,18 +381,22 @@ class PosnBitArray:
|
|
|
335
381
|
keys=np.asarray([doc_id], dtype=np.uint64))
|
|
336
382
|
return term_posns
|
|
337
383
|
|
|
338
|
-
def
|
|
339
|
-
|
|
384
|
+
def empty_buffer(self):
|
|
385
|
+
return np.zeros(int(self.max_doc_id + 1), dtype=np.float64)
|
|
386
|
+
|
|
387
|
+
def phrase_freqs(self, term_ids: List[int],
|
|
340
388
|
slop: int = 0,
|
|
389
|
+
doc_ids: Optional[np.ndarray] = None,
|
|
341
390
|
min_posn: Optional[int] = None,
|
|
342
391
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
392
|
+
phrase_freqs = self.empty_buffer()
|
|
343
393
|
if len(term_ids) < 2:
|
|
344
394
|
raise ValueError("Must have at least two terms")
|
|
345
|
-
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
|
|
395
|
+
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
|
|
346
396
|
enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
|
|
347
397
|
else:
|
|
348
398
|
keys = None
|
|
349
|
-
if
|
|
399
|
+
if doc_ids is not None:
|
|
350
400
|
keys = doc_ids.view(np.uint64)
|
|
351
401
|
|
|
352
402
|
enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
|
|
@@ -355,7 +405,7 @@ class PosnBitArray:
|
|
|
355
405
|
max_payload=max_posn) for term_id in term_ids]
|
|
356
406
|
|
|
357
407
|
if slop == 0:
|
|
358
|
-
return compute_phrase_freqs(enc_term_posns, phrase_freqs)
|
|
408
|
+
return compute_phrase_freqs(enc_term_posns, phrase_freqs, max_doc_id=np.uint64(self.max_doc_id))
|
|
359
409
|
else:
|
|
360
410
|
return span_search(enc_term_posns, phrase_freqs, slop)
|
|
361
411
|
|
|
@@ -12,8 +12,6 @@ from typing import List, Union, Optional, Iterable
|
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
import numpy as np
|
|
15
|
-
from searcharray.phrase.scan_merge import scan_merge_ins
|
|
16
|
-
from searcharray.phrase.posn_diffs import compute_phrase_freqs
|
|
17
15
|
from searcharray.phrase.middle_out import PosnBitArray
|
|
18
16
|
from searcharray.similarity import Similarity, default_bm25
|
|
19
17
|
from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
|
|
@@ -318,7 +316,7 @@ class SearchArray(ExtensionArray):
|
|
|
318
316
|
if not self.avoid_copies:
|
|
319
317
|
sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
|
|
320
318
|
else:
|
|
321
|
-
sliced_posns = self.posns
|
|
319
|
+
sliced_posns = self.posns.filter(sliced_tfs.rows)
|
|
322
320
|
|
|
323
321
|
arr = SearchArray([], tokenizer=self.tokenizer)
|
|
324
322
|
arr.term_mat = sliced_tfs
|
|
@@ -525,8 +523,42 @@ class SearchArray(ExtensionArray):
|
|
|
525
523
|
else:
|
|
526
524
|
raise TypeError("Expected a string or list of strings for phrases")
|
|
527
525
|
|
|
526
|
+
def memory_report(self):
|
|
527
|
+
"""Return a string with memory usage information."""
|
|
528
|
+
term_mat_bytes = self.term_mat.nbytes
|
|
529
|
+
posns_bytes = self.posns.nbytes
|
|
530
|
+
term_dict_bytes = self.term_dict.nbytes
|
|
531
|
+
num_terms = len(self.term_dict)
|
|
532
|
+
top_10_terms_size = []
|
|
533
|
+
|
|
534
|
+
for i in range(10):
|
|
535
|
+
term_len = len(self.posns.encoded_term_posns[i])
|
|
536
|
+
term = self.term_dict.get_term(i)
|
|
537
|
+
top_10_terms_size.append((term, term_len))
|
|
538
|
+
|
|
539
|
+
return f"""
|
|
540
|
+
SearchArray Memory Report
|
|
541
|
+
-------------------------
|
|
542
|
+
Number of Terms: {num_terms}
|
|
543
|
+
-------------------------
|
|
544
|
+
Term Matrix: {term_mat_bytes}
|
|
545
|
+
Positions: {posns_bytes}
|
|
546
|
+
Term Dictionary: {term_dict_bytes}
|
|
547
|
+
--------------------------
|
|
548
|
+
Term 0: {top_10_terms_size[0]}
|
|
549
|
+
Term 1: {top_10_terms_size[1]}
|
|
550
|
+
Term 2: {top_10_terms_size[2]}
|
|
551
|
+
Term 3: {top_10_terms_size[3]}
|
|
552
|
+
Term 4: {top_10_terms_size[4]}
|
|
553
|
+
Term 5: {top_10_terms_size[5]}
|
|
554
|
+
Term 6: {top_10_terms_size[6]}
|
|
555
|
+
Term 7: {top_10_terms_size[7]}
|
|
556
|
+
Term 8: {top_10_terms_size[8]}
|
|
557
|
+
Term 9: {top_10_terms_size[9]}
|
|
558
|
+
"""
|
|
559
|
+
|
|
528
560
|
# ***********************************************************
|
|
529
|
-
# Search
|
|
561
|
+
# Search API
|
|
530
562
|
# ***********************************************************
|
|
531
563
|
def termfreqs(self, token: Union[List[str], str],
|
|
532
564
|
slop: int = 0,
|
|
@@ -534,23 +566,25 @@ class SearchArray(ExtensionArray):
|
|
|
534
566
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
535
567
|
token = self._check_token_arg(token)
|
|
536
568
|
if isinstance(token, list):
|
|
537
|
-
return self.
|
|
569
|
+
return self._phrase_freq(token, slop=slop, min_posn=min_posn, max_posn=max_posn)
|
|
538
570
|
|
|
539
571
|
try:
|
|
540
572
|
term_id = self.term_dict.get_term_id(token)
|
|
541
|
-
slice_of_rows =
|
|
573
|
+
slice_of_rows = self.term_mat.rows
|
|
542
574
|
if self.term_mat.subset:
|
|
543
575
|
matches = np.zeros(len(self), dtype=np.float32)
|
|
544
|
-
slice_of_rows = self.term_mat.rows
|
|
545
576
|
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
546
|
-
doc_ids=slice_of_rows,
|
|
577
|
+
doc_ids=slice_of_rows,
|
|
578
|
+
min_posn=min_posn,
|
|
579
|
+
max_posn=max_posn)
|
|
547
580
|
mask = np.isin(self.term_mat.rows, doc_ids)
|
|
548
581
|
matches[mask] = termfreqs
|
|
549
582
|
return matches
|
|
550
583
|
else:
|
|
551
584
|
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
552
|
-
doc_ids=
|
|
553
|
-
|
|
585
|
+
doc_ids=None,
|
|
586
|
+
min_posn=min_posn,
|
|
587
|
+
max_posn=max_posn)
|
|
554
588
|
return as_dense(doc_ids, termfreqs, len(self))
|
|
555
589
|
except TermMissingError:
|
|
556
590
|
return np.zeros(len(self), dtype=np.float32)
|
|
@@ -567,16 +601,8 @@ class SearchArray(ExtensionArray):
|
|
|
567
601
|
def doclengths(self) -> np.ndarray:
|
|
568
602
|
return self.doc_lens
|
|
569
603
|
|
|
570
|
-
def
|
|
571
|
-
|
|
572
|
-
token = self._check_token_arg(token)
|
|
573
|
-
if isinstance(token, list):
|
|
574
|
-
term_freq = self.phrase_freq(token, slop=slop)
|
|
575
|
-
else:
|
|
576
|
-
term_freq = self.termfreqs(token)
|
|
577
|
-
return term_freq > 0
|
|
578
|
-
|
|
579
|
-
def score(self, token: Union[str, List[str]], similarity: Similarity = default_bm25,
|
|
604
|
+
def score(self, token: Union[str, List[str]],
|
|
605
|
+
similarity: Similarity = default_bm25,
|
|
580
606
|
min_posn: Optional[int] = None,
|
|
581
607
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
582
608
|
"""Score each doc using a similarity function.
|
|
@@ -585,6 +611,8 @@ class SearchArray(ExtensionArray):
|
|
|
585
611
|
----------
|
|
586
612
|
token : str or list of str of what to search (already tokenized)
|
|
587
613
|
similarity : How to score the documents. Default is BM25.
|
|
614
|
+
min_posn : int - minimum position of the term in the document, in multiples of 18
|
|
615
|
+
max_posn : int - maximum position of the term in the document, in multiples of 18
|
|
588
616
|
"""
|
|
589
617
|
# Get term freqs per token
|
|
590
618
|
token = self._check_token_arg(token)
|
|
@@ -608,67 +636,21 @@ class SearchArray(ExtensionArray):
|
|
|
608
636
|
posns = self.posns.positions(term_id, doc_ids=key)
|
|
609
637
|
return posns
|
|
610
638
|
|
|
611
|
-
def
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
for curr_mask in masks:
|
|
616
|
-
mask = mask & curr_mask
|
|
617
|
-
return mask
|
|
618
|
-
|
|
619
|
-
def or_query(self, tokens: Union[List[str], List[List[str]]], min_should_match: int = 1) -> np.ndarray:
|
|
620
|
-
"""Return a mask on the postings array indicating which elements contain all terms."""
|
|
621
|
-
masks = [self.match(term) for term in tokens]
|
|
622
|
-
mask = np.sum(masks, axis=0) >= min_should_match
|
|
623
|
-
return mask
|
|
624
|
-
|
|
625
|
-
def phrase_freq(self, tokens: List[str],
|
|
626
|
-
slop=0,
|
|
627
|
-
min_posn: Optional[int] = None,
|
|
628
|
-
max_posn: Optional[int] = None) -> np.ndarray:
|
|
629
|
-
phrase_freqs = np.zeros(len(self))
|
|
639
|
+
def _phrase_freq(self, tokens: List[str],
|
|
640
|
+
slop=0,
|
|
641
|
+
min_posn: Optional[int] = None,
|
|
642
|
+
max_posn: Optional[int] = None) -> np.ndarray:
|
|
630
643
|
try:
|
|
631
|
-
|
|
644
|
+
# Decide how/if we need to filter doc ids
|
|
632
645
|
term_ids = [self.term_dict.get_term_id(token) for token in tokens]
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
return phrase_freqs
|
|
640
|
-
|
|
641
|
-
def phrase_freq_scan(self, tokens: List[str], mask=None, slop=0) -> np.ndarray:
|
|
642
|
-
if mask is None:
|
|
643
|
-
mask = self.and_query(tokens)
|
|
644
|
-
|
|
645
|
-
if np.sum(mask) == 0:
|
|
646
|
-
return mask
|
|
647
|
-
|
|
648
|
-
# Gather positions
|
|
649
|
-
posns = [self.positions(token, mask) for token in tokens]
|
|
650
|
-
phrase_freqs = np.zeros(len(self))
|
|
651
|
-
|
|
652
|
-
phrase_freqs[mask] = scan_merge_ins(posns, phrase_freqs[mask], slop=slop + 1)
|
|
653
|
-
return phrase_freqs
|
|
654
|
-
|
|
655
|
-
def phrase_freq_every_diff(self, tokens: List[str], slop=0) -> np.ndarray:
|
|
656
|
-
phrase_freqs = -np.ones(len(self))
|
|
657
|
-
|
|
658
|
-
mask = self.and_query(tokens)
|
|
659
|
-
phrase_freqs[~mask] = 0
|
|
660
|
-
if np.sum(mask) == 0:
|
|
646
|
+
phrase_freqs = self.posns.phrase_freqs(term_ids,
|
|
647
|
+
slop=slop,
|
|
648
|
+
min_posn=min_posn,
|
|
649
|
+
max_posn=max_posn)
|
|
650
|
+
if self.term_mat.subset:
|
|
651
|
+
return phrase_freqs[self.term_mat.rows]
|
|
661
652
|
return phrase_freqs
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
phrase_freqs[mask],
|
|
667
|
-
slop=slop + 1,
|
|
668
|
-
width=width)
|
|
669
|
-
|
|
670
|
-
remaining_mask = phrase_freqs == -1
|
|
671
|
-
if np.any(remaining_mask):
|
|
672
|
-
remainder_freqs = self.phrase_freq_scan(tokens, mask=remaining_mask, slop=slop)
|
|
673
|
-
phrase_freqs[remaining_mask] = remainder_freqs[remaining_mask]
|
|
674
|
-
return phrase_freqs
|
|
653
|
+
except TermMissingError:
|
|
654
|
+
if self.term_mat.subset:
|
|
655
|
+
return np.zeros(len(self), dtype=np.float32)
|
|
656
|
+
return self.posns.empty_buffer()
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
|
|
2
|
+
from searcharray.roaringish.search import count_odds # noqa
|
|
3
|
+
from searcharray.roaringish.popcount import popcount_reduce_at, popcount64 # noqa
|
|
4
|
+
from searcharray.roaringish.intersect import intersect, adjacent # noqa
|
|
5
|
+
from searcharray.roaringish.unique import unique # noqa
|
|
6
|
+
from searcharray.roaringish.merge import merge # noqa
|
|
7
|
+
from searcharray.roaringish.spans import span_search # noqa
|