searcharray 0.0.73__cp313-cp313-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- searcharray/__init__.py +2 -0
- searcharray/bm25/__init__.py +1 -0
- searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
- searcharray/bm25/bm25.pyi +11 -0
- searcharray/bm25/bm25.pyx +41 -0
- searcharray/indexing.py +342 -0
- searcharray/phrase/__init__.py +0 -0
- searcharray/phrase/bigram_freqs.py +307 -0
- searcharray/phrase/memmap_arrays.py +208 -0
- searcharray/phrase/middle_out.py +553 -0
- searcharray/phrase/spans.py +187 -0
- searcharray/postings.py +708 -0
- searcharray/roaringish/__init__.py +7 -0
- searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/intersect.pyi +25 -0
- searcharray/roaringish/intersect.pyx +390 -0
- searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/merge.pyi +16 -0
- searcharray/roaringish/merge.pyx +232 -0
- searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/popcount.pyi +36 -0
- searcharray/roaringish/popcount.pyx +278 -0
- searcharray/roaringish/roaringish.py +298 -0
- searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/roaringish_ops.pyx +98 -0
- searcharray/roaringish/scatter_assign.h +31 -0
- searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/search.pyi +25 -0
- searcharray/roaringish/search.pyx +140 -0
- searcharray/roaringish/snp_ops.pxd +13 -0
- searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/spans.pyx +330 -0
- searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/unique.pyi +6 -0
- searcharray/roaringish/unique.pyx +145 -0
- searcharray/similarity.py +92 -0
- searcharray/solr.py +355 -0
- searcharray/term_dict.py +59 -0
- searcharray/utils/__init__.py +0 -0
- searcharray/utils/bitcount.py +34 -0
- searcharray/utils/mat_set.py +170 -0
- searcharray/utils/row_viewable_matrix.py +91 -0
- searcharray/utils/sort.py +45 -0
- searcharray-0.0.73.dist-info/METADATA +245 -0
- searcharray-0.0.73.dist-info/RECORD +48 -0
- searcharray-0.0.73.dist-info/WHEEL +5 -0
- searcharray-0.0.73.dist-info/entry_points.txt +2 -0
- searcharray-0.0.73.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,553 @@
|
|
|
1
|
+
"""Encode positions in bits along with some neighboring information for wrapping.
|
|
2
|
+
|
|
3
|
+
See this notebook for motivation:
|
|
4
|
+
|
|
5
|
+
https://colab.research.google.com/drive/10tIEkdlCE_1J_CcgEcV0jkLfBc-0H4am?authuser=1#scrollTo=XWzy-n9dF3PG
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
import numpy as np
|
|
9
|
+
from copy import deepcopy
|
|
10
|
+
from typing import List, Tuple, Dict, Union, cast, Optional
|
|
11
|
+
from searcharray.roaringish import RoaringishEncoder, convert_keys, merge, intersect
|
|
12
|
+
from searcharray.phrase.memmap_arrays import MemoryMappedArrays, ArrayDict
|
|
13
|
+
from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
|
|
14
|
+
from searcharray.phrase.spans import span_search
|
|
15
|
+
import numbers
|
|
16
|
+
import logging
|
|
17
|
+
from collections import defaultdict, abc
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
# When running in pytest
|
|
23
|
+
import sys # noqa
|
|
24
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
25
|
+
handler.setLevel(logging.ERROR)
|
|
26
|
+
formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
|
|
27
|
+
handler.setFormatter(formatter)
|
|
28
|
+
logger.addHandler(handler)
|
|
29
|
+
logger.setLevel(logging.ERROR)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
encoder = RoaringishEncoder()
|
|
33
|
+
|
|
34
|
+
# To not constantly type coerce
|
|
35
|
+
_64 = np.uint64(64)
|
|
36
|
+
_2 = np.uint64(2)
|
|
37
|
+
_1 = np.uint64(1)
|
|
38
|
+
_0 = np.uint64(0)
|
|
39
|
+
_neg1 = np.int64(-1)
|
|
40
|
+
|
|
41
|
+
MAX_POSN = encoder.max_payload
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def trim_phrase_search(encoded_posns: List[np.ndarray]) -> List[np.ndarray]:
|
|
45
|
+
"""Trim long phrases by searching the rarest terms first."""
|
|
46
|
+
|
|
47
|
+
# Start with rarest term
|
|
48
|
+
shortest_keys = None
|
|
49
|
+
shortest_idx = None
|
|
50
|
+
min_len = 1e100
|
|
51
|
+
max_len = 0
|
|
52
|
+
for idx, enc_posn in enumerate(encoded_posns):
|
|
53
|
+
if len(enc_posn) < min_len:
|
|
54
|
+
shortest_keys = encoder.keys(enc_posn)
|
|
55
|
+
shortest_idx = idx
|
|
56
|
+
min_len = len(enc_posn)
|
|
57
|
+
if len(enc_posn) > max_len:
|
|
58
|
+
max_len = len(enc_posn)
|
|
59
|
+
|
|
60
|
+
if shortest_keys is None:
|
|
61
|
+
return encoded_posns
|
|
62
|
+
|
|
63
|
+
for enc_posn_idx in range(len(encoded_posns)):
|
|
64
|
+
if enc_posn_idx == shortest_idx:
|
|
65
|
+
continue
|
|
66
|
+
if len(encoded_posns[enc_posn_idx]) > (20 * min_len):
|
|
67
|
+
encoded_posns[enc_posn_idx] = encoder.slice(encoded_posns[enc_posn_idx],
|
|
68
|
+
shortest_keys)
|
|
69
|
+
|
|
70
|
+
return encoded_posns
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _intersect_bigram_matches(ids: Optional[np.ndarray],
|
|
74
|
+
counts: Optional[np.ndarray],
|
|
75
|
+
new_ids: np.ndarray,
|
|
76
|
+
new_counts: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
|
77
|
+
"""Intersect bigram matches."""
|
|
78
|
+
if ids is None or counts is None:
|
|
79
|
+
return new_ids, new_counts
|
|
80
|
+
else:
|
|
81
|
+
# Intersect (Are these sorted?)
|
|
82
|
+
is_sorted = np.all(np.diff(ids) >= 0)
|
|
83
|
+
assert is_sorted
|
|
84
|
+
is_sorted = np.all(np.diff(new_ids) >= 0)
|
|
85
|
+
assert is_sorted
|
|
86
|
+
|
|
87
|
+
ids_idx, new_ids_idx = intersect(ids, new_ids)
|
|
88
|
+
intersected_counts = counts[ids_idx]
|
|
89
|
+
counts = np.minimum(intersected_counts, new_counts[new_ids_idx])
|
|
90
|
+
ids = ids[ids_idx]
|
|
91
|
+
assert ids is not None
|
|
92
|
+
assert counts is not None
|
|
93
|
+
return ids, counts
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
|
|
97
|
+
max_doc_id: np.uint64 = _0,
|
|
98
|
+
trim: bool = True) -> Tuple[np.ndarray, np.ndarray]:
|
|
99
|
+
"""Compute phrase freqs from a set of encoded positions."""
|
|
100
|
+
if len(encoded_posns) < 2:
|
|
101
|
+
raise ValueError("phrase must have at least two terms")
|
|
102
|
+
|
|
103
|
+
# Trim long phrases by searching the rarest terms first
|
|
104
|
+
if trim and len(encoded_posns) > 3:
|
|
105
|
+
encoded_posns = trim_phrase_search(encoded_posns)
|
|
106
|
+
|
|
107
|
+
ids = None
|
|
108
|
+
counts = None
|
|
109
|
+
|
|
110
|
+
lhs = encoded_posns[0]
|
|
111
|
+
for rhs in encoded_posns[1:]:
|
|
112
|
+
# Only count the count of the last bigram (ignoring the ones where priors did not match)
|
|
113
|
+
phrase_freqs, conts = bigram_freqs(lhs, rhs,
|
|
114
|
+
cont=Continuation.RHS)
|
|
115
|
+
assert conts[1] is not None
|
|
116
|
+
lhs = conts[1]
|
|
117
|
+
ids, counts = _intersect_bigram_matches(ids, counts, phrase_freqs[0], phrase_freqs[1])
|
|
118
|
+
|
|
119
|
+
if ids is None or counts is None:
|
|
120
|
+
return np.array([], dtype=np.uint64), np.array([], dtype=np.float32)
|
|
121
|
+
|
|
122
|
+
return (ids, counts)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
|
|
126
|
+
max_doc_id: np.uint64 = _0,
|
|
127
|
+
trim: bool = True) -> Tuple[np.ndarray, np.ndarray]:
|
|
128
|
+
"""Compute phrase freqs from a set of encoded positions."""
|
|
129
|
+
if len(encoded_posns) < 2:
|
|
130
|
+
raise ValueError("phrase must have at least two terms")
|
|
131
|
+
|
|
132
|
+
# Trim long phrases by searching the rarest terms first
|
|
133
|
+
if trim and len(encoded_posns) > 3:
|
|
134
|
+
encoded_posns = trim_phrase_search(encoded_posns)
|
|
135
|
+
|
|
136
|
+
ids = None
|
|
137
|
+
counts = None
|
|
138
|
+
|
|
139
|
+
rhs = encoded_posns[-1]
|
|
140
|
+
for lhs in encoded_posns[-2::-1]:
|
|
141
|
+
# Only count the count of the last bigram (ignoring the ones where priors did not match)
|
|
142
|
+
phrase_freqs, conts = bigram_freqs(lhs, rhs,
|
|
143
|
+
cont=Continuation.LHS)
|
|
144
|
+
|
|
145
|
+
assert conts[0] is not None
|
|
146
|
+
rhs = conts[0]
|
|
147
|
+
ids, counts = _intersect_bigram_matches(ids, counts, phrase_freqs[0], phrase_freqs[1])
|
|
148
|
+
|
|
149
|
+
if ids is None or counts is None:
|
|
150
|
+
return np.array([], dtype=np.uint64), np.array([], dtype=np.float32)
|
|
151
|
+
return (ids, counts)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def compute_phrase_freqs(encoded_posns: List[np.ndarray],
|
|
155
|
+
max_doc_id: np.uint64 = _0,
|
|
156
|
+
trim: bool = False) -> Tuple[np.ndarray, np.ndarray]:
|
|
157
|
+
"""Compute phrase freqs from a set of encoded positions."""
|
|
158
|
+
shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
|
|
159
|
+
if shortest_len_index <= 1:
|
|
160
|
+
return _compute_phrase_freqs_left_to_right(encoded_posns, trim=trim, max_doc_id=max_doc_id)
|
|
161
|
+
elif shortest_len_index >= len(encoded_posns) - 2:
|
|
162
|
+
return _compute_phrase_freqs_right_to_left(encoded_posns, trim=trim, max_doc_id=max_doc_id)
|
|
163
|
+
else:
|
|
164
|
+
# We optimize this case by going middle-out
|
|
165
|
+
# We can take the min of both directions phrase freqs
|
|
166
|
+
lhs_ids, lhs_counts = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], trim=trim, max_doc_id=max_doc_id)
|
|
167
|
+
rhs_ids, rhs_counts = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], trim=trim, max_doc_id=max_doc_id)
|
|
168
|
+
return _intersect_bigram_matches(lhs_ids, lhs_counts, rhs_ids, rhs_counts)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class PosnBitArrayFromFlatBuilder:
|
|
172
|
+
""" Build from sorted array shape num terms x 3.
|
|
173
|
+
|
|
174
|
+
0th is term id
|
|
175
|
+
1st is doc id
|
|
176
|
+
2nd is posn
|
|
177
|
+
|
|
178
|
+
Sorted by term id then posns
|
|
179
|
+
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
def __init__(self, flat_array: np.ndarray, max_doc_id: int) -> None:
|
|
183
|
+
self.flat_array = flat_array
|
|
184
|
+
self.max_doc_id = max_doc_id
|
|
185
|
+
|
|
186
|
+
def build(self):
|
|
187
|
+
"""Slice the flat array into a 2d array of doc ids and posns."""
|
|
188
|
+
term_boundaries = np.argwhere(np.diff(self.flat_array[0]) > 0).flatten() + 1
|
|
189
|
+
term_boundaries = np.concatenate([[_0],
|
|
190
|
+
term_boundaries.view(np.uint64),
|
|
191
|
+
np.asarray([len(self.flat_array[1])], dtype=np.uint64)])
|
|
192
|
+
term_boundaries = term_boundaries.view(np.uint64)
|
|
193
|
+
|
|
194
|
+
encoded, enc_term_boundaries = encoder.encode(keys=self.flat_array[1].view(np.uint64),
|
|
195
|
+
boundaries=term_boundaries[:-1],
|
|
196
|
+
payload=self.flat_array[2].view(np.uint64))
|
|
197
|
+
if len(encoded) == 0:
|
|
198
|
+
return PosnBitArray(ArrayDict(), self.max_doc_id)
|
|
199
|
+
term_ids = self.flat_array[0][term_boundaries[:-1]]
|
|
200
|
+
|
|
201
|
+
encoded_term_posns = ArrayDict.from_array_with_boundaries(encoded,
|
|
202
|
+
boundaries=enc_term_boundaries,
|
|
203
|
+
ids=term_ids)
|
|
204
|
+
largest_doc_id_with_term = np.max(self.flat_array[1])
|
|
205
|
+
assert self.max_doc_id >= largest_doc_id_with_term
|
|
206
|
+
return PosnBitArray(encoded_term_posns, self.max_doc_id)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
class PosnBitArrayBuilder:
|
|
210
|
+
|
|
211
|
+
def __init__(self):
|
|
212
|
+
self.term_posns = defaultdict(list)
|
|
213
|
+
self.term_posn_doc_ids = defaultdict(list)
|
|
214
|
+
self.max_doc_id = 0
|
|
215
|
+
|
|
216
|
+
def add_posns(self, doc_id: int, term_id: int, posns: List[int]):
|
|
217
|
+
doc_ids = [doc_id] * len(posns)
|
|
218
|
+
self.term_posns[term_id].extend(posns)
|
|
219
|
+
self.term_posn_doc_ids[term_id].extend(doc_ids)
|
|
220
|
+
|
|
221
|
+
def ensure_capacity(self, doc_id):
|
|
222
|
+
self.max_doc_id = max(self.max_doc_id, doc_id)
|
|
223
|
+
|
|
224
|
+
def build(self, check=False):
|
|
225
|
+
encoded_term_posns = ArrayDict()
|
|
226
|
+
for term_id, posns in self.term_posns.items():
|
|
227
|
+
if len(posns) == 0:
|
|
228
|
+
posns = np.asarray([], dtype=np.uint32).flatten()
|
|
229
|
+
elif isinstance(posns, list):
|
|
230
|
+
posns_arr = np.asarray(posns, dtype=np.uint32).flatten()
|
|
231
|
+
posns = posns_arr
|
|
232
|
+
doc_ids = self.term_posn_doc_ids[term_id]
|
|
233
|
+
if isinstance(doc_ids, list):
|
|
234
|
+
doc_ids = np.asarray(doc_ids, dtype=np.uint32)
|
|
235
|
+
encoded = encoder.encode(keys=doc_ids, payload=posns)
|
|
236
|
+
if check:
|
|
237
|
+
decode_again = encoder.decode(encoded)
|
|
238
|
+
docs_to_posns = dict(decode_again)
|
|
239
|
+
doc_ids_again = []
|
|
240
|
+
posns_again = []
|
|
241
|
+
for doc_id, posns_dec in docs_to_posns.items():
|
|
242
|
+
for posn in posns_dec:
|
|
243
|
+
doc_ids_again.append(doc_id)
|
|
244
|
+
posns_again.append(posn)
|
|
245
|
+
assert np.array_equal(doc_ids_again, doc_ids)
|
|
246
|
+
assert np.array_equal(posns, posns_again)
|
|
247
|
+
encoded_term_posns[term_id] = encoded
|
|
248
|
+
|
|
249
|
+
return PosnBitArray(encoded_term_posns, self.max_doc_id)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
class PosnBitArrayAlreadyEncBuilder:
|
|
253
|
+
|
|
254
|
+
def __init__(self):
|
|
255
|
+
self.encoded_term_posns = ArrayDict()
|
|
256
|
+
self.max_doc_id = 0
|
|
257
|
+
|
|
258
|
+
def add_posns(self, doc_id: int, term_id: int, posns):
|
|
259
|
+
self.encoded_term_posns[term_id] = posns
|
|
260
|
+
|
|
261
|
+
def ensure_capacity(self, doc_id):
|
|
262
|
+
self.max_doc_id = max(self.max_doc_id, doc_id)
|
|
263
|
+
|
|
264
|
+
def build(self, check=False):
|
|
265
|
+
return PosnBitArray(self.encoded_term_posns, self.max_doc_id)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def index_range(rng, key):
|
|
269
|
+
if key is None:
|
|
270
|
+
return rng
|
|
271
|
+
if isinstance(rng, np.ndarray):
|
|
272
|
+
return rng[key]
|
|
273
|
+
|
|
274
|
+
if isinstance(key, slice):
|
|
275
|
+
return rng[key]
|
|
276
|
+
elif isinstance(key, numbers.Number):
|
|
277
|
+
return rng[key]
|
|
278
|
+
elif isinstance(key, np.ndarray):
|
|
279
|
+
try:
|
|
280
|
+
# UNFORTUNATE COPY
|
|
281
|
+
r_val = np.asarray(list(rng))[key]
|
|
282
|
+
return r_val
|
|
283
|
+
except IndexError as e:
|
|
284
|
+
raise e
|
|
285
|
+
# Last resort
|
|
286
|
+
# UNFORTUNATE COPY
|
|
287
|
+
# Here probably elipses or a tuple of various things
|
|
288
|
+
return np.asarray(list(rng))[key]
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
class FilteredPosns(abc.Mapping):
|
|
292
|
+
"""When someone slices, this lets us avoid repeating constantly
|
|
293
|
+
slicing the keys out of the encoded positions."""
|
|
294
|
+
|
|
295
|
+
def __init__(self, base, doc_ids):
|
|
296
|
+
self.base = base
|
|
297
|
+
self.sliced = {}
|
|
298
|
+
self.doc_ids = doc_ids
|
|
299
|
+
|
|
300
|
+
def __getitem__(self, key):
|
|
301
|
+
"""Slice with encoder, cache, and return."""
|
|
302
|
+
if key in self.sliced:
|
|
303
|
+
return self.sliced[key]
|
|
304
|
+
|
|
305
|
+
sliced = encoder.slice(self.base[key],
|
|
306
|
+
keys=self.doc_ids)
|
|
307
|
+
self.sliced[key] = sliced
|
|
308
|
+
return sliced
|
|
309
|
+
|
|
310
|
+
def __setitem__(self, key, value):
|
|
311
|
+
self.base[key] = value
|
|
312
|
+
|
|
313
|
+
def __iter__(self):
|
|
314
|
+
return iter(self.doc_ids)
|
|
315
|
+
|
|
316
|
+
def __len__(self):
|
|
317
|
+
return len(self.doc_ids)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
class PosnBitArray:
|
|
321
|
+
|
|
322
|
+
def __init__(self, encoded_term_posns: Union[ArrayDict, FilteredPosns], max_doc_id: int,
|
|
323
|
+
cache_gt_than=25):
|
|
324
|
+
self.encoded_term_posns = encoded_term_posns
|
|
325
|
+
self.max_doc_id = max_doc_id
|
|
326
|
+
self.docfreq_cache : Dict[int, np.uint64] = {}
|
|
327
|
+
self.termfreq_cache : Dict[int, Tuple[np.ndarray, np.ndarray]] = {}
|
|
328
|
+
self.cache_gt_than = cache_gt_than
|
|
329
|
+
|
|
330
|
+
def __repr__(self):
|
|
331
|
+
return f"""PosnBitArray(encoded_term_posns={self.encoded_term_posns}, max_doc_id={self.max_doc_id})"""
|
|
332
|
+
|
|
333
|
+
def memmap(self, data_dir):
|
|
334
|
+
if self.encoded_term_posns:
|
|
335
|
+
self.encoded_term_posns = MemoryMappedArrays(data_dir, self.encoded_term_posns)
|
|
336
|
+
|
|
337
|
+
def warm(self):
|
|
338
|
+
"""Warm tf / df cache of most common terms."""
|
|
339
|
+
for term_id, encoded in self.encoded_term_posns.items():
|
|
340
|
+
if len(encoded) > 255:
|
|
341
|
+
self.docfreq(term_id)
|
|
342
|
+
self.termfreqs(term_id)
|
|
343
|
+
|
|
344
|
+
def filter(self, doc_ids) -> "PosnBitArray":
|
|
345
|
+
"""Filter my doc ids to only those in doc_ids."""
|
|
346
|
+
enc_term_posns = self.encoded_term_posns
|
|
347
|
+
if isinstance(enc_term_posns, FilteredPosns):
|
|
348
|
+
self.encoded_term_posns = enc_term_posns.base
|
|
349
|
+
filtered = FilteredPosns(self.encoded_term_posns, doc_ids)
|
|
350
|
+
new_bit_array = PosnBitArray(filtered, self.max_doc_id)
|
|
351
|
+
return new_bit_array
|
|
352
|
+
|
|
353
|
+
def _reset_filter(self):
|
|
354
|
+
if isinstance(self.encoded_term_posns, FilteredPosns):
|
|
355
|
+
self.encoded_term_posns = self.encoded_term_posns.base
|
|
356
|
+
|
|
357
|
+
def clear_cache(self):
|
|
358
|
+
self.docfreq_cache = {}
|
|
359
|
+
self.termfreq_cache = {}
|
|
360
|
+
self._reset_filter()
|
|
361
|
+
|
|
362
|
+
def copy(self):
|
|
363
|
+
new = PosnBitArray(deepcopy(self.encoded_term_posns), self.max_doc_id)
|
|
364
|
+
return new
|
|
365
|
+
|
|
366
|
+
def concat(self, other):
|
|
367
|
+
"""Merge other into self.
|
|
368
|
+
|
|
369
|
+
Assumes other's doc ids are not overlapping with self's doc ids.
|
|
370
|
+
"""
|
|
371
|
+
if self.encoded_term_posns == {}:
|
|
372
|
+
self.encoded_term_posns = other.encoded_term_posns
|
|
373
|
+
self.max_doc_id = other.max_doc_id
|
|
374
|
+
self.clear_cache()
|
|
375
|
+
return self
|
|
376
|
+
self.encoded_term_posns = ArrayDict.concat(self.encoded_term_posns, other.encoded_term_posns)
|
|
377
|
+
self.max_doc_id = max(self.max_doc_id, other.max_doc_id)
|
|
378
|
+
self.clear_cache()
|
|
379
|
+
|
|
380
|
+
def slice(self, key):
|
|
381
|
+
sliced_term_posns = {}
|
|
382
|
+
doc_ids = convert_keys(key)
|
|
383
|
+
max_doc_id = np.max(doc_ids)
|
|
384
|
+
for term_id, posns in self.encoded_term_posns.items():
|
|
385
|
+
encoded = self.encoded_term_posns[term_id]
|
|
386
|
+
assert len(encoded.shape) == 1
|
|
387
|
+
sliced_term_posns[term_id] = encoder.slice(encoded, keys=doc_ids)
|
|
388
|
+
|
|
389
|
+
return PosnBitArray(sliced_term_posns, max_doc_id)
|
|
390
|
+
|
|
391
|
+
def __getitem__(self, key):
|
|
392
|
+
return self.slice(key)
|
|
393
|
+
|
|
394
|
+
def merge(self, other):
|
|
395
|
+
# Unique terms
|
|
396
|
+
unique_terms = set(self.encoded_term_posns.keys()).union(set(other.encoded_term_posns.keys()))
|
|
397
|
+
|
|
398
|
+
for term_id in unique_terms:
|
|
399
|
+
if term_id not in other.encoded_term_posns:
|
|
400
|
+
continue
|
|
401
|
+
elif term_id not in self.encoded_term_posns:
|
|
402
|
+
self.encoded_term_posns[term_id] = other.encoded_term_posns[term_id]
|
|
403
|
+
else:
|
|
404
|
+
posns_self = self.encoded_term_posns[term_id]
|
|
405
|
+
posns_other = other.encoded_term_posns[term_id]
|
|
406
|
+
self.encoded_term_posns[term_id] = merge(posns_self, posns_other)
|
|
407
|
+
self.max_doc_id = self.max_doc_id + other.max_doc_id
|
|
408
|
+
self.clear_cache()
|
|
409
|
+
|
|
410
|
+
def doc_encoded_posns(self, term_id: int, doc_id: int) -> np.ndarray:
|
|
411
|
+
term_posns = encoder.slice(self.encoded_term_posns[term_id],
|
|
412
|
+
keys=np.asarray([doc_id], dtype=np.uint64))
|
|
413
|
+
return term_posns
|
|
414
|
+
|
|
415
|
+
def empty_buffer(self):
|
|
416
|
+
return np.zeros(int(self.max_doc_id + 1), dtype=np.float32)
|
|
417
|
+
|
|
418
|
+
def phrase_freqs(self, term_ids: List[int],
|
|
419
|
+
slop: int = 0,
|
|
420
|
+
doc_ids: Optional[np.ndarray] = None,
|
|
421
|
+
min_posn: Optional[int] = None,
|
|
422
|
+
max_posn: Optional[int] = None) -> np.ndarray:
|
|
423
|
+
phrase_freqs = self.empty_buffer()
|
|
424
|
+
|
|
425
|
+
if len(term_ids) < 2:
|
|
426
|
+
raise ValueError("Must have at least two terms")
|
|
427
|
+
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
|
|
428
|
+
enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
|
|
429
|
+
else:
|
|
430
|
+
keys = None
|
|
431
|
+
if doc_ids is not None:
|
|
432
|
+
keys = doc_ids.view(np.uint64)
|
|
433
|
+
|
|
434
|
+
enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
|
|
435
|
+
keys=keys,
|
|
436
|
+
min_payload=min_posn,
|
|
437
|
+
max_payload=max_posn) for term_id in term_ids]
|
|
438
|
+
|
|
439
|
+
if slop == 0:
|
|
440
|
+
ids, counts = compute_phrase_freqs(enc_term_posns, max_doc_id=np.uint64(self.max_doc_id))
|
|
441
|
+
phrase_freqs[ids] = counts
|
|
442
|
+
return phrase_freqs
|
|
443
|
+
else:
|
|
444
|
+
ids, counts = span_search(enc_term_posns, slop)
|
|
445
|
+
phrase_freqs[ids] = counts
|
|
446
|
+
return phrase_freqs
|
|
447
|
+
|
|
448
|
+
def positions(self, term_id: int, doc_ids) -> Union[List[np.ndarray], np.ndarray]:
|
|
449
|
+
if isinstance(doc_ids, numbers.Number):
|
|
450
|
+
doc_ids = np.asarray([doc_ids])
|
|
451
|
+
|
|
452
|
+
try:
|
|
453
|
+
np_doc_ids = convert_keys(doc_ids)
|
|
454
|
+
term_posns = encoder.slice(self.encoded_term_posns[term_id],
|
|
455
|
+
keys=np_doc_ids)
|
|
456
|
+
except KeyError:
|
|
457
|
+
r_val = [np.array([], dtype=np.uint32) for doc_id in doc_ids]
|
|
458
|
+
if len(r_val) == 1 and len(doc_ids) == 1 and isinstance(doc_ids[0], numbers.Number):
|
|
459
|
+
return [r_val[0]]
|
|
460
|
+
return r_val
|
|
461
|
+
|
|
462
|
+
decoded = encoder.decode(encoded=term_posns, get_keys=True)
|
|
463
|
+
|
|
464
|
+
if len(decoded) == 0:
|
|
465
|
+
return [np.array([], dtype=np.uint32)]
|
|
466
|
+
if len(decoded) != len(doc_ids):
|
|
467
|
+
# Fill non matches
|
|
468
|
+
decoded = cast(List[Tuple[np.uint64, np.ndarray]], decoded)
|
|
469
|
+
as_dict: Dict[np.uint64, np.ndarray] = dict(decoded)
|
|
470
|
+
decs = []
|
|
471
|
+
for doc_id in doc_ids:
|
|
472
|
+
if doc_id in as_dict:
|
|
473
|
+
decs.append(as_dict[doc_id])
|
|
474
|
+
else:
|
|
475
|
+
decs.append(np.array([], dtype=np.uint32))
|
|
476
|
+
return decs
|
|
477
|
+
else:
|
|
478
|
+
decs = [dec[1] for dec in decoded]
|
|
479
|
+
return decs
|
|
480
|
+
|
|
481
|
+
def termfreqs(self, term_id: int,
|
|
482
|
+
doc_ids: Optional[np.ndarray] = None,
|
|
483
|
+
min_posn: Optional[int] = None,
|
|
484
|
+
max_posn: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray]:
|
|
485
|
+
"""Count term freqs using unique positions."""
|
|
486
|
+
if doc_ids is None and max_posn is None and min_posn is None:
|
|
487
|
+
return self._termfreqs_with_cache(term_id)
|
|
488
|
+
|
|
489
|
+
encoded = self.encoded_term_posns[term_id]
|
|
490
|
+
term_posns = encoded
|
|
491
|
+
term_posns = encoder.slice(encoded,
|
|
492
|
+
keys=doc_ids,
|
|
493
|
+
min_payload=min_posn,
|
|
494
|
+
max_payload=max_posn)
|
|
495
|
+
|
|
496
|
+
return self._computed_term_freqs(term_posns)
|
|
497
|
+
|
|
498
|
+
def _computed_term_freqs(self, term_posns) -> Tuple[np.ndarray, np.ndarray]:
|
|
499
|
+
return encoder.num_values_per_key(term_posns)
|
|
500
|
+
|
|
501
|
+
def _termfreqs_with_cache(self, term_id: int) -> Tuple[np.ndarray, np.ndarray]:
|
|
502
|
+
try:
|
|
503
|
+
return self.termfreq_cache[term_id]
|
|
504
|
+
except KeyError:
|
|
505
|
+
term_posns = self.encoded_term_posns[term_id]
|
|
506
|
+
doc_ids, term_freqs = self._computed_term_freqs(term_posns)
|
|
507
|
+
if self._is_cached(term_id):
|
|
508
|
+
self.termfreq_cache[term_id] = (doc_ids, term_freqs)
|
|
509
|
+
return doc_ids, term_freqs
|
|
510
|
+
|
|
511
|
+
def _is_cached(self, term_id: int) -> bool:
|
|
512
|
+
return term_id in self.docfreq_cache
|
|
513
|
+
|
|
514
|
+
def _docfreq_from_cache(self, term_id: int) -> np.uint64:
|
|
515
|
+
return self.docfreq_cache[term_id]
|
|
516
|
+
|
|
517
|
+
def _maybe_cache_docfreq(self, term_id: int, docfreq: np.uint64):
|
|
518
|
+
if len(self.encoded_term_posns[term_id]) > self.cache_gt_than:
|
|
519
|
+
self.docfreq_cache[term_id] = docfreq
|
|
520
|
+
|
|
521
|
+
def docfreq(self, term_id: int) -> np.uint64:
|
|
522
|
+
try:
|
|
523
|
+
return self.docfreq_cache[term_id]
|
|
524
|
+
except KeyError:
|
|
525
|
+
encoded = self.encoded_term_posns[term_id]
|
|
526
|
+
docfreq = np.uint64(encoder.keys_unique(encoded).size)
|
|
527
|
+
self._maybe_cache_docfreq(term_id, docfreq)
|
|
528
|
+
return docfreq
|
|
529
|
+
|
|
530
|
+
def insert(self, key, term_ids_to_posns, is_encoded=False):
|
|
531
|
+
new_posns = PosnBitArrayBuilder()
|
|
532
|
+
if is_encoded:
|
|
533
|
+
new_posns = PosnBitArrayAlreadyEncBuilder()
|
|
534
|
+
max_doc_id = 0
|
|
535
|
+
for doc_id, new_posns_row in enumerate(term_ids_to_posns):
|
|
536
|
+
for term_id, positions in new_posns_row:
|
|
537
|
+
new_posns.add_posns(doc_id, term_id, positions)
|
|
538
|
+
max_doc_id = max(doc_id, max_doc_id)
|
|
539
|
+
new_posns.max_doc_id = max_doc_id
|
|
540
|
+
ins_arr = new_posns.build()
|
|
541
|
+
self.merge(ins_arr)
|
|
542
|
+
|
|
543
|
+
@property
|
|
544
|
+
def nbytes(self):
|
|
545
|
+
arr_bytes = 0
|
|
546
|
+
for doc_id, posn in self.encoded_term_posns.items():
|
|
547
|
+
arr_bytes += posn.nbytes
|
|
548
|
+
for term_id, (doc_ids, term_freqs) in self.termfreq_cache.items():
|
|
549
|
+
arr_bytes += doc_ids.nbytes
|
|
550
|
+
arr_bytes += term_freqs.nbytes
|
|
551
|
+
for term_id, docfreq in self.docfreq_cache.items():
|
|
552
|
+
arr_bytes += docfreq.nbytes
|
|
553
|
+
return arr_bytes
|