searcharray 0.0.73__cp313-cp313-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. searcharray/__init__.py +2 -0
  2. searcharray/bm25/__init__.py +1 -0
  3. searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
  4. searcharray/bm25/bm25.pyi +11 -0
  5. searcharray/bm25/bm25.pyx +41 -0
  6. searcharray/indexing.py +342 -0
  7. searcharray/phrase/__init__.py +0 -0
  8. searcharray/phrase/bigram_freqs.py +307 -0
  9. searcharray/phrase/memmap_arrays.py +208 -0
  10. searcharray/phrase/middle_out.py +553 -0
  11. searcharray/phrase/spans.py +187 -0
  12. searcharray/postings.py +708 -0
  13. searcharray/roaringish/__init__.py +7 -0
  14. searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
  15. searcharray/roaringish/intersect.pyi +25 -0
  16. searcharray/roaringish/intersect.pyx +390 -0
  17. searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
  18. searcharray/roaringish/merge.pyi +16 -0
  19. searcharray/roaringish/merge.pyx +232 -0
  20. searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
  21. searcharray/roaringish/popcount.pyi +36 -0
  22. searcharray/roaringish/popcount.pyx +278 -0
  23. searcharray/roaringish/roaringish.py +298 -0
  24. searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
  25. searcharray/roaringish/roaringish_ops.pyx +98 -0
  26. searcharray/roaringish/scatter_assign.h +31 -0
  27. searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
  28. searcharray/roaringish/search.pyi +25 -0
  29. searcharray/roaringish/search.pyx +140 -0
  30. searcharray/roaringish/snp_ops.pxd +13 -0
  31. searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
  32. searcharray/roaringish/spans.pyx +330 -0
  33. searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
  34. searcharray/roaringish/unique.pyi +6 -0
  35. searcharray/roaringish/unique.pyx +145 -0
  36. searcharray/similarity.py +92 -0
  37. searcharray/solr.py +355 -0
  38. searcharray/term_dict.py +59 -0
  39. searcharray/utils/__init__.py +0 -0
  40. searcharray/utils/bitcount.py +34 -0
  41. searcharray/utils/mat_set.py +170 -0
  42. searcharray/utils/row_viewable_matrix.py +91 -0
  43. searcharray/utils/sort.py +45 -0
  44. searcharray-0.0.73.dist-info/METADATA +245 -0
  45. searcharray-0.0.73.dist-info/RECORD +48 -0
  46. searcharray-0.0.73.dist-info/WHEEL +5 -0
  47. searcharray-0.0.73.dist-info/entry_points.txt +2 -0
  48. searcharray-0.0.73.dist-info/top_level.txt +1 -0
@@ -0,0 +1,553 @@
1
+ """Encode positions in bits along with some neighboring information for wrapping.
2
+
3
+ See this notebook for motivation:
4
+
5
+ https://colab.research.google.com/drive/10tIEkdlCE_1J_CcgEcV0jkLfBc-0H4am?authuser=1#scrollTo=XWzy-n9dF3PG
6
+
7
+ """
8
+ import numpy as np
9
+ from copy import deepcopy
10
+ from typing import List, Tuple, Dict, Union, cast, Optional
11
+ from searcharray.roaringish import RoaringishEncoder, convert_keys, merge, intersect
12
+ from searcharray.phrase.memmap_arrays import MemoryMappedArrays, ArrayDict
13
+ from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
14
+ from searcharray.phrase.spans import span_search
15
+ import numbers
16
+ import logging
17
+ from collections import defaultdict, abc
18
+
19
+
20
+ logger = logging.getLogger(__name__)
21
+
22
+ # When running in pytest
23
+ import sys # noqa
24
+ handler = logging.StreamHandler(sys.stdout)
25
+ handler.setLevel(logging.ERROR)
26
+ formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
27
+ handler.setFormatter(formatter)
28
+ logger.addHandler(handler)
29
+ logger.setLevel(logging.ERROR)
30
+
31
+
32
+ encoder = RoaringishEncoder()
33
+
34
+ # To not constantly type coerce
35
+ _64 = np.uint64(64)
36
+ _2 = np.uint64(2)
37
+ _1 = np.uint64(1)
38
+ _0 = np.uint64(0)
39
+ _neg1 = np.int64(-1)
40
+
41
+ MAX_POSN = encoder.max_payload
42
+
43
+
44
+ def trim_phrase_search(encoded_posns: List[np.ndarray]) -> List[np.ndarray]:
45
+ """Trim long phrases by searching the rarest terms first."""
46
+
47
+ # Start with rarest term
48
+ shortest_keys = None
49
+ shortest_idx = None
50
+ min_len = 1e100
51
+ max_len = 0
52
+ for idx, enc_posn in enumerate(encoded_posns):
53
+ if len(enc_posn) < min_len:
54
+ shortest_keys = encoder.keys(enc_posn)
55
+ shortest_idx = idx
56
+ min_len = len(enc_posn)
57
+ if len(enc_posn) > max_len:
58
+ max_len = len(enc_posn)
59
+
60
+ if shortest_keys is None:
61
+ return encoded_posns
62
+
63
+ for enc_posn_idx in range(len(encoded_posns)):
64
+ if enc_posn_idx == shortest_idx:
65
+ continue
66
+ if len(encoded_posns[enc_posn_idx]) > (20 * min_len):
67
+ encoded_posns[enc_posn_idx] = encoder.slice(encoded_posns[enc_posn_idx],
68
+ shortest_keys)
69
+
70
+ return encoded_posns
71
+
72
+
73
+ def _intersect_bigram_matches(ids: Optional[np.ndarray],
74
+ counts: Optional[np.ndarray],
75
+ new_ids: np.ndarray,
76
+ new_counts: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
77
+ """Intersect bigram matches."""
78
+ if ids is None or counts is None:
79
+ return new_ids, new_counts
80
+ else:
81
+ # Intersect (Are these sorted?)
82
+ is_sorted = np.all(np.diff(ids) >= 0)
83
+ assert is_sorted
84
+ is_sorted = np.all(np.diff(new_ids) >= 0)
85
+ assert is_sorted
86
+
87
+ ids_idx, new_ids_idx = intersect(ids, new_ids)
88
+ intersected_counts = counts[ids_idx]
89
+ counts = np.minimum(intersected_counts, new_counts[new_ids_idx])
90
+ ids = ids[ids_idx]
91
+ assert ids is not None
92
+ assert counts is not None
93
+ return ids, counts
94
+
95
+
96
+ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
97
+ max_doc_id: np.uint64 = _0,
98
+ trim: bool = True) -> Tuple[np.ndarray, np.ndarray]:
99
+ """Compute phrase freqs from a set of encoded positions."""
100
+ if len(encoded_posns) < 2:
101
+ raise ValueError("phrase must have at least two terms")
102
+
103
+ # Trim long phrases by searching the rarest terms first
104
+ if trim and len(encoded_posns) > 3:
105
+ encoded_posns = trim_phrase_search(encoded_posns)
106
+
107
+ ids = None
108
+ counts = None
109
+
110
+ lhs = encoded_posns[0]
111
+ for rhs in encoded_posns[1:]:
112
+ # Only count the count of the last bigram (ignoring the ones where priors did not match)
113
+ phrase_freqs, conts = bigram_freqs(lhs, rhs,
114
+ cont=Continuation.RHS)
115
+ assert conts[1] is not None
116
+ lhs = conts[1]
117
+ ids, counts = _intersect_bigram_matches(ids, counts, phrase_freqs[0], phrase_freqs[1])
118
+
119
+ if ids is None or counts is None:
120
+ return np.array([], dtype=np.uint64), np.array([], dtype=np.float32)
121
+
122
+ return (ids, counts)
123
+
124
+
125
+ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
126
+ max_doc_id: np.uint64 = _0,
127
+ trim: bool = True) -> Tuple[np.ndarray, np.ndarray]:
128
+ """Compute phrase freqs from a set of encoded positions."""
129
+ if len(encoded_posns) < 2:
130
+ raise ValueError("phrase must have at least two terms")
131
+
132
+ # Trim long phrases by searching the rarest terms first
133
+ if trim and len(encoded_posns) > 3:
134
+ encoded_posns = trim_phrase_search(encoded_posns)
135
+
136
+ ids = None
137
+ counts = None
138
+
139
+ rhs = encoded_posns[-1]
140
+ for lhs in encoded_posns[-2::-1]:
141
+ # Only count the count of the last bigram (ignoring the ones where priors did not match)
142
+ phrase_freqs, conts = bigram_freqs(lhs, rhs,
143
+ cont=Continuation.LHS)
144
+
145
+ assert conts[0] is not None
146
+ rhs = conts[0]
147
+ ids, counts = _intersect_bigram_matches(ids, counts, phrase_freqs[0], phrase_freqs[1])
148
+
149
+ if ids is None or counts is None:
150
+ return np.array([], dtype=np.uint64), np.array([], dtype=np.float32)
151
+ return (ids, counts)
152
+
153
+
154
+ def compute_phrase_freqs(encoded_posns: List[np.ndarray],
155
+ max_doc_id: np.uint64 = _0,
156
+ trim: bool = False) -> Tuple[np.ndarray, np.ndarray]:
157
+ """Compute phrase freqs from a set of encoded positions."""
158
+ shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
159
+ if shortest_len_index <= 1:
160
+ return _compute_phrase_freqs_left_to_right(encoded_posns, trim=trim, max_doc_id=max_doc_id)
161
+ elif shortest_len_index >= len(encoded_posns) - 2:
162
+ return _compute_phrase_freqs_right_to_left(encoded_posns, trim=trim, max_doc_id=max_doc_id)
163
+ else:
164
+ # We optimize this case by going middle-out
165
+ # We can take the min of both directions phrase freqs
166
+ lhs_ids, lhs_counts = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], trim=trim, max_doc_id=max_doc_id)
167
+ rhs_ids, rhs_counts = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], trim=trim, max_doc_id=max_doc_id)
168
+ return _intersect_bigram_matches(lhs_ids, lhs_counts, rhs_ids, rhs_counts)
169
+
170
+
171
+ class PosnBitArrayFromFlatBuilder:
172
+ """ Build from sorted array shape num terms x 3.
173
+
174
+ 0th is term id
175
+ 1st is doc id
176
+ 2nd is posn
177
+
178
+ Sorted by term id then posns
179
+
180
+ """
181
+
182
+ def __init__(self, flat_array: np.ndarray, max_doc_id: int) -> None:
183
+ self.flat_array = flat_array
184
+ self.max_doc_id = max_doc_id
185
+
186
+ def build(self):
187
+ """Slice the flat array into a 2d array of doc ids and posns."""
188
+ term_boundaries = np.argwhere(np.diff(self.flat_array[0]) > 0).flatten() + 1
189
+ term_boundaries = np.concatenate([[_0],
190
+ term_boundaries.view(np.uint64),
191
+ np.asarray([len(self.flat_array[1])], dtype=np.uint64)])
192
+ term_boundaries = term_boundaries.view(np.uint64)
193
+
194
+ encoded, enc_term_boundaries = encoder.encode(keys=self.flat_array[1].view(np.uint64),
195
+ boundaries=term_boundaries[:-1],
196
+ payload=self.flat_array[2].view(np.uint64))
197
+ if len(encoded) == 0:
198
+ return PosnBitArray(ArrayDict(), self.max_doc_id)
199
+ term_ids = self.flat_array[0][term_boundaries[:-1]]
200
+
201
+ encoded_term_posns = ArrayDict.from_array_with_boundaries(encoded,
202
+ boundaries=enc_term_boundaries,
203
+ ids=term_ids)
204
+ largest_doc_id_with_term = np.max(self.flat_array[1])
205
+ assert self.max_doc_id >= largest_doc_id_with_term
206
+ return PosnBitArray(encoded_term_posns, self.max_doc_id)
207
+
208
+
209
+ class PosnBitArrayBuilder:
210
+
211
+ def __init__(self):
212
+ self.term_posns = defaultdict(list)
213
+ self.term_posn_doc_ids = defaultdict(list)
214
+ self.max_doc_id = 0
215
+
216
+ def add_posns(self, doc_id: int, term_id: int, posns: List[int]):
217
+ doc_ids = [doc_id] * len(posns)
218
+ self.term_posns[term_id].extend(posns)
219
+ self.term_posn_doc_ids[term_id].extend(doc_ids)
220
+
221
+ def ensure_capacity(self, doc_id):
222
+ self.max_doc_id = max(self.max_doc_id, doc_id)
223
+
224
+ def build(self, check=False):
225
+ encoded_term_posns = ArrayDict()
226
+ for term_id, posns in self.term_posns.items():
227
+ if len(posns) == 0:
228
+ posns = np.asarray([], dtype=np.uint32).flatten()
229
+ elif isinstance(posns, list):
230
+ posns_arr = np.asarray(posns, dtype=np.uint32).flatten()
231
+ posns = posns_arr
232
+ doc_ids = self.term_posn_doc_ids[term_id]
233
+ if isinstance(doc_ids, list):
234
+ doc_ids = np.asarray(doc_ids, dtype=np.uint32)
235
+ encoded = encoder.encode(keys=doc_ids, payload=posns)
236
+ if check:
237
+ decode_again = encoder.decode(encoded)
238
+ docs_to_posns = dict(decode_again)
239
+ doc_ids_again = []
240
+ posns_again = []
241
+ for doc_id, posns_dec in docs_to_posns.items():
242
+ for posn in posns_dec:
243
+ doc_ids_again.append(doc_id)
244
+ posns_again.append(posn)
245
+ assert np.array_equal(doc_ids_again, doc_ids)
246
+ assert np.array_equal(posns, posns_again)
247
+ encoded_term_posns[term_id] = encoded
248
+
249
+ return PosnBitArray(encoded_term_posns, self.max_doc_id)
250
+
251
+
252
+ class PosnBitArrayAlreadyEncBuilder:
253
+
254
+ def __init__(self):
255
+ self.encoded_term_posns = ArrayDict()
256
+ self.max_doc_id = 0
257
+
258
+ def add_posns(self, doc_id: int, term_id: int, posns):
259
+ self.encoded_term_posns[term_id] = posns
260
+
261
+ def ensure_capacity(self, doc_id):
262
+ self.max_doc_id = max(self.max_doc_id, doc_id)
263
+
264
+ def build(self, check=False):
265
+ return PosnBitArray(self.encoded_term_posns, self.max_doc_id)
266
+
267
+
268
+ def index_range(rng, key):
269
+ if key is None:
270
+ return rng
271
+ if isinstance(rng, np.ndarray):
272
+ return rng[key]
273
+
274
+ if isinstance(key, slice):
275
+ return rng[key]
276
+ elif isinstance(key, numbers.Number):
277
+ return rng[key]
278
+ elif isinstance(key, np.ndarray):
279
+ try:
280
+ # UNFORTUNATE COPY
281
+ r_val = np.asarray(list(rng))[key]
282
+ return r_val
283
+ except IndexError as e:
284
+ raise e
285
+ # Last resort
286
+ # UNFORTUNATE COPY
287
+ # Here probably elipses or a tuple of various things
288
+ return np.asarray(list(rng))[key]
289
+
290
+
291
+ class FilteredPosns(abc.Mapping):
292
+ """When someone slices, this lets us avoid repeating constantly
293
+ slicing the keys out of the encoded positions."""
294
+
295
+ def __init__(self, base, doc_ids):
296
+ self.base = base
297
+ self.sliced = {}
298
+ self.doc_ids = doc_ids
299
+
300
+ def __getitem__(self, key):
301
+ """Slice with encoder, cache, and return."""
302
+ if key in self.sliced:
303
+ return self.sliced[key]
304
+
305
+ sliced = encoder.slice(self.base[key],
306
+ keys=self.doc_ids)
307
+ self.sliced[key] = sliced
308
+ return sliced
309
+
310
+ def __setitem__(self, key, value):
311
+ self.base[key] = value
312
+
313
+ def __iter__(self):
314
+ return iter(self.doc_ids)
315
+
316
+ def __len__(self):
317
+ return len(self.doc_ids)
318
+
319
+
320
+ class PosnBitArray:
321
+
322
+ def __init__(self, encoded_term_posns: Union[ArrayDict, FilteredPosns], max_doc_id: int,
323
+ cache_gt_than=25):
324
+ self.encoded_term_posns = encoded_term_posns
325
+ self.max_doc_id = max_doc_id
326
+ self.docfreq_cache : Dict[int, np.uint64] = {}
327
+ self.termfreq_cache : Dict[int, Tuple[np.ndarray, np.ndarray]] = {}
328
+ self.cache_gt_than = cache_gt_than
329
+
330
+ def __repr__(self):
331
+ return f"""PosnBitArray(encoded_term_posns={self.encoded_term_posns}, max_doc_id={self.max_doc_id})"""
332
+
333
+ def memmap(self, data_dir):
334
+ if self.encoded_term_posns:
335
+ self.encoded_term_posns = MemoryMappedArrays(data_dir, self.encoded_term_posns)
336
+
337
+ def warm(self):
338
+ """Warm tf / df cache of most common terms."""
339
+ for term_id, encoded in self.encoded_term_posns.items():
340
+ if len(encoded) > 255:
341
+ self.docfreq(term_id)
342
+ self.termfreqs(term_id)
343
+
344
+ def filter(self, doc_ids) -> "PosnBitArray":
345
+ """Filter my doc ids to only those in doc_ids."""
346
+ enc_term_posns = self.encoded_term_posns
347
+ if isinstance(enc_term_posns, FilteredPosns):
348
+ self.encoded_term_posns = enc_term_posns.base
349
+ filtered = FilteredPosns(self.encoded_term_posns, doc_ids)
350
+ new_bit_array = PosnBitArray(filtered, self.max_doc_id)
351
+ return new_bit_array
352
+
353
+ def _reset_filter(self):
354
+ if isinstance(self.encoded_term_posns, FilteredPosns):
355
+ self.encoded_term_posns = self.encoded_term_posns.base
356
+
357
+ def clear_cache(self):
358
+ self.docfreq_cache = {}
359
+ self.termfreq_cache = {}
360
+ self._reset_filter()
361
+
362
+ def copy(self):
363
+ new = PosnBitArray(deepcopy(self.encoded_term_posns), self.max_doc_id)
364
+ return new
365
+
366
+ def concat(self, other):
367
+ """Merge other into self.
368
+
369
+ Assumes other's doc ids are not overlapping with self's doc ids.
370
+ """
371
+ if self.encoded_term_posns == {}:
372
+ self.encoded_term_posns = other.encoded_term_posns
373
+ self.max_doc_id = other.max_doc_id
374
+ self.clear_cache()
375
+ return self
376
+ self.encoded_term_posns = ArrayDict.concat(self.encoded_term_posns, other.encoded_term_posns)
377
+ self.max_doc_id = max(self.max_doc_id, other.max_doc_id)
378
+ self.clear_cache()
379
+
380
+ def slice(self, key):
381
+ sliced_term_posns = {}
382
+ doc_ids = convert_keys(key)
383
+ max_doc_id = np.max(doc_ids)
384
+ for term_id, posns in self.encoded_term_posns.items():
385
+ encoded = self.encoded_term_posns[term_id]
386
+ assert len(encoded.shape) == 1
387
+ sliced_term_posns[term_id] = encoder.slice(encoded, keys=doc_ids)
388
+
389
+ return PosnBitArray(sliced_term_posns, max_doc_id)
390
+
391
+ def __getitem__(self, key):
392
+ return self.slice(key)
393
+
394
+ def merge(self, other):
395
+ # Unique terms
396
+ unique_terms = set(self.encoded_term_posns.keys()).union(set(other.encoded_term_posns.keys()))
397
+
398
+ for term_id in unique_terms:
399
+ if term_id not in other.encoded_term_posns:
400
+ continue
401
+ elif term_id not in self.encoded_term_posns:
402
+ self.encoded_term_posns[term_id] = other.encoded_term_posns[term_id]
403
+ else:
404
+ posns_self = self.encoded_term_posns[term_id]
405
+ posns_other = other.encoded_term_posns[term_id]
406
+ self.encoded_term_posns[term_id] = merge(posns_self, posns_other)
407
+ self.max_doc_id = self.max_doc_id + other.max_doc_id
408
+ self.clear_cache()
409
+
410
+ def doc_encoded_posns(self, term_id: int, doc_id: int) -> np.ndarray:
411
+ term_posns = encoder.slice(self.encoded_term_posns[term_id],
412
+ keys=np.asarray([doc_id], dtype=np.uint64))
413
+ return term_posns
414
+
415
+ def empty_buffer(self):
416
+ return np.zeros(int(self.max_doc_id + 1), dtype=np.float32)
417
+
418
+ def phrase_freqs(self, term_ids: List[int],
419
+ slop: int = 0,
420
+ doc_ids: Optional[np.ndarray] = None,
421
+ min_posn: Optional[int] = None,
422
+ max_posn: Optional[int] = None) -> np.ndarray:
423
+ phrase_freqs = self.empty_buffer()
424
+
425
+ if len(term_ids) < 2:
426
+ raise ValueError("Must have at least two terms")
427
+ if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
428
+ enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
429
+ else:
430
+ keys = None
431
+ if doc_ids is not None:
432
+ keys = doc_ids.view(np.uint64)
433
+
434
+ enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
435
+ keys=keys,
436
+ min_payload=min_posn,
437
+ max_payload=max_posn) for term_id in term_ids]
438
+
439
+ if slop == 0:
440
+ ids, counts = compute_phrase_freqs(enc_term_posns, max_doc_id=np.uint64(self.max_doc_id))
441
+ phrase_freqs[ids] = counts
442
+ return phrase_freqs
443
+ else:
444
+ ids, counts = span_search(enc_term_posns, slop)
445
+ phrase_freqs[ids] = counts
446
+ return phrase_freqs
447
+
448
+ def positions(self, term_id: int, doc_ids) -> Union[List[np.ndarray], np.ndarray]:
449
+ if isinstance(doc_ids, numbers.Number):
450
+ doc_ids = np.asarray([doc_ids])
451
+
452
+ try:
453
+ np_doc_ids = convert_keys(doc_ids)
454
+ term_posns = encoder.slice(self.encoded_term_posns[term_id],
455
+ keys=np_doc_ids)
456
+ except KeyError:
457
+ r_val = [np.array([], dtype=np.uint32) for doc_id in doc_ids]
458
+ if len(r_val) == 1 and len(doc_ids) == 1 and isinstance(doc_ids[0], numbers.Number):
459
+ return [r_val[0]]
460
+ return r_val
461
+
462
+ decoded = encoder.decode(encoded=term_posns, get_keys=True)
463
+
464
+ if len(decoded) == 0:
465
+ return [np.array([], dtype=np.uint32)]
466
+ if len(decoded) != len(doc_ids):
467
+ # Fill non matches
468
+ decoded = cast(List[Tuple[np.uint64, np.ndarray]], decoded)
469
+ as_dict: Dict[np.uint64, np.ndarray] = dict(decoded)
470
+ decs = []
471
+ for doc_id in doc_ids:
472
+ if doc_id in as_dict:
473
+ decs.append(as_dict[doc_id])
474
+ else:
475
+ decs.append(np.array([], dtype=np.uint32))
476
+ return decs
477
+ else:
478
+ decs = [dec[1] for dec in decoded]
479
+ return decs
480
+
481
+ def termfreqs(self, term_id: int,
482
+ doc_ids: Optional[np.ndarray] = None,
483
+ min_posn: Optional[int] = None,
484
+ max_posn: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray]:
485
+ """Count term freqs using unique positions."""
486
+ if doc_ids is None and max_posn is None and min_posn is None:
487
+ return self._termfreqs_with_cache(term_id)
488
+
489
+ encoded = self.encoded_term_posns[term_id]
490
+ term_posns = encoded
491
+ term_posns = encoder.slice(encoded,
492
+ keys=doc_ids,
493
+ min_payload=min_posn,
494
+ max_payload=max_posn)
495
+
496
+ return self._computed_term_freqs(term_posns)
497
+
498
+ def _computed_term_freqs(self, term_posns) -> Tuple[np.ndarray, np.ndarray]:
499
+ return encoder.num_values_per_key(term_posns)
500
+
501
+ def _termfreqs_with_cache(self, term_id: int) -> Tuple[np.ndarray, np.ndarray]:
502
+ try:
503
+ return self.termfreq_cache[term_id]
504
+ except KeyError:
505
+ term_posns = self.encoded_term_posns[term_id]
506
+ doc_ids, term_freqs = self._computed_term_freqs(term_posns)
507
+ if self._is_cached(term_id):
508
+ self.termfreq_cache[term_id] = (doc_ids, term_freqs)
509
+ return doc_ids, term_freqs
510
+
511
+ def _is_cached(self, term_id: int) -> bool:
512
+ return term_id in self.docfreq_cache
513
+
514
+ def _docfreq_from_cache(self, term_id: int) -> np.uint64:
515
+ return self.docfreq_cache[term_id]
516
+
517
+ def _maybe_cache_docfreq(self, term_id: int, docfreq: np.uint64):
518
+ if len(self.encoded_term_posns[term_id]) > self.cache_gt_than:
519
+ self.docfreq_cache[term_id] = docfreq
520
+
521
+ def docfreq(self, term_id: int) -> np.uint64:
522
+ try:
523
+ return self.docfreq_cache[term_id]
524
+ except KeyError:
525
+ encoded = self.encoded_term_posns[term_id]
526
+ docfreq = np.uint64(encoder.keys_unique(encoded).size)
527
+ self._maybe_cache_docfreq(term_id, docfreq)
528
+ return docfreq
529
+
530
+ def insert(self, key, term_ids_to_posns, is_encoded=False):
531
+ new_posns = PosnBitArrayBuilder()
532
+ if is_encoded:
533
+ new_posns = PosnBitArrayAlreadyEncBuilder()
534
+ max_doc_id = 0
535
+ for doc_id, new_posns_row in enumerate(term_ids_to_posns):
536
+ for term_id, positions in new_posns_row:
537
+ new_posns.add_posns(doc_id, term_id, positions)
538
+ max_doc_id = max(doc_id, max_doc_id)
539
+ new_posns.max_doc_id = max_doc_id
540
+ ins_arr = new_posns.build()
541
+ self.merge(ins_arr)
542
+
543
+ @property
544
+ def nbytes(self):
545
+ arr_bytes = 0
546
+ for doc_id, posn in self.encoded_term_posns.items():
547
+ arr_bytes += posn.nbytes
548
+ for term_id, (doc_ids, term_freqs) in self.termfreq_cache.items():
549
+ arr_bytes += doc_ids.nbytes
550
+ arr_bytes += term_freqs.nbytes
551
+ for term_id, docfreq in self.docfreq_cache.items():
552
+ arr_bytes += docfreq.nbytes
553
+ return arr_bytes