searcharray 0.0.37__tar.gz → 0.0.39__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.37 → searcharray-0.0.39}/PKG-INFO +1 -1
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/phrase/middle_out.py +74 -27
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/postings.py +59 -16
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/similarity.py +16 -2
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/utils/bitcount.py +9 -5
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/SOURCES.txt +1 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/setup.py +1 -1
- searcharray-0.0.39/test/test_bitcount64.py +53 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/test/test_msmarco.py +57 -3
- {searcharray-0.0.37 → searcharray-0.0.39}/test/test_search.py +21 -5
- {searcharray-0.0.37 → searcharray-0.0.39}/test/test_tmdb.py +9 -2
- {searcharray-0.0.37 → searcharray-0.0.39}/README.md +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/pyproject.toml +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/__init__.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/indexing.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/solr.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/utils/roaringish.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/setup.cfg +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/test/test_extension_array.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/test/test_solr.py +0 -0
- {searcharray-0.0.37 → searcharray-0.0.39}/test/test_utils.py +0 -0
|
@@ -9,7 +9,7 @@ import numpy as np
|
|
|
9
9
|
import sortednp as snp
|
|
10
10
|
from copy import deepcopy
|
|
11
11
|
from typing import List, Tuple, Dict, Union, cast, Optional
|
|
12
|
-
from searcharray.utils.roaringish import RoaringishEncoder, convert_keys
|
|
12
|
+
from searcharray.utils.roaringish import RoaringishEncoder, convert_keys, sorted_unique
|
|
13
13
|
import numbers
|
|
14
14
|
import logging
|
|
15
15
|
from collections import defaultdict
|
|
@@ -203,7 +203,7 @@ class PosnBitArrayFromFlatBuilder:
|
|
|
203
203
|
sliced = encoded[beg_idx:end_idx]
|
|
204
204
|
encoded_term_posns[term_ids[into_terms]] = sliced
|
|
205
205
|
|
|
206
|
-
return PosnBitArray(encoded_term_posns,
|
|
206
|
+
return PosnBitArray(encoded_term_posns, self.flat_array[1].max())
|
|
207
207
|
|
|
208
208
|
|
|
209
209
|
class PosnBitArrayBuilder:
|
|
@@ -246,7 +246,7 @@ class PosnBitArrayBuilder:
|
|
|
246
246
|
assert np.array_equal(posns, posns_again)
|
|
247
247
|
encoded_term_posns[term_id] = encoded
|
|
248
248
|
|
|
249
|
-
return PosnBitArray(encoded_term_posns,
|
|
249
|
+
return PosnBitArray(encoded_term_posns, self.max_doc_id)
|
|
250
250
|
|
|
251
251
|
|
|
252
252
|
class PosnBitArrayAlreadyEncBuilder:
|
|
@@ -262,7 +262,7 @@ class PosnBitArrayAlreadyEncBuilder:
|
|
|
262
262
|
self.max_doc_id = max(self.max_doc_id, doc_id)
|
|
263
263
|
|
|
264
264
|
def build(self, check=False):
|
|
265
|
-
return PosnBitArray(self.encoded_term_posns,
|
|
265
|
+
return PosnBitArray(self.encoded_term_posns, self.max_doc_id)
|
|
266
266
|
|
|
267
267
|
|
|
268
268
|
def index_range(rng, key):
|
|
@@ -290,13 +290,25 @@ def index_range(rng, key):
|
|
|
290
290
|
|
|
291
291
|
class PosnBitArray:
|
|
292
292
|
|
|
293
|
-
def __init__(self, encoded_term_posns,
|
|
293
|
+
def __init__(self, encoded_term_posns, max_doc_id: int):
|
|
294
294
|
self.encoded_term_posns = encoded_term_posns
|
|
295
|
-
self.
|
|
295
|
+
self.max_doc_id = max_doc_id
|
|
296
|
+
self.docfreq_cache : Dict[int, np.uint64] = {}
|
|
297
|
+
self.termfreq_cache : Dict[int, Tuple[np.ndarray, np.ndarray]] = {}
|
|
298
|
+
|
|
299
|
+
def warm(self):
|
|
300
|
+
"""Warm tf / df cache of most common terms."""
|
|
301
|
+
for term_id, encoded in self.encoded_term_posns.items():
|
|
302
|
+
if len(encoded) > 255:
|
|
303
|
+
self.docfreq(term_id)
|
|
304
|
+
self.termfreqs(term_id)
|
|
305
|
+
|
|
306
|
+
def clear_cache(self):
|
|
307
|
+
self.docfreq_cache = {}
|
|
308
|
+
self.termfreq_cache = {}
|
|
296
309
|
|
|
297
310
|
def copy(self):
|
|
298
|
-
new = PosnBitArray(deepcopy(self.encoded_term_posns),
|
|
299
|
-
self.doc_ids)
|
|
311
|
+
new = PosnBitArray(deepcopy(self.encoded_term_posns), self.max_doc_id)
|
|
300
312
|
return new
|
|
301
313
|
|
|
302
314
|
def concat(self, other):
|
|
@@ -314,17 +326,19 @@ class PosnBitArray:
|
|
|
314
326
|
only_other_terms = set(other.encoded_term_posns.keys()).difference(set(self.encoded_term_posns.keys()))
|
|
315
327
|
for term_id in only_other_terms:
|
|
316
328
|
self.encoded_term_posns[term_id] = other.encoded_term_posns[term_id]
|
|
329
|
+
self.max_doc_id = max(self.max_doc_id, other.max_doc_id)
|
|
330
|
+
self.clear_cache()
|
|
317
331
|
|
|
318
332
|
def slice(self, key):
|
|
319
333
|
sliced_term_posns = {}
|
|
320
|
-
doc_ids =
|
|
321
|
-
|
|
334
|
+
doc_ids = convert_keys(key)
|
|
335
|
+
max_doc_id = np.max(doc_ids)
|
|
322
336
|
for term_id, posns in self.encoded_term_posns.items():
|
|
323
337
|
encoded = self.encoded_term_posns[term_id]
|
|
324
338
|
assert len(encoded.shape) == 1
|
|
325
|
-
sliced_term_posns[term_id] = encoder.slice(encoded, keys=
|
|
339
|
+
sliced_term_posns[term_id] = encoder.slice(encoded, keys=doc_ids)
|
|
326
340
|
|
|
327
|
-
return PosnBitArray(sliced_term_posns,
|
|
341
|
+
return PosnBitArray(sliced_term_posns, max_doc_id)
|
|
328
342
|
|
|
329
343
|
def __getitem__(self, key):
|
|
330
344
|
return self.slice(key)
|
|
@@ -342,6 +356,8 @@ class PosnBitArray:
|
|
|
342
356
|
posns_self = self.encoded_term_posns[term_id]
|
|
343
357
|
posns_other = other.encoded_term_posns[term_id]
|
|
344
358
|
self.encoded_term_posns[term_id] = snp.merge(posns_self, posns_other)
|
|
359
|
+
self.max_doc_id = self.max_doc_id + other.max_doc_id
|
|
360
|
+
self.clear_cache()
|
|
345
361
|
|
|
346
362
|
def doc_encoded_posns(self, term_id: int, doc_id: int) -> np.ndarray:
|
|
347
363
|
term_posns = encoder.slice(self.encoded_term_posns[term_id],
|
|
@@ -352,16 +368,14 @@ class PosnBitArray:
|
|
|
352
368
|
doc_ids: np.ndarray) -> np.ndarray:
|
|
353
369
|
if len(term_ids) < 2:
|
|
354
370
|
raise ValueError("Must have at least two terms")
|
|
355
|
-
if phrase_freqs.shape[0] ==
|
|
371
|
+
if phrase_freqs.shape[0] == self.max_doc_id + 1:
|
|
356
372
|
enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
|
|
357
373
|
else:
|
|
358
374
|
enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
|
|
359
375
|
keys=doc_ids.view(np.uint64)) for term_id in term_ids]
|
|
360
376
|
return compute_phrase_freqs(enc_term_posns, phrase_freqs)
|
|
361
377
|
|
|
362
|
-
def positions(self, term_id: int,
|
|
363
|
-
# Check if key is in doc ids?
|
|
364
|
-
doc_ids = index_range(self.doc_ids, key)
|
|
378
|
+
def positions(self, term_id: int, doc_ids) -> Union[List[np.ndarray], np.ndarray]:
|
|
365
379
|
if isinstance(doc_ids, numbers.Number):
|
|
366
380
|
doc_ids = np.asarray([doc_ids])
|
|
367
381
|
|
|
@@ -371,7 +385,7 @@ class PosnBitArray:
|
|
|
371
385
|
keys=np_doc_ids)
|
|
372
386
|
except KeyError:
|
|
373
387
|
r_val = [np.array([], dtype=np.uint32) for doc_id in doc_ids]
|
|
374
|
-
if len(r_val) == 1 and isinstance(
|
|
388
|
+
if len(r_val) == 1 and len(doc_ids) == 1 and isinstance(doc_ids[0], numbers.Number):
|
|
375
389
|
return [r_val[0]]
|
|
376
390
|
return r_val
|
|
377
391
|
|
|
@@ -392,29 +406,57 @@ class PosnBitArray:
|
|
|
392
406
|
return decs
|
|
393
407
|
else:
|
|
394
408
|
decs = [dec[1] for dec in decoded]
|
|
395
|
-
if len(decs) == 1 and isinstance(key, numbers.Number):
|
|
396
|
-
return decs[0]
|
|
397
409
|
return decs
|
|
398
410
|
|
|
399
411
|
def termfreqs(self, term_id: int, doc_ids: Optional[np.ndarray] = None) -> Tuple[np.ndarray, np.ndarray]:
|
|
400
412
|
"""Count term freqs using unique positions."""
|
|
413
|
+
if doc_ids is None:
|
|
414
|
+
return self._termfreqs_with_cache(term_id)
|
|
415
|
+
|
|
401
416
|
encoded = self.encoded_term_posns[term_id]
|
|
402
417
|
term_posns = encoded
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
418
|
+
term_posns = encoder.slice(encoded,
|
|
419
|
+
keys=doc_ids.astype(np.uint64))
|
|
420
|
+
return self._computed_term_freqs(term_posns)
|
|
421
|
+
|
|
422
|
+
def _computed_term_freqs(self, term_posns) -> Tuple[np.ndarray, np.ndarray]:
|
|
406
423
|
doc_ids = encoder.keys(term_posns)
|
|
407
424
|
change_indices = np.nonzero(np.diff(doc_ids))[0]
|
|
408
425
|
change_indices = np.concatenate((np.asarray([0]), change_indices + 1))
|
|
409
426
|
posns = term_posns & encoder.payload_lsb_mask
|
|
410
427
|
bit_counts = bit_count64(posns)
|
|
411
428
|
|
|
412
|
-
term_freqs = np.add.reduceat(bit_counts, change_indices)
|
|
413
|
-
return doc_ids, term_freqs
|
|
429
|
+
term_freqs = np.add.reduceat(bit_counts, change_indices, dtype=np.float32)
|
|
430
|
+
return sorted_unique(doc_ids), term_freqs
|
|
414
431
|
|
|
415
|
-
def
|
|
416
|
-
|
|
417
|
-
|
|
432
|
+
def _termfreqs_with_cache(self, term_id: int) -> Tuple[np.ndarray, np.ndarray]:
|
|
433
|
+
try:
|
|
434
|
+
return self.termfreq_cache[term_id]
|
|
435
|
+
except KeyError:
|
|
436
|
+
term_posns = self.encoded_term_posns[term_id]
|
|
437
|
+
doc_ids, term_freqs = self._computed_term_freqs(term_posns)
|
|
438
|
+
if self._is_cached(term_id):
|
|
439
|
+
self.termfreq_cache[term_id] = (doc_ids, term_freqs)
|
|
440
|
+
return doc_ids, term_freqs
|
|
441
|
+
|
|
442
|
+
def _is_cached(self, term_id: int) -> bool:
|
|
443
|
+
return term_id in self.docfreq_cache
|
|
444
|
+
|
|
445
|
+
def _docfreq_from_cache(self, term_id: int) -> np.uint64:
|
|
446
|
+
return self.docfreq_cache[term_id]
|
|
447
|
+
|
|
448
|
+
def _maybe_cache_docfreq(self, term_id: int, docfreq: np.uint64):
|
|
449
|
+
if self.max_doc_id >= 100000 and docfreq > (self.max_doc_id // 100):
|
|
450
|
+
self.docfreq_cache[term_id] = docfreq
|
|
451
|
+
|
|
452
|
+
def docfreq(self, term_id: int) -> np.uint64:
|
|
453
|
+
try:
|
|
454
|
+
return self.docfreq_cache[term_id]
|
|
455
|
+
except KeyError:
|
|
456
|
+
encoded = self.encoded_term_posns[term_id]
|
|
457
|
+
docfreq = np.uint64(encoder.keys_unique(encoded).size)
|
|
458
|
+
self._maybe_cache_docfreq(term_id, docfreq)
|
|
459
|
+
return docfreq
|
|
418
460
|
|
|
419
461
|
def insert(self, key, term_ids_to_posns, is_encoded=False):
|
|
420
462
|
new_posns = PosnBitArrayBuilder()
|
|
@@ -434,4 +476,9 @@ class PosnBitArray:
|
|
|
434
476
|
arr_bytes = 0
|
|
435
477
|
for doc_id, posn in self.encoded_term_posns.items():
|
|
436
478
|
arr_bytes += posn.nbytes
|
|
479
|
+
for term_id, (doc_ids, term_freqs) in self.termfreq_cache.items():
|
|
480
|
+
arr_bytes += doc_ids.nbytes
|
|
481
|
+
arr_bytes += term_freqs.nbytes
|
|
482
|
+
for term_id, docfreq in self.docfreq_cache.items():
|
|
483
|
+
arr_bytes += docfreq.nbytes
|
|
437
484
|
return arr_bytes
|
|
@@ -203,7 +203,11 @@ def _row_to_postings_row(doc_id, row, doc_len, term_dict, posns: PosnBitArray):
|
|
|
203
203
|
|
|
204
204
|
|
|
205
205
|
class SearchArray(ExtensionArray):
|
|
206
|
-
"""An array of tokenized text (
|
|
206
|
+
"""An array of tokenized text (Terms).
|
|
207
|
+
|
|
208
|
+
Use the class method `index` to create a SearchArray from a list of strings and tokenizer
|
|
209
|
+
|
|
210
|
+
"""
|
|
207
211
|
|
|
208
212
|
dtype = TermsDtype()
|
|
209
213
|
|
|
@@ -219,9 +223,33 @@ class SearchArray(ExtensionArray):
|
|
|
219
223
|
self.doc_lens = build_index_from_terms_list(postings, Terms)
|
|
220
224
|
|
|
221
225
|
@classmethod
|
|
222
|
-
def index(cls, array: Iterable,
|
|
223
|
-
|
|
224
|
-
|
|
226
|
+
def index(cls, array: Iterable,
|
|
227
|
+
tokenizer=ws_tokenizer,
|
|
228
|
+
truncate=False,
|
|
229
|
+
batch_size=100000,
|
|
230
|
+
avoid_copies=True,
|
|
231
|
+
autowarm=True) -> 'SearchArray':
|
|
232
|
+
"""Index an array of strings into SearchArray using tokenizer.
|
|
233
|
+
|
|
234
|
+
Parameters
|
|
235
|
+
----------
|
|
236
|
+
array : Iterable
|
|
237
|
+
A list of strings to index
|
|
238
|
+
tokenizer : Callable
|
|
239
|
+
A function that takes a string and returns an iterable of tokens
|
|
240
|
+
truncate : bool
|
|
241
|
+
If true, truncate the tokenized strings to a maximum length (2^18-1)
|
|
242
|
+
batch_size : int
|
|
243
|
+
For tuning memory usage, the number of strings to encode at once.
|
|
244
|
+
(which will be merged into one index at the end). Higher the more memory
|
|
245
|
+
will be used.
|
|
246
|
+
avoid_copies : bool
|
|
247
|
+
If true, avoid copying the term dictionary and positions during normal
|
|
248
|
+
pandas copy operations
|
|
249
|
+
autowarm : bool
|
|
250
|
+
If true, precompute docfreq / term freqs for most common terms
|
|
251
|
+
after indexing
|
|
252
|
+
"""
|
|
225
253
|
if not is_list_like(array):
|
|
226
254
|
raise TypeError("Expected list-like object, got {}".format(type(array)))
|
|
227
255
|
|
|
@@ -229,7 +257,10 @@ class SearchArray(ExtensionArray):
|
|
|
229
257
|
build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
|
|
230
258
|
truncate=truncate)
|
|
231
259
|
|
|
232
|
-
|
|
260
|
+
if autowarm:
|
|
261
|
+
posns.warm()
|
|
262
|
+
|
|
263
|
+
postings = cls([], tokenizer=tokenizer, avoid_copies=avoid_copies)
|
|
233
264
|
postings.term_mat = term_mat
|
|
234
265
|
postings.posns = posns
|
|
235
266
|
postings.term_dict = term_dict
|
|
@@ -237,6 +268,9 @@ class SearchArray(ExtensionArray):
|
|
|
237
268
|
postings.doc_lens = doc_lens
|
|
238
269
|
return postings
|
|
239
270
|
|
|
271
|
+
def warm(self):
|
|
272
|
+
self.posns.warm()
|
|
273
|
+
|
|
240
274
|
@classmethod
|
|
241
275
|
def _from_sequence(cls, scalars, dtype=None, copy=False):
|
|
242
276
|
"""Construct a new SearchArray from a sequence of scalars (PostingRow or convertible into)."""
|
|
@@ -278,7 +312,7 @@ class SearchArray(ExtensionArray):
|
|
|
278
312
|
else:
|
|
279
313
|
# Construct a sliced view of this array
|
|
280
314
|
sliced_tfs = self.term_mat.slice(key)
|
|
281
|
-
sliced_posns = self.posns
|
|
315
|
+
sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
|
|
282
316
|
arr = SearchArray([], tokenizer=self.tokenizer)
|
|
283
317
|
arr.term_mat = sliced_tfs
|
|
284
318
|
arr.doc_lens = self.doc_lens[key]
|
|
@@ -492,17 +526,22 @@ class SearchArray(ExtensionArray):
|
|
|
492
526
|
|
|
493
527
|
try:
|
|
494
528
|
term_id = self.term_dict.get_term_id(token)
|
|
495
|
-
matches = np.zeros(len(self), dtype=
|
|
529
|
+
matches = np.zeros(len(self), dtype=np.float32)
|
|
496
530
|
slice_of_rows = None
|
|
497
531
|
if self.term_mat.subset:
|
|
498
532
|
slice_of_rows = self.term_mat.rows
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
533
|
+
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
534
|
+
doc_ids=slice_of_rows)
|
|
535
|
+
mask = np.isin(self.term_mat.rows, doc_ids)
|
|
536
|
+
matches[mask] = termfreqs
|
|
537
|
+
return matches
|
|
538
|
+
else:
|
|
539
|
+
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
540
|
+
doc_ids=slice_of_rows)
|
|
541
|
+
matches[doc_ids] = termfreqs
|
|
542
|
+
return matches
|
|
504
543
|
except TermMissingError:
|
|
505
|
-
return np.zeros(len(self), dtype=
|
|
544
|
+
return np.zeros(len(self), dtype=np.float32)
|
|
506
545
|
|
|
507
546
|
def docfreq(self, token: str) -> int:
|
|
508
547
|
if not isinstance(token, str):
|
|
@@ -536,10 +575,13 @@ class SearchArray(ExtensionArray):
|
|
|
536
575
|
# Get term freqs per token
|
|
537
576
|
token = self._check_token_arg(token)
|
|
538
577
|
|
|
539
|
-
|
|
540
|
-
|
|
578
|
+
# For expensive toknes, we compute doc freq first, so we
|
|
579
|
+
# cache them in the DF cache, to let TF cache know it should be cached
|
|
541
580
|
tokens_l = [token] if isinstance(token, str) else token
|
|
542
581
|
all_dfs = np.asarray([self.docfreq(token) for token in tokens_l])
|
|
582
|
+
|
|
583
|
+
tfs = self.termfreqs(token)
|
|
584
|
+
token = self._check_token_arg(token)
|
|
543
585
|
doc_lens = self.doclengths()
|
|
544
586
|
scores = similarity(term_freqs=tfs, doc_freqs=all_dfs,
|
|
545
587
|
doc_lens=doc_lens, avg_doc_lens=self.avg_doc_length,
|
|
@@ -549,7 +591,8 @@ class SearchArray(ExtensionArray):
|
|
|
549
591
|
def positions(self, token: str, key=None) -> List[np.ndarray]:
|
|
550
592
|
"""Return a list of lists of positions of the given term."""
|
|
551
593
|
term_id = self.term_dict.get_term_id(token)
|
|
552
|
-
|
|
594
|
+
key = self.term_mat.rows[key] if key is not None else self.term_mat.rows
|
|
595
|
+
posns = self.posns.positions(term_id, doc_ids=key)
|
|
553
596
|
return posns
|
|
554
597
|
|
|
555
598
|
def and_query(self, tokens: Union[List[str], List[List[str]]]) -> np.ndarray:
|
|
@@ -12,6 +12,19 @@ class Similarity(Protocol):
|
|
|
12
12
|
...
|
|
13
13
|
|
|
14
14
|
|
|
15
|
+
def compute_idf(num_docs, sum_dfs):
|
|
16
|
+
"""Calculate idf."""
|
|
17
|
+
return np.log(1 + (num_docs - sum_dfs + 0.5) / (sum_dfs + 0.5))
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def compute_tfs(term_freqs: np.ndarray, doc_lens, avg_doc_lens, k1, b):
|
|
21
|
+
adj_doc_lens = k1 * (1 - b + b * doc_lens / avg_doc_lens)
|
|
22
|
+
# Divide tf in place for perf, but this means
|
|
23
|
+
# we can't use the same term_freqs for different k1, b
|
|
24
|
+
term_freqs /= (term_freqs + adj_doc_lens)
|
|
25
|
+
return term_freqs
|
|
26
|
+
|
|
27
|
+
|
|
15
28
|
def bm25_similarity(k1: float = 1.2, b: float = 0.75) -> Similarity:
|
|
16
29
|
"""BM25 similarity function, as in Lucene 9."""
|
|
17
30
|
def bm25(term_freqs: np.ndarray, doc_freqs: np.ndarray,
|
|
@@ -21,9 +34,10 @@ def bm25_similarity(k1: float = 1.2, b: float = 0.75) -> Similarity:
|
|
|
21
34
|
# Sum doc freqs
|
|
22
35
|
sum_dfs = np.sum(doc_freqs, axis=0)
|
|
23
36
|
# Calculate idf
|
|
24
|
-
idf =
|
|
37
|
+
idf = compute_idf(num_docs, sum_dfs)
|
|
25
38
|
# Calculate tf
|
|
26
|
-
tf = term_freqs / (term_freqs + k1 * (1 - b + b * doc_lens / avg_doc_lens))
|
|
39
|
+
# tf = term_freqs / (term_freqs + k1 * (1 - b + b * doc_lens / avg_doc_lens))
|
|
40
|
+
tf = compute_tfs(term_freqs, doc_lens, avg_doc_lens, k1, b)
|
|
27
41
|
return idf * tf
|
|
28
42
|
return bm25
|
|
29
43
|
|
|
@@ -9,22 +9,26 @@ m4 = np.uint64(0x0101010101010101)
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
mask = np.uint64(-1)
|
|
12
|
-
# TODO - precompute type specific hashes
|
|
13
12
|
s55 = np.uint64(m1 & mask) # Add more digits for 128bit support
|
|
14
13
|
s33 = np.uint64(m2 & mask)
|
|
15
14
|
s0F = np.uint64(m3 & mask)
|
|
16
15
|
s01 = np.uint64(m4 & mask)
|
|
17
16
|
num_bytes_64 = 8
|
|
17
|
+
all_but_one_bit = np.uint64(8 * (num_bytes_64 - 1))
|
|
18
|
+
|
|
19
|
+
_1 = np.uint64(1)
|
|
20
|
+
_2 = np.uint64(2)
|
|
21
|
+
_4 = np.uint64(4)
|
|
18
22
|
|
|
19
23
|
|
|
20
24
|
def bit_count64(arr):
|
|
21
25
|
"""Count the number of bits set in each element in the array."""
|
|
22
|
-
arr = arr - ((arr >>
|
|
23
|
-
arr = (arr & s33) + ((arr >>
|
|
26
|
+
arr = arr - ((arr >> _1) & s55)
|
|
27
|
+
arr = (arr & s33) + ((arr >> _2) & s33)
|
|
24
28
|
|
|
25
|
-
arr += (arr >>
|
|
29
|
+
arr += (arr >> _4)
|
|
26
30
|
arr &= s0F
|
|
27
31
|
arr *= s01
|
|
28
|
-
arr >>=
|
|
32
|
+
arr >>= all_but_one_bit
|
|
29
33
|
|
|
30
34
|
return arr
|
|
@@ -36,7 +36,7 @@ setup(
|
|
|
36
36
|
# For a discussion on single-sourcing the version across setup.py and the
|
|
37
37
|
# project code, see
|
|
38
38
|
# https://packaging.python.org/guides/single-sourcing-package-version/
|
|
39
|
-
version="0.0.
|
|
39
|
+
version="0.0.39", # Required
|
|
40
40
|
# This is a one-line description or tagline of what your project does. This
|
|
41
41
|
# corresponds to the "Summary" metadata field:
|
|
42
42
|
# https://packaging.python.org/specifications/core-metadata/#summary
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pytest
|
|
3
|
+
from searcharray.utils.bitcount import bit_count64
|
|
4
|
+
from test_utils import w_scenarios
|
|
5
|
+
from test_utils import Profiler, profile_enabled
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
scenarios = {
|
|
9
|
+
"base": {
|
|
10
|
+
"bits": np.asarray([np.uint64(0b0001)]),
|
|
11
|
+
"expected": [1],
|
|
12
|
+
},
|
|
13
|
+
"alternating_ones_lower32": {
|
|
14
|
+
"bits": np.asarray([np.uint64(0b01010101010101010101010101010101)]),
|
|
15
|
+
"expected": [16],
|
|
16
|
+
},
|
|
17
|
+
"alternating_ones_upper32": {
|
|
18
|
+
"bits": np.asarray([np.uint64(0b0101010101010101010101010101010100000000000000000000000000000000)]),
|
|
19
|
+
"expected": [16],
|
|
20
|
+
},
|
|
21
|
+
"one_at_end": {
|
|
22
|
+
"bits": np.asarray([np.uint64(0b0101010101010101010101010101010100000000000000000000000000000001)]),
|
|
23
|
+
"expected": [17],
|
|
24
|
+
},
|
|
25
|
+
"alt_ones": {
|
|
26
|
+
"bits": np.asarray([np.uint64(0b0101010101010101010101010101010101010101010101010101010101010101)]),
|
|
27
|
+
"expected": [32],
|
|
28
|
+
},
|
|
29
|
+
"all_ones": {
|
|
30
|
+
"bits": np.asarray([np.uint64(0b1111111111111111111111111111111111111111111111111111111111111111)]),
|
|
31
|
+
"expected": [64],
|
|
32
|
+
},
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@w_scenarios(scenarios)
|
|
37
|
+
def test_bitcount64(bits, expected):
|
|
38
|
+
bits_before = np.copy(bits)
|
|
39
|
+
assert bit_count64(bits) == expected
|
|
40
|
+
assert np.array_equal(bits, bits_before)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
44
|
+
def test_bitcount64_benchmark(benchmark):
|
|
45
|
+
profiler = Profiler(benchmark)
|
|
46
|
+
|
|
47
|
+
def bitcounts(arr, times=100):
|
|
48
|
+
for _ in range(times):
|
|
49
|
+
bit_count64(arr)
|
|
50
|
+
|
|
51
|
+
arr_size = 100000
|
|
52
|
+
arr = np.random.randint(0, 2**64, size=arr_size, dtype=np.uint64)
|
|
53
|
+
profiler.run(bitcounts, arr, times=1000)
|
|
@@ -271,7 +271,16 @@ def test_msmarco10k_indexing(msmarco100k_raw, benchmark):
|
|
|
271
271
|
profiler = Profiler(benchmark)
|
|
272
272
|
# Random 10k
|
|
273
273
|
tenk = msmarco100k_raw['body'].sample(10000)
|
|
274
|
-
results = profiler.run(SearchArray.index, tenk)
|
|
274
|
+
results = profiler.run(SearchArray.index, tenk, autowarm=False)
|
|
275
|
+
assert len(results) == 10000
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
279
|
+
def test_msmarco10k_indexing_warm(msmarco100k_raw, benchmark):
|
|
280
|
+
profiler = Profiler(benchmark)
|
|
281
|
+
# Random 10k
|
|
282
|
+
tenk = msmarco100k_raw['body'].sample(10000)
|
|
283
|
+
results = profiler.run(SearchArray.index, tenk, autowarm=True)
|
|
275
284
|
assert len(results) == 10000
|
|
276
285
|
|
|
277
286
|
|
|
@@ -306,13 +315,43 @@ def test_msmarco_indexall(msmarco_unzipped, benchmark, caplog):
|
|
|
306
315
|
|
|
307
316
|
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
308
317
|
@pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
|
|
309
|
-
def
|
|
318
|
+
def test_msmarco1m_or_search_unwarmed(query, msmarco1m, benchmark, caplog):
|
|
319
|
+
profiler = Profiler(benchmark)
|
|
320
|
+
|
|
321
|
+
caplog.set_level(logging.DEBUG)
|
|
322
|
+
|
|
323
|
+
def sum_scores(query):
|
|
324
|
+
return np.sum([msmarco1m['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
|
|
325
|
+
scores = profiler.run(sum_scores, query)
|
|
326
|
+
assert len(scores) == len(msmarco1m['body_ws'].array)
|
|
327
|
+
assert np.any(scores > 0)
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
331
|
+
@pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
|
|
332
|
+
def test_msmarco1m_or_search_warmed(query, msmarco1m, benchmark, caplog):
|
|
333
|
+
profiler = Profiler(benchmark)
|
|
334
|
+
|
|
335
|
+
caplog.set_level(logging.DEBUG)
|
|
336
|
+
|
|
337
|
+
def sum_scores(query):
|
|
338
|
+
return np.sum([msmarco1m['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
|
|
339
|
+
sum_scores(query) # Warmup
|
|
340
|
+
scores = profiler.run(sum_scores, query)
|
|
341
|
+
assert len(scores) == len(msmarco1m['body_ws'].array)
|
|
342
|
+
assert np.any(scores > 0)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
346
|
+
@pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
|
|
347
|
+
def test_msmarco1m_or_search_warming(query, msmarco1m, benchmark, caplog):
|
|
310
348
|
profiler = Profiler(benchmark)
|
|
311
349
|
|
|
312
350
|
caplog.set_level(logging.DEBUG)
|
|
313
351
|
|
|
314
352
|
def sum_scores(query):
|
|
315
353
|
return np.sum([msmarco1m['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
|
|
354
|
+
sum_scores(query) # Warmup
|
|
316
355
|
scores = profiler.run(sum_scores, query)
|
|
317
356
|
assert len(scores) == len(msmarco1m['body_ws'].array)
|
|
318
357
|
assert np.any(scores > 0)
|
|
@@ -320,13 +359,28 @@ def test_msmarco1m_or_search(query, msmarco1m, benchmark, caplog):
|
|
|
320
359
|
|
|
321
360
|
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
322
361
|
@pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
|
|
323
|
-
def
|
|
362
|
+
def test_msmarco100k_or_search_unwarmed(query, msmarco100k, benchmark, caplog):
|
|
363
|
+
profiler = Profiler(benchmark)
|
|
364
|
+
|
|
365
|
+
caplog.set_level(logging.DEBUG)
|
|
366
|
+
|
|
367
|
+
def sum_scores(query):
|
|
368
|
+
return np.sum([msmarco100k['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
|
|
369
|
+
scores = profiler.run(sum_scores, query)
|
|
370
|
+
assert len(scores) == len(msmarco100k['body_ws'].array)
|
|
371
|
+
assert np.any(scores > 0)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
375
|
+
@pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
|
|
376
|
+
def test_msmarco100k_or_search_warmed(query, msmarco100k, benchmark, caplog):
|
|
324
377
|
profiler = Profiler(benchmark)
|
|
325
378
|
|
|
326
379
|
caplog.set_level(logging.DEBUG)
|
|
327
380
|
|
|
328
381
|
def sum_scores(query):
|
|
329
382
|
return np.sum([msmarco100k['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
|
|
383
|
+
sum_scores(query) # Warmup
|
|
330
384
|
scores = profiler.run(sum_scores, query)
|
|
331
385
|
assert len(scores) == len(msmarco100k['body_ws'].array)
|
|
332
386
|
assert np.any(scores > 0)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Test postings array search functionality."""
|
|
2
|
-
import pytest
|
|
3
2
|
import numpy as np
|
|
3
|
+
import pytest
|
|
4
4
|
from searcharray.postings import SearchArray
|
|
5
5
|
from test_utils import w_scenarios
|
|
6
6
|
|
|
@@ -103,6 +103,12 @@ or_scenarios = {
|
|
|
103
103
|
"expected": [True, False, False, False] * 25,
|
|
104
104
|
"min_should_match": 1,
|
|
105
105
|
},
|
|
106
|
+
"or_with_phrase_on_copy": {
|
|
107
|
+
"docs": lambda: SearchArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"] * 25, avoid_copies=False),
|
|
108
|
+
"keywords": [["foo", "bar"], "baz"],
|
|
109
|
+
"expected": [True, False, False, False] * 25,
|
|
110
|
+
"min_should_match": 1,
|
|
111
|
+
},
|
|
106
112
|
"or_with_phrase_mm2": {
|
|
107
113
|
"docs": lambda: SearchArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"] * 25),
|
|
108
114
|
"keywords": [["foo", "bar"], ["bar", "baz"]],
|
|
@@ -113,17 +119,27 @@ or_scenarios = {
|
|
|
113
119
|
|
|
114
120
|
|
|
115
121
|
@w_scenarios(or_scenarios)
|
|
116
|
-
def test_or_query(
|
|
122
|
+
def test_or_query(docs, keywords, expected, min_should_match):
|
|
117
123
|
docs = docs()
|
|
118
|
-
matches =
|
|
124
|
+
matches = docs.or_query(keywords, min_should_match=min_should_match)
|
|
119
125
|
assert (expected == matches).all()
|
|
120
126
|
|
|
121
127
|
|
|
122
128
|
@w_scenarios(or_scenarios)
|
|
123
|
-
def test_or_query_sliced(
|
|
129
|
+
def test_or_query_sliced(docs, keywords, expected, min_should_match):
|
|
130
|
+
docs = docs()
|
|
131
|
+
num_docs = len(docs)
|
|
132
|
+
sliced = docs[:num_docs // 2]
|
|
133
|
+
expected_sliced = expected[:num_docs // 2]
|
|
134
|
+
matches = sliced.or_query(keywords, min_should_match=min_should_match)
|
|
135
|
+
assert (expected_sliced == matches).all()
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@w_scenarios(or_scenarios)
|
|
139
|
+
def test_or_query_copy(docs, keywords, expected, min_should_match):
|
|
124
140
|
docs = docs()
|
|
125
141
|
num_docs = len(docs)
|
|
126
|
-
sliced =
|
|
142
|
+
sliced = docs[:num_docs // 2].copy()
|
|
127
143
|
expected_sliced = expected[:num_docs // 2]
|
|
128
144
|
matches = sliced.or_query(keywords, min_should_match=min_should_match)
|
|
129
145
|
assert (expected_sliced == matches).all()
|
|
@@ -134,7 +134,14 @@ def test_phrase_match_tmdb(phrase, expected_matches, tmdb_data, benchmark):
|
|
|
134
134
|
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
135
135
|
def test_index_benchmark(benchmark, tmdb_pd_data):
|
|
136
136
|
prof = Profiler(benchmark)
|
|
137
|
-
results = prof.run(SearchArray.index, tmdb_pd_data['overview'])
|
|
137
|
+
results = prof.run(SearchArray.index, tmdb_pd_data['overview'], autowarm=False)
|
|
138
|
+
assert len(results) == len(tmdb_pd_data)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
142
|
+
def test_index_benchmark_warmed(benchmark, tmdb_pd_data):
|
|
143
|
+
prof = Profiler(benchmark)
|
|
144
|
+
results = prof.run(SearchArray.index, tmdb_pd_data['overview'], autowarm=True)
|
|
138
145
|
assert len(results) == len(tmdb_pd_data)
|
|
139
146
|
|
|
140
147
|
|
|
@@ -142,7 +149,7 @@ def test_index_benchmark(benchmark, tmdb_pd_data):
|
|
|
142
149
|
def test_index_benchmark_1k_random(benchmark, tmdb_pd_data):
|
|
143
150
|
prof = Profiler(benchmark)
|
|
144
151
|
thousand_random = np.random.choice(tmdb_pd_data['overview'], size=1000)
|
|
145
|
-
results = prof.run(SearchArray.index, thousand_random)
|
|
152
|
+
results = prof.run(SearchArray.index, thousand_random, autowarm=False)
|
|
146
153
|
assert len(results) == 1000
|
|
147
154
|
|
|
148
155
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|