searcharray 0.0.37__tar.gz → 0.0.39__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {searcharray-0.0.37 → searcharray-0.0.39}/PKG-INFO +1 -1
  2. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/phrase/middle_out.py +74 -27
  3. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/postings.py +59 -16
  4. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/similarity.py +16 -2
  5. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/utils/bitcount.py +9 -5
  6. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/PKG-INFO +1 -1
  7. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/SOURCES.txt +1 -0
  8. {searcharray-0.0.37 → searcharray-0.0.39}/setup.py +1 -1
  9. searcharray-0.0.39/test/test_bitcount64.py +53 -0
  10. {searcharray-0.0.37 → searcharray-0.0.39}/test/test_msmarco.py +57 -3
  11. {searcharray-0.0.37 → searcharray-0.0.39}/test/test_search.py +21 -5
  12. {searcharray-0.0.37 → searcharray-0.0.39}/test/test_tmdb.py +9 -2
  13. {searcharray-0.0.37 → searcharray-0.0.39}/README.md +0 -0
  14. {searcharray-0.0.37 → searcharray-0.0.39}/pyproject.toml +0 -0
  15. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/__init__.py +0 -0
  16. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/indexing.py +0 -0
  17. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/phrase/__init__.py +0 -0
  18. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/phrase/posn_diffs.py +0 -0
  19. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/phrase/scan_merge.py +0 -0
  20. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/solr.py +0 -0
  21. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/term_dict.py +0 -0
  22. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/utils/__init__.py +0 -0
  23. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/utils/mat_set.py +0 -0
  24. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/utils/roaringish.py +0 -0
  25. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray/utils/row_viewable_matrix.py +0 -0
  26. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/dependency_links.txt +0 -0
  27. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/entry_points.txt +0 -0
  28. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/requires.txt +0 -0
  29. {searcharray-0.0.37 → searcharray-0.0.39}/searcharray.egg-info/top_level.txt +0 -0
  30. {searcharray-0.0.37 → searcharray-0.0.39}/setup.cfg +0 -0
  31. {searcharray-0.0.37 → searcharray-0.0.39}/test/test_extension_array.py +0 -0
  32. {searcharray-0.0.37 → searcharray-0.0.39}/test/test_phrase_matches.py +0 -0
  33. {searcharray-0.0.37 → searcharray-0.0.39}/test/test_solr.py +0 -0
  34. {searcharray-0.0.37 → searcharray-0.0.39}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.37
3
+ Version: 0.0.39
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -9,7 +9,7 @@ import numpy as np
9
9
  import sortednp as snp
10
10
  from copy import deepcopy
11
11
  from typing import List, Tuple, Dict, Union, cast, Optional
12
- from searcharray.utils.roaringish import RoaringishEncoder, convert_keys
12
+ from searcharray.utils.roaringish import RoaringishEncoder, convert_keys, sorted_unique
13
13
  import numbers
14
14
  import logging
15
15
  from collections import defaultdict
@@ -203,7 +203,7 @@ class PosnBitArrayFromFlatBuilder:
203
203
  sliced = encoded[beg_idx:end_idx]
204
204
  encoded_term_posns[term_ids[into_terms]] = sliced
205
205
 
206
- return PosnBitArray(encoded_term_posns, range(0, np.max(self.flat_array[1]) + 1))
206
+ return PosnBitArray(encoded_term_posns, self.flat_array[1].max())
207
207
 
208
208
 
209
209
  class PosnBitArrayBuilder:
@@ -246,7 +246,7 @@ class PosnBitArrayBuilder:
246
246
  assert np.array_equal(posns, posns_again)
247
247
  encoded_term_posns[term_id] = encoded
248
248
 
249
- return PosnBitArray(encoded_term_posns, range(0, self.max_doc_id + 1))
249
+ return PosnBitArray(encoded_term_posns, self.max_doc_id)
250
250
 
251
251
 
252
252
  class PosnBitArrayAlreadyEncBuilder:
@@ -262,7 +262,7 @@ class PosnBitArrayAlreadyEncBuilder:
262
262
  self.max_doc_id = max(self.max_doc_id, doc_id)
263
263
 
264
264
  def build(self, check=False):
265
- return PosnBitArray(self.encoded_term_posns, range(0, self.max_doc_id + 1))
265
+ return PosnBitArray(self.encoded_term_posns, self.max_doc_id)
266
266
 
267
267
 
268
268
  def index_range(rng, key):
@@ -290,13 +290,25 @@ def index_range(rng, key):
290
290
 
291
291
  class PosnBitArray:
292
292
 
293
- def __init__(self, encoded_term_posns, doc_ids):
293
+ def __init__(self, encoded_term_posns, max_doc_id: int):
294
294
  self.encoded_term_posns = encoded_term_posns
295
- self.doc_ids = doc_ids
295
+ self.max_doc_id = max_doc_id
296
+ self.docfreq_cache : Dict[int, np.uint64] = {}
297
+ self.termfreq_cache : Dict[int, Tuple[np.ndarray, np.ndarray]] = {}
298
+
299
+ def warm(self):
300
+ """Warm tf / df cache of most common terms."""
301
+ for term_id, encoded in self.encoded_term_posns.items():
302
+ if len(encoded) > 255:
303
+ self.docfreq(term_id)
304
+ self.termfreqs(term_id)
305
+
306
+ def clear_cache(self):
307
+ self.docfreq_cache = {}
308
+ self.termfreq_cache = {}
296
309
 
297
310
  def copy(self):
298
- new = PosnBitArray(deepcopy(self.encoded_term_posns),
299
- self.doc_ids)
311
+ new = PosnBitArray(deepcopy(self.encoded_term_posns), self.max_doc_id)
300
312
  return new
301
313
 
302
314
  def concat(self, other):
@@ -314,17 +326,19 @@ class PosnBitArray:
314
326
  only_other_terms = set(other.encoded_term_posns.keys()).difference(set(self.encoded_term_posns.keys()))
315
327
  for term_id in only_other_terms:
316
328
  self.encoded_term_posns[term_id] = other.encoded_term_posns[term_id]
329
+ self.max_doc_id = max(self.max_doc_id, other.max_doc_id)
330
+ self.clear_cache()
317
331
 
318
332
  def slice(self, key):
319
333
  sliced_term_posns = {}
320
- doc_ids = index_range(self.doc_ids, key)
321
- np_doc_ids = convert_keys(doc_ids)
334
+ doc_ids = convert_keys(key)
335
+ max_doc_id = np.max(doc_ids)
322
336
  for term_id, posns in self.encoded_term_posns.items():
323
337
  encoded = self.encoded_term_posns[term_id]
324
338
  assert len(encoded.shape) == 1
325
- sliced_term_posns[term_id] = encoder.slice(encoded, keys=np_doc_ids)
339
+ sliced_term_posns[term_id] = encoder.slice(encoded, keys=doc_ids)
326
340
 
327
- return PosnBitArray(sliced_term_posns, doc_ids=doc_ids)
341
+ return PosnBitArray(sliced_term_posns, max_doc_id)
328
342
 
329
343
  def __getitem__(self, key):
330
344
  return self.slice(key)
@@ -342,6 +356,8 @@ class PosnBitArray:
342
356
  posns_self = self.encoded_term_posns[term_id]
343
357
  posns_other = other.encoded_term_posns[term_id]
344
358
  self.encoded_term_posns[term_id] = snp.merge(posns_self, posns_other)
359
+ self.max_doc_id = self.max_doc_id + other.max_doc_id
360
+ self.clear_cache()
345
361
 
346
362
  def doc_encoded_posns(self, term_id: int, doc_id: int) -> np.ndarray:
347
363
  term_posns = encoder.slice(self.encoded_term_posns[term_id],
@@ -352,16 +368,14 @@ class PosnBitArray:
352
368
  doc_ids: np.ndarray) -> np.ndarray:
353
369
  if len(term_ids) < 2:
354
370
  raise ValueError("Must have at least two terms")
355
- if phrase_freqs.shape[0] == len(self.doc_ids):
371
+ if phrase_freqs.shape[0] == self.max_doc_id + 1:
356
372
  enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
357
373
  else:
358
374
  enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
359
375
  keys=doc_ids.view(np.uint64)) for term_id in term_ids]
360
376
  return compute_phrase_freqs(enc_term_posns, phrase_freqs)
361
377
 
362
- def positions(self, term_id: int, key) -> Union[List[np.ndarray], np.ndarray]:
363
- # Check if key is in doc ids?
364
- doc_ids = index_range(self.doc_ids, key)
378
+ def positions(self, term_id: int, doc_ids) -> Union[List[np.ndarray], np.ndarray]:
365
379
  if isinstance(doc_ids, numbers.Number):
366
380
  doc_ids = np.asarray([doc_ids])
367
381
 
@@ -371,7 +385,7 @@ class PosnBitArray:
371
385
  keys=np_doc_ids)
372
386
  except KeyError:
373
387
  r_val = [np.array([], dtype=np.uint32) for doc_id in doc_ids]
374
- if len(r_val) == 1 and isinstance(key, numbers.Number):
388
+ if len(r_val) == 1 and len(doc_ids) == 1 and isinstance(doc_ids[0], numbers.Number):
375
389
  return [r_val[0]]
376
390
  return r_val
377
391
 
@@ -392,29 +406,57 @@ class PosnBitArray:
392
406
  return decs
393
407
  else:
394
408
  decs = [dec[1] for dec in decoded]
395
- if len(decs) == 1 and isinstance(key, numbers.Number):
396
- return decs[0]
397
409
  return decs
398
410
 
399
411
  def termfreqs(self, term_id: int, doc_ids: Optional[np.ndarray] = None) -> Tuple[np.ndarray, np.ndarray]:
400
412
  """Count term freqs using unique positions."""
413
+ if doc_ids is None:
414
+ return self._termfreqs_with_cache(term_id)
415
+
401
416
  encoded = self.encoded_term_posns[term_id]
402
417
  term_posns = encoded
403
- if doc_ids is not None:
404
- term_posns = encoder.slice(encoded,
405
- keys=doc_ids.astype(np.uint64))
418
+ term_posns = encoder.slice(encoded,
419
+ keys=doc_ids.astype(np.uint64))
420
+ return self._computed_term_freqs(term_posns)
421
+
422
+ def _computed_term_freqs(self, term_posns) -> Tuple[np.ndarray, np.ndarray]:
406
423
  doc_ids = encoder.keys(term_posns)
407
424
  change_indices = np.nonzero(np.diff(doc_ids))[0]
408
425
  change_indices = np.concatenate((np.asarray([0]), change_indices + 1))
409
426
  posns = term_posns & encoder.payload_lsb_mask
410
427
  bit_counts = bit_count64(posns)
411
428
 
412
- term_freqs = np.add.reduceat(bit_counts, change_indices)
413
- return doc_ids, term_freqs
429
+ term_freqs = np.add.reduceat(bit_counts, change_indices, dtype=np.float32)
430
+ return sorted_unique(doc_ids), term_freqs
414
431
 
415
- def docfreq(self, term_id: int) -> np.uint32:
416
- encoded = self.encoded_term_posns[term_id]
417
- return np.uint32(encoder.keys_unique(encoded).size)
432
+ def _termfreqs_with_cache(self, term_id: int) -> Tuple[np.ndarray, np.ndarray]:
433
+ try:
434
+ return self.termfreq_cache[term_id]
435
+ except KeyError:
436
+ term_posns = self.encoded_term_posns[term_id]
437
+ doc_ids, term_freqs = self._computed_term_freqs(term_posns)
438
+ if self._is_cached(term_id):
439
+ self.termfreq_cache[term_id] = (doc_ids, term_freqs)
440
+ return doc_ids, term_freqs
441
+
442
+ def _is_cached(self, term_id: int) -> bool:
443
+ return term_id in self.docfreq_cache
444
+
445
+ def _docfreq_from_cache(self, term_id: int) -> np.uint64:
446
+ return self.docfreq_cache[term_id]
447
+
448
+ def _maybe_cache_docfreq(self, term_id: int, docfreq: np.uint64):
449
+ if self.max_doc_id >= 100000 and docfreq > (self.max_doc_id // 100):
450
+ self.docfreq_cache[term_id] = docfreq
451
+
452
+ def docfreq(self, term_id: int) -> np.uint64:
453
+ try:
454
+ return self.docfreq_cache[term_id]
455
+ except KeyError:
456
+ encoded = self.encoded_term_posns[term_id]
457
+ docfreq = np.uint64(encoder.keys_unique(encoded).size)
458
+ self._maybe_cache_docfreq(term_id, docfreq)
459
+ return docfreq
418
460
 
419
461
  def insert(self, key, term_ids_to_posns, is_encoded=False):
420
462
  new_posns = PosnBitArrayBuilder()
@@ -434,4 +476,9 @@ class PosnBitArray:
434
476
  arr_bytes = 0
435
477
  for doc_id, posn in self.encoded_term_posns.items():
436
478
  arr_bytes += posn.nbytes
479
+ for term_id, (doc_ids, term_freqs) in self.termfreq_cache.items():
480
+ arr_bytes += doc_ids.nbytes
481
+ arr_bytes += term_freqs.nbytes
482
+ for term_id, docfreq in self.docfreq_cache.items():
483
+ arr_bytes += docfreq.nbytes
437
484
  return arr_bytes
@@ -203,7 +203,11 @@ def _row_to_postings_row(doc_id, row, doc_len, term_dict, posns: PosnBitArray):
203
203
 
204
204
 
205
205
  class SearchArray(ExtensionArray):
206
- """An array of tokenized text (Termss)."""
206
+ """An array of tokenized text (Terms).
207
+
208
+ Use the class method `index` to create a SearchArray from a list of strings and tokenizer
209
+
210
+ """
207
211
 
208
212
  dtype = TermsDtype()
209
213
 
@@ -219,9 +223,33 @@ class SearchArray(ExtensionArray):
219
223
  self.doc_lens = build_index_from_terms_list(postings, Terms)
220
224
 
221
225
  @classmethod
222
- def index(cls, array: Iterable, tokenizer=ws_tokenizer,
223
- truncate=False, batch_size=100000) -> 'SearchArray':
224
- """Index an array of strings using tokenizer."""
226
+ def index(cls, array: Iterable,
227
+ tokenizer=ws_tokenizer,
228
+ truncate=False,
229
+ batch_size=100000,
230
+ avoid_copies=True,
231
+ autowarm=True) -> 'SearchArray':
232
+ """Index an array of strings into SearchArray using tokenizer.
233
+
234
+ Parameters
235
+ ----------
236
+ array : Iterable
237
+ A list of strings to index
238
+ tokenizer : Callable
239
+ A function that takes a string and returns an iterable of tokens
240
+ truncate : bool
241
+ If true, truncate the tokenized strings to a maximum length (2^18-1)
242
+ batch_size : int
243
+ For tuning memory usage, the number of strings to encode at once.
244
+ (which will be merged into one index at the end). Higher the more memory
245
+ will be used.
246
+ avoid_copies : bool
247
+ If true, avoid copying the term dictionary and positions during normal
248
+ pandas copy operations
249
+ autowarm : bool
250
+ If true, precompute docfreq / term freqs for most common terms
251
+ after indexing
252
+ """
225
253
  if not is_list_like(array):
226
254
  raise TypeError("Expected list-like object, got {}".format(type(array)))
227
255
 
@@ -229,7 +257,10 @@ class SearchArray(ExtensionArray):
229
257
  build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
230
258
  truncate=truncate)
231
259
 
232
- postings = cls([], tokenizer=tokenizer)
260
+ if autowarm:
261
+ posns.warm()
262
+
263
+ postings = cls([], tokenizer=tokenizer, avoid_copies=avoid_copies)
233
264
  postings.term_mat = term_mat
234
265
  postings.posns = posns
235
266
  postings.term_dict = term_dict
@@ -237,6 +268,9 @@ class SearchArray(ExtensionArray):
237
268
  postings.doc_lens = doc_lens
238
269
  return postings
239
270
 
271
+ def warm(self):
272
+ self.posns.warm()
273
+
240
274
  @classmethod
241
275
  def _from_sequence(cls, scalars, dtype=None, copy=False):
242
276
  """Construct a new SearchArray from a sequence of scalars (PostingRow or convertible into)."""
@@ -278,7 +312,7 @@ class SearchArray(ExtensionArray):
278
312
  else:
279
313
  # Construct a sliced view of this array
280
314
  sliced_tfs = self.term_mat.slice(key)
281
- sliced_posns = self.posns
315
+ sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
282
316
  arr = SearchArray([], tokenizer=self.tokenizer)
283
317
  arr.term_mat = sliced_tfs
284
318
  arr.doc_lens = self.doc_lens[key]
@@ -492,17 +526,22 @@ class SearchArray(ExtensionArray):
492
526
 
493
527
  try:
494
528
  term_id = self.term_dict.get_term_id(token)
495
- matches = np.zeros(len(self), dtype=int)
529
+ matches = np.zeros(len(self), dtype=np.float32)
496
530
  slice_of_rows = None
497
531
  if self.term_mat.subset:
498
532
  slice_of_rows = self.term_mat.rows
499
- doc_ids, termfreqs = self.posns.termfreqs(term_id,
500
- doc_ids=slice_of_rows)
501
- mask = np.isin(self.term_mat.rows, doc_ids)
502
- matches[mask] = termfreqs
503
- return matches
533
+ doc_ids, termfreqs = self.posns.termfreqs(term_id,
534
+ doc_ids=slice_of_rows)
535
+ mask = np.isin(self.term_mat.rows, doc_ids)
536
+ matches[mask] = termfreqs
537
+ return matches
538
+ else:
539
+ doc_ids, termfreqs = self.posns.termfreqs(term_id,
540
+ doc_ids=slice_of_rows)
541
+ matches[doc_ids] = termfreqs
542
+ return matches
504
543
  except TermMissingError:
505
- return np.zeros(len(self), dtype=int)
544
+ return np.zeros(len(self), dtype=np.float32)
506
545
 
507
546
  def docfreq(self, token: str) -> int:
508
547
  if not isinstance(token, str):
@@ -536,10 +575,13 @@ class SearchArray(ExtensionArray):
536
575
  # Get term freqs per token
537
576
  token = self._check_token_arg(token)
538
577
 
539
- tfs = self.termfreqs(token)
540
- token = self._check_token_arg(token)
578
+ # For expensive toknes, we compute doc freq first, so we
579
+ # cache them in the DF cache, to let TF cache know it should be cached
541
580
  tokens_l = [token] if isinstance(token, str) else token
542
581
  all_dfs = np.asarray([self.docfreq(token) for token in tokens_l])
582
+
583
+ tfs = self.termfreqs(token)
584
+ token = self._check_token_arg(token)
543
585
  doc_lens = self.doclengths()
544
586
  scores = similarity(term_freqs=tfs, doc_freqs=all_dfs,
545
587
  doc_lens=doc_lens, avg_doc_lens=self.avg_doc_length,
@@ -549,7 +591,8 @@ class SearchArray(ExtensionArray):
549
591
  def positions(self, token: str, key=None) -> List[np.ndarray]:
550
592
  """Return a list of lists of positions of the given term."""
551
593
  term_id = self.term_dict.get_term_id(token)
552
- posns = self.posns.positions(term_id, key=key)
594
+ key = self.term_mat.rows[key] if key is not None else self.term_mat.rows
595
+ posns = self.posns.positions(term_id, doc_ids=key)
553
596
  return posns
554
597
 
555
598
  def and_query(self, tokens: Union[List[str], List[List[str]]]) -> np.ndarray:
@@ -12,6 +12,19 @@ class Similarity(Protocol):
12
12
  ...
13
13
 
14
14
 
15
+ def compute_idf(num_docs, sum_dfs):
16
+ """Calculate idf."""
17
+ return np.log(1 + (num_docs - sum_dfs + 0.5) / (sum_dfs + 0.5))
18
+
19
+
20
+ def compute_tfs(term_freqs: np.ndarray, doc_lens, avg_doc_lens, k1, b):
21
+ adj_doc_lens = k1 * (1 - b + b * doc_lens / avg_doc_lens)
22
+ # Divide tf in place for perf, but this means
23
+ # we can't use the same term_freqs for different k1, b
24
+ term_freqs /= (term_freqs + adj_doc_lens)
25
+ return term_freqs
26
+
27
+
15
28
  def bm25_similarity(k1: float = 1.2, b: float = 0.75) -> Similarity:
16
29
  """BM25 similarity function, as in Lucene 9."""
17
30
  def bm25(term_freqs: np.ndarray, doc_freqs: np.ndarray,
@@ -21,9 +34,10 @@ def bm25_similarity(k1: float = 1.2, b: float = 0.75) -> Similarity:
21
34
  # Sum doc freqs
22
35
  sum_dfs = np.sum(doc_freqs, axis=0)
23
36
  # Calculate idf
24
- idf = np.log(1 + (num_docs - sum_dfs + 0.5) / (sum_dfs + 0.5))
37
+ idf = compute_idf(num_docs, sum_dfs)
25
38
  # Calculate tf
26
- tf = term_freqs / (term_freqs + k1 * (1 - b + b * doc_lens / avg_doc_lens))
39
+ # tf = term_freqs / (term_freqs + k1 * (1 - b + b * doc_lens / avg_doc_lens))
40
+ tf = compute_tfs(term_freqs, doc_lens, avg_doc_lens, k1, b)
27
41
  return idf * tf
28
42
  return bm25
29
43
 
@@ -9,22 +9,26 @@ m4 = np.uint64(0x0101010101010101)
9
9
 
10
10
 
11
11
  mask = np.uint64(-1)
12
- # TODO - precompute type specific hashes
13
12
  s55 = np.uint64(m1 & mask) # Add more digits for 128bit support
14
13
  s33 = np.uint64(m2 & mask)
15
14
  s0F = np.uint64(m3 & mask)
16
15
  s01 = np.uint64(m4 & mask)
17
16
  num_bytes_64 = 8
17
+ all_but_one_bit = np.uint64(8 * (num_bytes_64 - 1))
18
+
19
+ _1 = np.uint64(1)
20
+ _2 = np.uint64(2)
21
+ _4 = np.uint64(4)
18
22
 
19
23
 
20
24
  def bit_count64(arr):
21
25
  """Count the number of bits set in each element in the array."""
22
- arr = arr - ((arr >> 1) & s55)
23
- arr = (arr & s33) + ((arr >> 2) & s33)
26
+ arr = arr - ((arr >> _1) & s55)
27
+ arr = (arr & s33) + ((arr >> _2) & s33)
24
28
 
25
- arr += (arr >> 4)
29
+ arr += (arr >> _4)
26
30
  arr &= s0F
27
31
  arr *= s01
28
- arr >>= (8 * (num_bytes_64 - 1))
32
+ arr >>= all_but_one_bit
29
33
 
30
34
  return arr
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.37
3
+ Version: 0.0.39
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -23,6 +23,7 @@ searcharray/utils/bitcount.py
23
23
  searcharray/utils/mat_set.py
24
24
  searcharray/utils/roaringish.py
25
25
  searcharray/utils/row_viewable_matrix.py
26
+ test/test_bitcount64.py
26
27
  test/test_extension_array.py
27
28
  test/test_msmarco.py
28
29
  test/test_phrase_matches.py
@@ -36,7 +36,7 @@ setup(
36
36
  # For a discussion on single-sourcing the version across setup.py and the
37
37
  # project code, see
38
38
  # https://packaging.python.org/guides/single-sourcing-package-version/
39
- version="0.0.37", # Required
39
+ version="0.0.39", # Required
40
40
  # This is a one-line description or tagline of what your project does. This
41
41
  # corresponds to the "Summary" metadata field:
42
42
  # https://packaging.python.org/specifications/core-metadata/#summary
@@ -0,0 +1,53 @@
1
+ import numpy as np
2
+ import pytest
3
+ from searcharray.utils.bitcount import bit_count64
4
+ from test_utils import w_scenarios
5
+ from test_utils import Profiler, profile_enabled
6
+
7
+
8
+ scenarios = {
9
+ "base": {
10
+ "bits": np.asarray([np.uint64(0b0001)]),
11
+ "expected": [1],
12
+ },
13
+ "alternating_ones_lower32": {
14
+ "bits": np.asarray([np.uint64(0b01010101010101010101010101010101)]),
15
+ "expected": [16],
16
+ },
17
+ "alternating_ones_upper32": {
18
+ "bits": np.asarray([np.uint64(0b0101010101010101010101010101010100000000000000000000000000000000)]),
19
+ "expected": [16],
20
+ },
21
+ "one_at_end": {
22
+ "bits": np.asarray([np.uint64(0b0101010101010101010101010101010100000000000000000000000000000001)]),
23
+ "expected": [17],
24
+ },
25
+ "alt_ones": {
26
+ "bits": np.asarray([np.uint64(0b0101010101010101010101010101010101010101010101010101010101010101)]),
27
+ "expected": [32],
28
+ },
29
+ "all_ones": {
30
+ "bits": np.asarray([np.uint64(0b1111111111111111111111111111111111111111111111111111111111111111)]),
31
+ "expected": [64],
32
+ },
33
+ }
34
+
35
+
36
+ @w_scenarios(scenarios)
37
+ def test_bitcount64(bits, expected):
38
+ bits_before = np.copy(bits)
39
+ assert bit_count64(bits) == expected
40
+ assert np.array_equal(bits, bits_before)
41
+
42
+
43
+ @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
44
+ def test_bitcount64_benchmark(benchmark):
45
+ profiler = Profiler(benchmark)
46
+
47
+ def bitcounts(arr, times=100):
48
+ for _ in range(times):
49
+ bit_count64(arr)
50
+
51
+ arr_size = 100000
52
+ arr = np.random.randint(0, 2**64, size=arr_size, dtype=np.uint64)
53
+ profiler.run(bitcounts, arr, times=1000)
@@ -271,7 +271,16 @@ def test_msmarco10k_indexing(msmarco100k_raw, benchmark):
271
271
  profiler = Profiler(benchmark)
272
272
  # Random 10k
273
273
  tenk = msmarco100k_raw['body'].sample(10000)
274
- results = profiler.run(SearchArray.index, tenk)
274
+ results = profiler.run(SearchArray.index, tenk, autowarm=False)
275
+ assert len(results) == 10000
276
+
277
+
278
+ @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
279
+ def test_msmarco10k_indexing_warm(msmarco100k_raw, benchmark):
280
+ profiler = Profiler(benchmark)
281
+ # Random 10k
282
+ tenk = msmarco100k_raw['body'].sample(10000)
283
+ results = profiler.run(SearchArray.index, tenk, autowarm=True)
275
284
  assert len(results) == 10000
276
285
 
277
286
 
@@ -306,13 +315,43 @@ def test_msmarco_indexall(msmarco_unzipped, benchmark, caplog):
306
315
 
307
316
  @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
308
317
  @pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
309
- def test_msmarco1m_or_search(query, msmarco1m, benchmark, caplog):
318
+ def test_msmarco1m_or_search_unwarmed(query, msmarco1m, benchmark, caplog):
319
+ profiler = Profiler(benchmark)
320
+
321
+ caplog.set_level(logging.DEBUG)
322
+
323
+ def sum_scores(query):
324
+ return np.sum([msmarco1m['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
325
+ scores = profiler.run(sum_scores, query)
326
+ assert len(scores) == len(msmarco1m['body_ws'].array)
327
+ assert np.any(scores > 0)
328
+
329
+
330
+ @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
331
+ @pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
332
+ def test_msmarco1m_or_search_warmed(query, msmarco1m, benchmark, caplog):
333
+ profiler = Profiler(benchmark)
334
+
335
+ caplog.set_level(logging.DEBUG)
336
+
337
+ def sum_scores(query):
338
+ return np.sum([msmarco1m['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
339
+ sum_scores(query) # Warmup
340
+ scores = profiler.run(sum_scores, query)
341
+ assert len(scores) == len(msmarco1m['body_ws'].array)
342
+ assert np.any(scores > 0)
343
+
344
+
345
+ @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
346
+ @pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
347
+ def test_msmarco1m_or_search_warming(query, msmarco1m, benchmark, caplog):
310
348
  profiler = Profiler(benchmark)
311
349
 
312
350
  caplog.set_level(logging.DEBUG)
313
351
 
314
352
  def sum_scores(query):
315
353
  return np.sum([msmarco1m['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
354
+ sum_scores(query) # Warmup
316
355
  scores = profiler.run(sum_scores, query)
317
356
  assert len(scores) == len(msmarco1m['body_ws'].array)
318
357
  assert np.any(scores > 0)
@@ -320,13 +359,28 @@ def test_msmarco1m_or_search(query, msmarco1m, benchmark, caplog):
320
359
 
321
360
  @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
322
361
  @pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
323
- def test_msmarco100k_or_search(query, msmarco100k, benchmark, caplog):
362
+ def test_msmarco100k_or_search_unwarmed(query, msmarco100k, benchmark, caplog):
363
+ profiler = Profiler(benchmark)
364
+
365
+ caplog.set_level(logging.DEBUG)
366
+
367
+ def sum_scores(query):
368
+ return np.sum([msmarco100k['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
369
+ scores = profiler.run(sum_scores, query)
370
+ assert len(scores) == len(msmarco100k['body_ws'].array)
371
+ assert np.any(scores > 0)
372
+
373
+
374
+ @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
375
+ @pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
376
+ def test_msmarco100k_or_search_warmed(query, msmarco100k, benchmark, caplog):
324
377
  profiler = Profiler(benchmark)
325
378
 
326
379
  caplog.set_level(logging.DEBUG)
327
380
 
328
381
  def sum_scores(query):
329
382
  return np.sum([msmarco100k['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
383
+ sum_scores(query) # Warmup
330
384
  scores = profiler.run(sum_scores, query)
331
385
  assert len(scores) == len(msmarco100k['body_ws'].array)
332
386
  assert np.any(scores > 0)
@@ -1,6 +1,6 @@
1
1
  """Test postings array search functionality."""
2
- import pytest
3
2
  import numpy as np
3
+ import pytest
4
4
  from searcharray.postings import SearchArray
5
5
  from test_utils import w_scenarios
6
6
 
@@ -103,6 +103,12 @@ or_scenarios = {
103
103
  "expected": [True, False, False, False] * 25,
104
104
  "min_should_match": 1,
105
105
  },
106
+ "or_with_phrase_on_copy": {
107
+ "docs": lambda: SearchArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"] * 25, avoid_copies=False),
108
+ "keywords": [["foo", "bar"], "baz"],
109
+ "expected": [True, False, False, False] * 25,
110
+ "min_should_match": 1,
111
+ },
106
112
  "or_with_phrase_mm2": {
107
113
  "docs": lambda: SearchArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"] * 25),
108
114
  "keywords": [["foo", "bar"], ["bar", "baz"]],
@@ -113,17 +119,27 @@ or_scenarios = {
113
119
 
114
120
 
115
121
  @w_scenarios(or_scenarios)
116
- def test_or_query(data, docs, keywords, expected, min_should_match):
122
+ def test_or_query(docs, keywords, expected, min_should_match):
117
123
  docs = docs()
118
- matches = data.or_query(keywords, min_should_match=min_should_match)
124
+ matches = docs.or_query(keywords, min_should_match=min_should_match)
119
125
  assert (expected == matches).all()
120
126
 
121
127
 
122
128
  @w_scenarios(or_scenarios)
123
- def test_or_query_sliced(data, docs, keywords, expected, min_should_match):
129
+ def test_or_query_sliced(docs, keywords, expected, min_should_match):
130
+ docs = docs()
131
+ num_docs = len(docs)
132
+ sliced = docs[:num_docs // 2]
133
+ expected_sliced = expected[:num_docs // 2]
134
+ matches = sliced.or_query(keywords, min_should_match=min_should_match)
135
+ assert (expected_sliced == matches).all()
136
+
137
+
138
+ @w_scenarios(or_scenarios)
139
+ def test_or_query_copy(docs, keywords, expected, min_should_match):
124
140
  docs = docs()
125
141
  num_docs = len(docs)
126
- sliced = data[:num_docs // 2]
142
+ sliced = docs[:num_docs // 2].copy()
127
143
  expected_sliced = expected[:num_docs // 2]
128
144
  matches = sliced.or_query(keywords, min_should_match=min_should_match)
129
145
  assert (expected_sliced == matches).all()
@@ -134,7 +134,14 @@ def test_phrase_match_tmdb(phrase, expected_matches, tmdb_data, benchmark):
134
134
  @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
135
135
  def test_index_benchmark(benchmark, tmdb_pd_data):
136
136
  prof = Profiler(benchmark)
137
- results = prof.run(SearchArray.index, tmdb_pd_data['overview'])
137
+ results = prof.run(SearchArray.index, tmdb_pd_data['overview'], autowarm=False)
138
+ assert len(results) == len(tmdb_pd_data)
139
+
140
+
141
+ @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
142
+ def test_index_benchmark_warmed(benchmark, tmdb_pd_data):
143
+ prof = Profiler(benchmark)
144
+ results = prof.run(SearchArray.index, tmdb_pd_data['overview'], autowarm=True)
138
145
  assert len(results) == len(tmdb_pd_data)
139
146
 
140
147
 
@@ -142,7 +149,7 @@ def test_index_benchmark(benchmark, tmdb_pd_data):
142
149
  def test_index_benchmark_1k_random(benchmark, tmdb_pd_data):
143
150
  prof = Profiler(benchmark)
144
151
  thousand_random = np.random.choice(tmdb_pd_data['overview'], size=1000)
145
- results = prof.run(SearchArray.index, thousand_random)
152
+ results = prof.run(SearchArray.index, thousand_random, autowarm=False)
146
153
  assert len(results) == 1000
147
154
 
148
155
 
File without changes
File without changes