searcharray 0.0.45__tar.gz → 0.0.46__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {searcharray-0.0.45 → searcharray-0.0.46}/PKG-INFO +1 -1
  2. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/postings.py +27 -9
  3. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/roaringish_ops.c +819 -235
  4. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/roaringish_ops.pyx +26 -2
  5. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/snp_ops.c +117 -117
  6. searcharray-0.0.46/searcharray/similarity.py +102 -0
  7. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/PKG-INFO +1 -1
  8. {searcharray-0.0.45 → searcharray-0.0.46}/setup.py +1 -1
  9. {searcharray-0.0.45 → searcharray-0.0.46}/test/test_msmarco.py +7 -4
  10. {searcharray-0.0.45 → searcharray-0.0.46}/test/test_solr.py +3 -2
  11. {searcharray-0.0.45 → searcharray-0.0.46}/test/test_tmdb.py +4 -1
  12. searcharray-0.0.45/searcharray/similarity.py +0 -82
  13. {searcharray-0.0.45 → searcharray-0.0.46}/README.md +0 -0
  14. {searcharray-0.0.45 → searcharray-0.0.46}/pyproject.toml +0 -0
  15. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/__init__.py +0 -0
  16. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/indexing.py +0 -0
  17. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/phrase/__init__.py +0 -0
  18. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/phrase/bigram_freqs.py +0 -0
  19. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/phrase/middle_out.py +0 -0
  20. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/phrase/posn_diffs.py +0 -0
  21. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/phrase/scan_merge.py +0 -0
  22. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/__init__.py +0 -0
  23. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/roaringish.py +0 -0
  24. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/snp_ops.pxd +0 -0
  25. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/snp_ops.pyx +0 -0
  26. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/solr.py +0 -0
  27. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/term_dict.py +0 -0
  28. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/utils/__init__.py +0 -0
  29. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/utils/bitcount.py +0 -0
  30. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/utils/mat_set.py +0 -0
  31. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/utils/row_viewable_matrix.py +0 -0
  32. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/SOURCES.txt +0 -0
  33. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/dependency_links.txt +0 -0
  34. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/entry_points.txt +0 -0
  35. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/requires.txt +0 -0
  36. {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/top_level.txt +0 -0
  37. {searcharray-0.0.45 → searcharray-0.0.46}/setup.cfg +0 -0
  38. {searcharray-0.0.45 → searcharray-0.0.46}/test/test_bitcount64.py +0 -0
  39. {searcharray-0.0.45 → searcharray-0.0.46}/test/test_extension_array.py +0 -0
  40. {searcharray-0.0.45 → searcharray-0.0.46}/test/test_minmax_posns.py +0 -0
  41. {searcharray-0.0.45 → searcharray-0.0.46}/test/test_phrase_matches.py +0 -0
  42. {searcharray-0.0.45 → searcharray-0.0.46}/test/test_search.py +0 -0
  43. {searcharray-0.0.45 → searcharray-0.0.46}/test/test_snp_ops.py +0 -0
  44. {searcharray-0.0.45 → searcharray-0.0.46}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.45
3
+ Version: 0.0.46
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -15,9 +15,10 @@ import numpy as np
15
15
  from searcharray.phrase.scan_merge import scan_merge_ins
16
16
  from searcharray.phrase.posn_diffs import compute_phrase_freqs
17
17
  from searcharray.phrase.middle_out import PosnBitArray
18
- from searcharray.similarity import Similarity, default_bm25
18
+ from searcharray.similarity import Similarity, default_bm25, ScoringContext
19
19
  from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
20
20
  from searcharray.term_dict import TermMissingError
21
+ from searcharray.roaringish.roaringish_ops import as_dense
21
22
 
22
23
  logger = logging.getLogger(__name__)
23
24
 
@@ -216,11 +217,13 @@ class SearchArray(ExtensionArray):
216
217
  if not is_list_like(postings):
217
218
  raise TypeError("Expected list-like object, got {}".format(type(postings)))
218
219
 
220
+ self.scoring_context = None
219
221
  self.avoid_copies = avoid_copies
220
222
  self.tokenizer = tokenizer
221
223
  self.term_mat, self.posns, \
222
224
  self.term_dict, self.avg_doc_length, \
223
225
  self.doc_lens = build_index_from_terms_list(postings, Terms)
226
+ self.corpus_size = len(self.doc_lens)
224
227
 
225
228
  @classmethod
226
229
  def index(cls, array: Iterable,
@@ -266,6 +269,7 @@ class SearchArray(ExtensionArray):
266
269
  postings.term_dict = term_dict
267
270
  postings.avg_doc_length = avg_doc_length
268
271
  postings.doc_lens = doc_lens
272
+ postings.corpus_size = len(doc_lens)
269
273
  return postings
270
274
 
271
275
  def warm(self):
@@ -312,13 +316,18 @@ class SearchArray(ExtensionArray):
312
316
  else:
313
317
  # Construct a sliced view of this array
314
318
  sliced_tfs = self.term_mat.slice(key)
315
- sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
319
+ if not self.avoid_copies:
320
+ sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
321
+ else:
322
+ sliced_posns = self.posns
323
+
316
324
  arr = SearchArray([], tokenizer=self.tokenizer)
317
325
  arr.term_mat = sliced_tfs
318
326
  arr.doc_lens = self.doc_lens[key]
319
327
  arr.posns = sliced_posns
320
328
  arr.term_dict = self.term_dict
321
329
  arr.avg_doc_length = self.avg_doc_length
330
+ arr.corpus_size = self.corpus_size
322
331
  return arr
323
332
 
324
333
  def __setitem__(self, key, value):
@@ -486,8 +495,11 @@ class SearchArray(ExtensionArray):
486
495
  postings_arr.posns = self.posns
487
496
  postings_arr.term_dict = self.term_dict
488
497
  postings_arr.avg_doc_length = self.avg_doc_length
498
+ postings_arr.scoring_context = self.scoring_context
499
+ postings_arr.corpus_size = self.corpus_size
489
500
 
490
501
  if not self.avoid_copies:
502
+ postings_arr.scoring_context = None
491
503
  postings_arr.posns = self.posns.copy()
492
504
  postings_arr.term_dict = self.term_dict.copy()
493
505
  return postings_arr
@@ -528,9 +540,9 @@ class SearchArray(ExtensionArray):
528
540
 
529
541
  try:
530
542
  term_id = self.term_dict.get_term_id(token)
531
- matches = np.zeros(len(self), dtype=np.float32)
532
543
  slice_of_rows = None
533
544
  if self.term_mat.subset:
545
+ matches = np.zeros(len(self), dtype=np.float32)
534
546
  slice_of_rows = self.term_mat.rows
535
547
  doc_ids, termfreqs = self.posns.termfreqs(term_id,
536
548
  doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
@@ -540,9 +552,9 @@ class SearchArray(ExtensionArray):
540
552
  else:
541
553
  doc_ids, termfreqs = self.posns.termfreqs(term_id,
542
554
  doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
543
-
544
- matches[doc_ids] = termfreqs
545
- return matches
555
+ # import pdb; pdb.set_trace()
556
+ # matches[doc_ids] = termfreqs
557
+ return as_dense(doc_ids, termfreqs, len(self))
546
558
  except TermMissingError:
547
559
  return np.zeros(len(self), dtype=np.float32)
548
560
 
@@ -588,9 +600,15 @@ class SearchArray(ExtensionArray):
588
600
  tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn)
589
601
  token = self._check_token_arg(token)
590
602
  doc_lens = self.doclengths()
591
- scores = similarity(term_freqs=tfs, doc_freqs=all_dfs,
592
- doc_lens=doc_lens, avg_doc_lens=self.avg_doc_length,
593
- num_docs=len(self))
603
+
604
+ context = ScoringContext(doc_lens=doc_lens,
605
+ avg_doc_lens=self.avg_doc_length,
606
+ num_docs=self.corpus_size)
607
+
608
+ if not context.same_as(self.scoring_context):
609
+ self.scoring_context = context
610
+
611
+ scores = similarity(tfs, all_dfs, self.scoring_context)
594
612
  return scores
595
613
 
596
614
  def positions(self, token: str, key=None) -> List[np.ndarray]: