searcharray 0.0.45__tar.gz → 0.0.46__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.45 → searcharray-0.0.46}/PKG-INFO +1 -1
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/postings.py +27 -9
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/roaringish_ops.c +819 -235
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/roaringish_ops.pyx +26 -2
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/snp_ops.c +117 -117
- searcharray-0.0.46/searcharray/similarity.py +102 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.45 → searcharray-0.0.46}/setup.py +1 -1
- {searcharray-0.0.45 → searcharray-0.0.46}/test/test_msmarco.py +7 -4
- {searcharray-0.0.45 → searcharray-0.0.46}/test/test_solr.py +3 -2
- {searcharray-0.0.45 → searcharray-0.0.46}/test/test_tmdb.py +4 -1
- searcharray-0.0.45/searcharray/similarity.py +0 -82
- {searcharray-0.0.45 → searcharray-0.0.46}/README.md +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/pyproject.toml +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/__init__.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/indexing.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/phrase/bigram_freqs.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/phrase/middle_out.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/__init__.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/roaringish/snp_ops.pyx +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/solr.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/setup.cfg +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/test/test_extension_array.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/test/test_search.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/test/test_snp_ops.py +0 -0
- {searcharray-0.0.45 → searcharray-0.0.46}/test/test_utils.py +0 -0
|
@@ -15,9 +15,10 @@ import numpy as np
|
|
|
15
15
|
from searcharray.phrase.scan_merge import scan_merge_ins
|
|
16
16
|
from searcharray.phrase.posn_diffs import compute_phrase_freqs
|
|
17
17
|
from searcharray.phrase.middle_out import PosnBitArray
|
|
18
|
-
from searcharray.similarity import Similarity, default_bm25
|
|
18
|
+
from searcharray.similarity import Similarity, default_bm25, ScoringContext
|
|
19
19
|
from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
|
|
20
20
|
from searcharray.term_dict import TermMissingError
|
|
21
|
+
from searcharray.roaringish.roaringish_ops import as_dense
|
|
21
22
|
|
|
22
23
|
logger = logging.getLogger(__name__)
|
|
23
24
|
|
|
@@ -216,11 +217,13 @@ class SearchArray(ExtensionArray):
|
|
|
216
217
|
if not is_list_like(postings):
|
|
217
218
|
raise TypeError("Expected list-like object, got {}".format(type(postings)))
|
|
218
219
|
|
|
220
|
+
self.scoring_context = None
|
|
219
221
|
self.avoid_copies = avoid_copies
|
|
220
222
|
self.tokenizer = tokenizer
|
|
221
223
|
self.term_mat, self.posns, \
|
|
222
224
|
self.term_dict, self.avg_doc_length, \
|
|
223
225
|
self.doc_lens = build_index_from_terms_list(postings, Terms)
|
|
226
|
+
self.corpus_size = len(self.doc_lens)
|
|
224
227
|
|
|
225
228
|
@classmethod
|
|
226
229
|
def index(cls, array: Iterable,
|
|
@@ -266,6 +269,7 @@ class SearchArray(ExtensionArray):
|
|
|
266
269
|
postings.term_dict = term_dict
|
|
267
270
|
postings.avg_doc_length = avg_doc_length
|
|
268
271
|
postings.doc_lens = doc_lens
|
|
272
|
+
postings.corpus_size = len(doc_lens)
|
|
269
273
|
return postings
|
|
270
274
|
|
|
271
275
|
def warm(self):
|
|
@@ -312,13 +316,18 @@ class SearchArray(ExtensionArray):
|
|
|
312
316
|
else:
|
|
313
317
|
# Construct a sliced view of this array
|
|
314
318
|
sliced_tfs = self.term_mat.slice(key)
|
|
315
|
-
|
|
319
|
+
if not self.avoid_copies:
|
|
320
|
+
sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
|
|
321
|
+
else:
|
|
322
|
+
sliced_posns = self.posns
|
|
323
|
+
|
|
316
324
|
arr = SearchArray([], tokenizer=self.tokenizer)
|
|
317
325
|
arr.term_mat = sliced_tfs
|
|
318
326
|
arr.doc_lens = self.doc_lens[key]
|
|
319
327
|
arr.posns = sliced_posns
|
|
320
328
|
arr.term_dict = self.term_dict
|
|
321
329
|
arr.avg_doc_length = self.avg_doc_length
|
|
330
|
+
arr.corpus_size = self.corpus_size
|
|
322
331
|
return arr
|
|
323
332
|
|
|
324
333
|
def __setitem__(self, key, value):
|
|
@@ -486,8 +495,11 @@ class SearchArray(ExtensionArray):
|
|
|
486
495
|
postings_arr.posns = self.posns
|
|
487
496
|
postings_arr.term_dict = self.term_dict
|
|
488
497
|
postings_arr.avg_doc_length = self.avg_doc_length
|
|
498
|
+
postings_arr.scoring_context = self.scoring_context
|
|
499
|
+
postings_arr.corpus_size = self.corpus_size
|
|
489
500
|
|
|
490
501
|
if not self.avoid_copies:
|
|
502
|
+
postings_arr.scoring_context = None
|
|
491
503
|
postings_arr.posns = self.posns.copy()
|
|
492
504
|
postings_arr.term_dict = self.term_dict.copy()
|
|
493
505
|
return postings_arr
|
|
@@ -528,9 +540,9 @@ class SearchArray(ExtensionArray):
|
|
|
528
540
|
|
|
529
541
|
try:
|
|
530
542
|
term_id = self.term_dict.get_term_id(token)
|
|
531
|
-
matches = np.zeros(len(self), dtype=np.float32)
|
|
532
543
|
slice_of_rows = None
|
|
533
544
|
if self.term_mat.subset:
|
|
545
|
+
matches = np.zeros(len(self), dtype=np.float32)
|
|
534
546
|
slice_of_rows = self.term_mat.rows
|
|
535
547
|
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
536
548
|
doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
|
|
@@ -540,9 +552,9 @@ class SearchArray(ExtensionArray):
|
|
|
540
552
|
else:
|
|
541
553
|
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
542
554
|
doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
|
|
543
|
-
|
|
544
|
-
matches[doc_ids] = termfreqs
|
|
545
|
-
return
|
|
555
|
+
# import pdb; pdb.set_trace()
|
|
556
|
+
# matches[doc_ids] = termfreqs
|
|
557
|
+
return as_dense(doc_ids, termfreqs, len(self))
|
|
546
558
|
except TermMissingError:
|
|
547
559
|
return np.zeros(len(self), dtype=np.float32)
|
|
548
560
|
|
|
@@ -588,9 +600,15 @@ class SearchArray(ExtensionArray):
|
|
|
588
600
|
tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn)
|
|
589
601
|
token = self._check_token_arg(token)
|
|
590
602
|
doc_lens = self.doclengths()
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
603
|
+
|
|
604
|
+
context = ScoringContext(doc_lens=doc_lens,
|
|
605
|
+
avg_doc_lens=self.avg_doc_length,
|
|
606
|
+
num_docs=self.corpus_size)
|
|
607
|
+
|
|
608
|
+
if not context.same_as(self.scoring_context):
|
|
609
|
+
self.scoring_context = context
|
|
610
|
+
|
|
611
|
+
scores = similarity(tfs, all_dfs, self.scoring_context)
|
|
594
612
|
return scores
|
|
595
613
|
|
|
596
614
|
def positions(self, token: str, key=None) -> List[np.ndarray]:
|