searcharray 0.0.44__tar.gz → 0.0.46__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.44 → searcharray-0.0.46}/PKG-INFO +1 -1
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/phrase/middle_out.py +6 -4
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/postings.py +27 -9
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/roaringish/roaringish.py +24 -20
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/roaringish/roaringish_ops.c +1585 -385
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/roaringish/roaringish_ops.pyx +51 -2
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/roaringish/snp_ops.c +117 -117
- searcharray-0.0.46/searcharray/similarity.py +102 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.44 → searcharray-0.0.46}/setup.py +1 -1
- {searcharray-0.0.44 → searcharray-0.0.46}/test/test_msmarco.py +38 -3
- {searcharray-0.0.44 → searcharray-0.0.46}/test/test_solr.py +3 -2
- {searcharray-0.0.44 → searcharray-0.0.46}/test/test_tmdb.py +4 -1
- searcharray-0.0.44/searcharray/similarity.py +0 -82
- {searcharray-0.0.44 → searcharray-0.0.46}/README.md +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/pyproject.toml +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/__init__.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/indexing.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/phrase/bigram_freqs.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/roaringish/__init__.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/roaringish/snp_ops.pyx +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/solr.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/setup.cfg +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/test/test_extension_array.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/test/test_search.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/test/test_snp_ops.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.46}/test/test_utils.py +0 -0
|
@@ -344,8 +344,12 @@ class PosnBitArray:
|
|
|
344
344
|
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
|
|
345
345
|
enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
|
|
346
346
|
else:
|
|
347
|
+
keys = None
|
|
348
|
+
if phrase_freqs.shape[0] != self.max_doc_id + 1:
|
|
349
|
+
keys = doc_ids.view(np.uint64)
|
|
350
|
+
|
|
347
351
|
enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
|
|
348
|
-
keys=
|
|
352
|
+
keys=keys,
|
|
349
353
|
min_payload=min_posn,
|
|
350
354
|
max_payload=max_posn) for term_id in term_ids]
|
|
351
355
|
return compute_phrase_freqs(enc_term_posns, phrase_freqs)
|
|
@@ -390,13 +394,11 @@ class PosnBitArray:
|
|
|
390
394
|
"""Count term freqs using unique positions."""
|
|
391
395
|
if doc_ids is None and max_posn is None and min_posn is None:
|
|
392
396
|
return self._termfreqs_with_cache(term_id)
|
|
393
|
-
if doc_ids is None:
|
|
394
|
-
doc_ids = encoder.keys_unique(self.encoded_term_posns[term_id])
|
|
395
397
|
|
|
396
398
|
encoded = self.encoded_term_posns[term_id]
|
|
397
399
|
term_posns = encoded
|
|
398
400
|
term_posns = encoder.slice(encoded,
|
|
399
|
-
keys=doc_ids
|
|
401
|
+
keys=doc_ids,
|
|
400
402
|
min_payload=min_posn,
|
|
401
403
|
max_payload=max_posn)
|
|
402
404
|
|
|
@@ -15,9 +15,10 @@ import numpy as np
|
|
|
15
15
|
from searcharray.phrase.scan_merge import scan_merge_ins
|
|
16
16
|
from searcharray.phrase.posn_diffs import compute_phrase_freqs
|
|
17
17
|
from searcharray.phrase.middle_out import PosnBitArray
|
|
18
|
-
from searcharray.similarity import Similarity, default_bm25
|
|
18
|
+
from searcharray.similarity import Similarity, default_bm25, ScoringContext
|
|
19
19
|
from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
|
|
20
20
|
from searcharray.term_dict import TermMissingError
|
|
21
|
+
from searcharray.roaringish.roaringish_ops import as_dense
|
|
21
22
|
|
|
22
23
|
logger = logging.getLogger(__name__)
|
|
23
24
|
|
|
@@ -216,11 +217,13 @@ class SearchArray(ExtensionArray):
|
|
|
216
217
|
if not is_list_like(postings):
|
|
217
218
|
raise TypeError("Expected list-like object, got {}".format(type(postings)))
|
|
218
219
|
|
|
220
|
+
self.scoring_context = None
|
|
219
221
|
self.avoid_copies = avoid_copies
|
|
220
222
|
self.tokenizer = tokenizer
|
|
221
223
|
self.term_mat, self.posns, \
|
|
222
224
|
self.term_dict, self.avg_doc_length, \
|
|
223
225
|
self.doc_lens = build_index_from_terms_list(postings, Terms)
|
|
226
|
+
self.corpus_size = len(self.doc_lens)
|
|
224
227
|
|
|
225
228
|
@classmethod
|
|
226
229
|
def index(cls, array: Iterable,
|
|
@@ -266,6 +269,7 @@ class SearchArray(ExtensionArray):
|
|
|
266
269
|
postings.term_dict = term_dict
|
|
267
270
|
postings.avg_doc_length = avg_doc_length
|
|
268
271
|
postings.doc_lens = doc_lens
|
|
272
|
+
postings.corpus_size = len(doc_lens)
|
|
269
273
|
return postings
|
|
270
274
|
|
|
271
275
|
def warm(self):
|
|
@@ -312,13 +316,18 @@ class SearchArray(ExtensionArray):
|
|
|
312
316
|
else:
|
|
313
317
|
# Construct a sliced view of this array
|
|
314
318
|
sliced_tfs = self.term_mat.slice(key)
|
|
315
|
-
|
|
319
|
+
if not self.avoid_copies:
|
|
320
|
+
sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
|
|
321
|
+
else:
|
|
322
|
+
sliced_posns = self.posns
|
|
323
|
+
|
|
316
324
|
arr = SearchArray([], tokenizer=self.tokenizer)
|
|
317
325
|
arr.term_mat = sliced_tfs
|
|
318
326
|
arr.doc_lens = self.doc_lens[key]
|
|
319
327
|
arr.posns = sliced_posns
|
|
320
328
|
arr.term_dict = self.term_dict
|
|
321
329
|
arr.avg_doc_length = self.avg_doc_length
|
|
330
|
+
arr.corpus_size = self.corpus_size
|
|
322
331
|
return arr
|
|
323
332
|
|
|
324
333
|
def __setitem__(self, key, value):
|
|
@@ -486,8 +495,11 @@ class SearchArray(ExtensionArray):
|
|
|
486
495
|
postings_arr.posns = self.posns
|
|
487
496
|
postings_arr.term_dict = self.term_dict
|
|
488
497
|
postings_arr.avg_doc_length = self.avg_doc_length
|
|
498
|
+
postings_arr.scoring_context = self.scoring_context
|
|
499
|
+
postings_arr.corpus_size = self.corpus_size
|
|
489
500
|
|
|
490
501
|
if not self.avoid_copies:
|
|
502
|
+
postings_arr.scoring_context = None
|
|
491
503
|
postings_arr.posns = self.posns.copy()
|
|
492
504
|
postings_arr.term_dict = self.term_dict.copy()
|
|
493
505
|
return postings_arr
|
|
@@ -528,9 +540,9 @@ class SearchArray(ExtensionArray):
|
|
|
528
540
|
|
|
529
541
|
try:
|
|
530
542
|
term_id = self.term_dict.get_term_id(token)
|
|
531
|
-
matches = np.zeros(len(self), dtype=np.float32)
|
|
532
543
|
slice_of_rows = None
|
|
533
544
|
if self.term_mat.subset:
|
|
545
|
+
matches = np.zeros(len(self), dtype=np.float32)
|
|
534
546
|
slice_of_rows = self.term_mat.rows
|
|
535
547
|
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
536
548
|
doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
|
|
@@ -540,9 +552,9 @@ class SearchArray(ExtensionArray):
|
|
|
540
552
|
else:
|
|
541
553
|
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
542
554
|
doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
|
|
543
|
-
|
|
544
|
-
matches[doc_ids] = termfreqs
|
|
545
|
-
return
|
|
555
|
+
# import pdb; pdb.set_trace()
|
|
556
|
+
# matches[doc_ids] = termfreqs
|
|
557
|
+
return as_dense(doc_ids, termfreqs, len(self))
|
|
546
558
|
except TermMissingError:
|
|
547
559
|
return np.zeros(len(self), dtype=np.float32)
|
|
548
560
|
|
|
@@ -588,9 +600,15 @@ class SearchArray(ExtensionArray):
|
|
|
588
600
|
tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn)
|
|
589
601
|
token = self._check_token_arg(token)
|
|
590
602
|
doc_lens = self.doclengths()
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
603
|
+
|
|
604
|
+
context = ScoringContext(doc_lens=doc_lens,
|
|
605
|
+
avg_doc_lens=self.avg_doc_length,
|
|
606
|
+
num_docs=self.corpus_size)
|
|
607
|
+
|
|
608
|
+
if not context.same_as(self.scoring_context):
|
|
609
|
+
self.scoring_context = context
|
|
610
|
+
|
|
611
|
+
scores = similarity(tfs, all_dfs, self.scoring_context)
|
|
594
612
|
return scores
|
|
595
613
|
|
|
596
614
|
def positions(self, token: str, key=None) -> List[np.ndarray]:
|
|
@@ -8,7 +8,7 @@ import numbers
|
|
|
8
8
|
from typing import Optional, Tuple, List, Union
|
|
9
9
|
|
|
10
10
|
from searcharray.roaringish.snp_ops import intersect, unique, adjacent, merge
|
|
11
|
-
from searcharray.roaringish.roaringish_ops import popcount64_reduce
|
|
11
|
+
from searcharray.roaringish.roaringish_ops import popcount64_reduce, payload_slice
|
|
12
12
|
|
|
13
13
|
logger = logging.getLogger(__name__)
|
|
14
14
|
|
|
@@ -210,31 +210,35 @@ class RoaringishEncoder:
|
|
|
210
210
|
lhs_idx, rhs_idx = intersect(lhs, rhs, mask=self.header_mask)
|
|
211
211
|
return lhs[lhs_idx], rhs[rhs_idx]
|
|
212
212
|
|
|
213
|
-
def slice(self,
|
|
214
|
-
|
|
213
|
+
def slice(self,
|
|
214
|
+
encoded: np.ndarray,
|
|
215
|
+
keys: Optional[np.ndarray] = None,
|
|
215
216
|
max_payload: Optional[int] = None,
|
|
216
217
|
min_payload: Optional[int] = None) -> np.ndarray:
|
|
217
218
|
"""Get list of encoded that have values in keys."""
|
|
218
219
|
# encoded_keys = encoded.view(np.uint64) >> (_64 - self.key_bits)
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
220
|
+
if keys is not None:
|
|
221
|
+
encoded_keys = self.keys(encoded)
|
|
222
|
+
idx_docs, idx_enc = intersect(keys.view(np.uint64),
|
|
223
|
+
encoded_keys.view(np.uint64),
|
|
224
|
+
drop_duplicates=False)
|
|
225
|
+
encoded = encoded[idx_enc]
|
|
222
226
|
if max_payload is None and min_payload is None:
|
|
223
|
-
return encoded
|
|
227
|
+
return encoded
|
|
224
228
|
else:
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
if
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
return
|
|
229
|
+
if min_payload is not None and min_payload % self.payload_lsb_bits != 0:
|
|
230
|
+
raise ValueError(f"min_payload must be a multiple of {self.payload_lsb_bits}")
|
|
231
|
+
if max_payload is not None and max_payload % self.payload_lsb_bits != self.payload_lsb_bits - 1:
|
|
232
|
+
raise ValueError(f"max_payload must be a multiple of {self.payload_lsb_bits} - 1")
|
|
233
|
+
min_payload = 0 if min_payload is None else min_payload
|
|
234
|
+
max_payload = 0xFFFFFFFFFFFFFFFF if max_payload is None else max_payload
|
|
235
|
+
|
|
236
|
+
def payl_slice():
|
|
237
|
+
return payload_slice(encoded, self.payload_msb_mask,
|
|
238
|
+
min_payload // self.payload_lsb_bits,
|
|
239
|
+
max_payload // self.payload_lsb_bits)
|
|
240
|
+
|
|
241
|
+
return payl_slice()
|
|
238
242
|
|
|
239
243
|
|
|
240
244
|
def convert_keys(keys) -> np.ndarray:
|