searcharray 0.0.43__tar.gz → 0.0.45__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.43 → searcharray-0.0.45}/PKG-INFO +1 -1
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/phrase/bigram_freqs.py +2 -2
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/phrase/middle_out.py +9 -16
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/postings.py +4 -2
- searcharray-0.0.45/searcharray/roaringish/__init__.py +3 -0
- {searcharray-0.0.43/searcharray/utils → searcharray-0.0.45/searcharray/roaringish}/roaringish.py +29 -20
- searcharray-0.0.45/searcharray/roaringish/roaringish_ops.c +30895 -0
- searcharray-0.0.45/searcharray/roaringish/roaringish_ops.pyx +80 -0
- {searcharray-0.0.43/searcharray/utils → searcharray-0.0.45/searcharray/roaringish}/snp_ops.c +1614 -2116
- searcharray-0.0.45/searcharray/roaringish/snp_ops.pxd +22 -0
- {searcharray-0.0.43/searcharray/utils → searcharray-0.0.45/searcharray/roaringish}/snp_ops.pyx +5 -19
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/SOURCES.txt +7 -3
- {searcharray-0.0.43 → searcharray-0.0.45}/setup.py +6 -4
- {searcharray-0.0.43 → searcharray-0.0.45}/test/test_bitcount64.py +1 -1
- {searcharray-0.0.43 → searcharray-0.0.45}/test/test_msmarco.py +34 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/test/test_snp_ops.py +1 -1
- {searcharray-0.0.43 → searcharray-0.0.45}/README.md +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/pyproject.toml +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/__init__.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/indexing.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/similarity.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/solr.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/setup.cfg +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/test/test_extension_array.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/test/test_search.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/test/test_solr.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/test/test_tmdb.py +0 -0
- {searcharray-0.0.43 → searcharray-0.0.45}/test/test_utils.py +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
"""Given two roaringish encoded arrays, count bigram matches between them."""
|
|
2
2
|
import numpy as np
|
|
3
3
|
from typing import Tuple, Optional
|
|
4
|
-
from searcharray.
|
|
4
|
+
from searcharray.roaringish import RoaringishEncoder
|
|
5
5
|
import logging
|
|
6
6
|
from enum import Enum
|
|
7
7
|
|
|
8
|
-
from searcharray.
|
|
8
|
+
from searcharray.roaringish import intersect, popcount64, merge
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
logger = logging.getLogger(__name__)
|
|
@@ -8,14 +8,12 @@ https://colab.research.google.com/drive/10tIEkdlCE_1J_CcgEcV0jkLfBc-0H4am?authus
|
|
|
8
8
|
import numpy as np
|
|
9
9
|
from copy import deepcopy
|
|
10
10
|
from typing import List, Tuple, Dict, Union, cast, Optional
|
|
11
|
-
from searcharray.
|
|
11
|
+
from searcharray.roaringish import RoaringishEncoder, convert_keys, merge
|
|
12
12
|
from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
|
|
13
13
|
import numbers
|
|
14
14
|
import logging
|
|
15
15
|
from collections import defaultdict
|
|
16
16
|
|
|
17
|
-
from searcharray.utils.snp_ops import popcount64, merge
|
|
18
|
-
|
|
19
17
|
|
|
20
18
|
logger = logging.getLogger(__name__)
|
|
21
19
|
|
|
@@ -346,8 +344,12 @@ class PosnBitArray:
|
|
|
346
344
|
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
|
|
347
345
|
enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
|
|
348
346
|
else:
|
|
347
|
+
keys = None
|
|
348
|
+
if phrase_freqs.shape[0] != self.max_doc_id + 1:
|
|
349
|
+
keys = doc_ids.view(np.uint64)
|
|
350
|
+
|
|
349
351
|
enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
|
|
350
|
-
keys=
|
|
352
|
+
keys=keys,
|
|
351
353
|
min_payload=min_posn,
|
|
352
354
|
max_payload=max_posn) for term_id in term_ids]
|
|
353
355
|
return compute_phrase_freqs(enc_term_posns, phrase_freqs)
|
|
@@ -392,27 +394,18 @@ class PosnBitArray:
|
|
|
392
394
|
"""Count term freqs using unique positions."""
|
|
393
395
|
if doc_ids is None and max_posn is None and min_posn is None:
|
|
394
396
|
return self._termfreqs_with_cache(term_id)
|
|
395
|
-
if doc_ids is None:
|
|
396
|
-
doc_ids = encoder.keys_unique(self.encoded_term_posns[term_id])
|
|
397
397
|
|
|
398
398
|
encoded = self.encoded_term_posns[term_id]
|
|
399
399
|
term_posns = encoded
|
|
400
400
|
term_posns = encoder.slice(encoded,
|
|
401
|
-
keys=doc_ids
|
|
401
|
+
keys=doc_ids,
|
|
402
402
|
min_payload=min_posn,
|
|
403
403
|
max_payload=max_posn)
|
|
404
404
|
|
|
405
405
|
return self._computed_term_freqs(term_posns)
|
|
406
406
|
|
|
407
407
|
def _computed_term_freqs(self, term_posns) -> Tuple[np.ndarray, np.ndarray]:
|
|
408
|
-
|
|
409
|
-
change_indices = np.nonzero(np.diff(doc_ids))[0]
|
|
410
|
-
change_indices = np.concatenate((np.asarray([0]), change_indices + 1))
|
|
411
|
-
posns = term_posns & encoder.payload_lsb_mask
|
|
412
|
-
bit_counts = popcount64(posns)
|
|
413
|
-
|
|
414
|
-
term_freqs = np.add.reduceat(bit_counts, change_indices, dtype=np.float32)
|
|
415
|
-
return sorted_unique(doc_ids), term_freqs
|
|
408
|
+
return encoder.num_values_per_key(term_posns)
|
|
416
409
|
|
|
417
410
|
def _termfreqs_with_cache(self, term_id: int) -> Tuple[np.ndarray, np.ndarray]:
|
|
418
411
|
try:
|
|
@@ -431,7 +424,7 @@ class PosnBitArray:
|
|
|
431
424
|
return self.docfreq_cache[term_id]
|
|
432
425
|
|
|
433
426
|
def _maybe_cache_docfreq(self, term_id: int, docfreq: np.uint64):
|
|
434
|
-
if self.max_doc_id >=
|
|
427
|
+
if self.max_doc_id >= 99999 and docfreq > (self.max_doc_id // 100):
|
|
435
428
|
self.docfreq_cache[term_id] = docfreq
|
|
436
429
|
|
|
437
430
|
def docfreq(self, term_id: int) -> np.uint64:
|
|
@@ -567,7 +567,9 @@ class SearchArray(ExtensionArray):
|
|
|
567
567
|
term_freq = self.termfreqs(token)
|
|
568
568
|
return term_freq > 0
|
|
569
569
|
|
|
570
|
-
def score(self, token: Union[str, List[str]], similarity: Similarity = default_bm25
|
|
570
|
+
def score(self, token: Union[str, List[str]], similarity: Similarity = default_bm25,
|
|
571
|
+
min_posn: Optional[int] = None,
|
|
572
|
+
max_posn: Optional[int] = None) -> np.ndarray:
|
|
571
573
|
"""Score each doc using a similarity function.
|
|
572
574
|
|
|
573
575
|
Parameters
|
|
@@ -583,7 +585,7 @@ class SearchArray(ExtensionArray):
|
|
|
583
585
|
tokens_l = [token] if isinstance(token, str) else token
|
|
584
586
|
all_dfs = np.asarray([self.docfreq(token) for token in tokens_l])
|
|
585
587
|
|
|
586
|
-
tfs = self.termfreqs(token)
|
|
588
|
+
tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn)
|
|
587
589
|
token = self._check_token_arg(token)
|
|
588
590
|
doc_lens = self.doclengths()
|
|
589
591
|
scores = similarity(term_freqs=tfs, doc_freqs=all_dfs,
|
{searcharray-0.0.43/searcharray/utils → searcharray-0.0.45/searcharray/roaringish}/roaringish.py
RENAMED
|
@@ -7,7 +7,8 @@ import logging
|
|
|
7
7
|
import numbers
|
|
8
8
|
from typing import Optional, Tuple, List, Union
|
|
9
9
|
|
|
10
|
-
from searcharray.
|
|
10
|
+
from searcharray.roaringish.snp_ops import intersect, unique, adjacent, merge
|
|
11
|
+
from searcharray.roaringish.roaringish_ops import popcount64_reduce, payload_slice
|
|
11
12
|
|
|
12
13
|
logger = logging.getLogger(__name__)
|
|
13
14
|
|
|
@@ -157,6 +158,10 @@ class RoaringishEncoder:
|
|
|
157
158
|
else:
|
|
158
159
|
return grouped
|
|
159
160
|
|
|
161
|
+
def num_values_per_key(self, encoded: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
|
162
|
+
pcr = popcount64_reduce(encoded, (_64 - self.key_bits), self.payload_lsb_mask)
|
|
163
|
+
return pcr
|
|
164
|
+
|
|
160
165
|
def keys(self, encoded: np.ndarray) -> np.ndarray:
|
|
161
166
|
"""Return keys from encoded."""
|
|
162
167
|
return encoded >> (_64 - self.key_bits)
|
|
@@ -205,31 +210,35 @@ class RoaringishEncoder:
|
|
|
205
210
|
lhs_idx, rhs_idx = intersect(lhs, rhs, mask=self.header_mask)
|
|
206
211
|
return lhs[lhs_idx], rhs[rhs_idx]
|
|
207
212
|
|
|
208
|
-
def slice(self,
|
|
209
|
-
|
|
213
|
+
def slice(self,
|
|
214
|
+
encoded: np.ndarray,
|
|
215
|
+
keys: Optional[np.ndarray] = None,
|
|
210
216
|
max_payload: Optional[int] = None,
|
|
211
217
|
min_payload: Optional[int] = None) -> np.ndarray:
|
|
212
218
|
"""Get list of encoded that have values in keys."""
|
|
213
219
|
# encoded_keys = encoded.view(np.uint64) >> (_64 - self.key_bits)
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
220
|
+
if keys is not None:
|
|
221
|
+
encoded_keys = self.keys(encoded)
|
|
222
|
+
idx_docs, idx_enc = intersect(keys.view(np.uint64),
|
|
223
|
+
encoded_keys.view(np.uint64),
|
|
224
|
+
drop_duplicates=False)
|
|
225
|
+
encoded = encoded[idx_enc]
|
|
217
226
|
if max_payload is None and min_payload is None:
|
|
218
|
-
return encoded
|
|
227
|
+
return encoded
|
|
219
228
|
else:
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
if
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
return
|
|
229
|
+
if min_payload is not None and min_payload % self.payload_lsb_bits != 0:
|
|
230
|
+
raise ValueError(f"min_payload must be a multiple of {self.payload_lsb_bits}")
|
|
231
|
+
if max_payload is not None and max_payload % self.payload_lsb_bits != self.payload_lsb_bits - 1:
|
|
232
|
+
raise ValueError(f"max_payload must be a multiple of {self.payload_lsb_bits} - 1")
|
|
233
|
+
min_payload = 0 if min_payload is None else min_payload
|
|
234
|
+
max_payload = 0xFFFFFFFFFFFFFFFF if max_payload is None else max_payload
|
|
235
|
+
|
|
236
|
+
def payl_slice():
|
|
237
|
+
return payload_slice(encoded, self.payload_msb_mask,
|
|
238
|
+
min_payload // self.payload_lsb_bits,
|
|
239
|
+
max_payload // self.payload_lsb_bits)
|
|
240
|
+
|
|
241
|
+
return payl_slice()
|
|
233
242
|
|
|
234
243
|
|
|
235
244
|
def convert_keys(keys) -> np.ndarray:
|