searcharray 0.0.44__tar.gz → 0.0.45__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.44 → searcharray-0.0.45}/PKG-INFO +1 -1
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/phrase/middle_out.py +6 -4
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/roaringish.py +24 -20
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/roaringish_ops.c +740 -124
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/roaringish_ops.pyx +25 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/snp_ops.c +117 -117
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.44 → searcharray-0.0.45}/setup.py +1 -1
- {searcharray-0.0.44 → searcharray-0.0.45}/test/test_msmarco.py +32 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/README.md +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/pyproject.toml +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/__init__.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/indexing.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/phrase/bigram_freqs.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/postings.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/__init__.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/roaringish/snp_ops.pyx +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/similarity.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/solr.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/setup.cfg +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/test/test_extension_array.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/test/test_search.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/test/test_snp_ops.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/test/test_solr.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/test/test_tmdb.py +0 -0
- {searcharray-0.0.44 → searcharray-0.0.45}/test/test_utils.py +0 -0
|
@@ -344,8 +344,12 @@ class PosnBitArray:
|
|
|
344
344
|
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
|
|
345
345
|
enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
|
|
346
346
|
else:
|
|
347
|
+
keys = None
|
|
348
|
+
if phrase_freqs.shape[0] != self.max_doc_id + 1:
|
|
349
|
+
keys = doc_ids.view(np.uint64)
|
|
350
|
+
|
|
347
351
|
enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
|
|
348
|
-
keys=
|
|
352
|
+
keys=keys,
|
|
349
353
|
min_payload=min_posn,
|
|
350
354
|
max_payload=max_posn) for term_id in term_ids]
|
|
351
355
|
return compute_phrase_freqs(enc_term_posns, phrase_freqs)
|
|
@@ -390,13 +394,11 @@ class PosnBitArray:
|
|
|
390
394
|
"""Count term freqs using unique positions."""
|
|
391
395
|
if doc_ids is None and max_posn is None and min_posn is None:
|
|
392
396
|
return self._termfreqs_with_cache(term_id)
|
|
393
|
-
if doc_ids is None:
|
|
394
|
-
doc_ids = encoder.keys_unique(self.encoded_term_posns[term_id])
|
|
395
397
|
|
|
396
398
|
encoded = self.encoded_term_posns[term_id]
|
|
397
399
|
term_posns = encoded
|
|
398
400
|
term_posns = encoder.slice(encoded,
|
|
399
|
-
keys=doc_ids
|
|
401
|
+
keys=doc_ids,
|
|
400
402
|
min_payload=min_posn,
|
|
401
403
|
max_payload=max_posn)
|
|
402
404
|
|
|
@@ -8,7 +8,7 @@ import numbers
|
|
|
8
8
|
from typing import Optional, Tuple, List, Union
|
|
9
9
|
|
|
10
10
|
from searcharray.roaringish.snp_ops import intersect, unique, adjacent, merge
|
|
11
|
-
from searcharray.roaringish.roaringish_ops import popcount64_reduce
|
|
11
|
+
from searcharray.roaringish.roaringish_ops import popcount64_reduce, payload_slice
|
|
12
12
|
|
|
13
13
|
logger = logging.getLogger(__name__)
|
|
14
14
|
|
|
@@ -210,31 +210,35 @@ class RoaringishEncoder:
|
|
|
210
210
|
lhs_idx, rhs_idx = intersect(lhs, rhs, mask=self.header_mask)
|
|
211
211
|
return lhs[lhs_idx], rhs[rhs_idx]
|
|
212
212
|
|
|
213
|
-
def slice(self,
|
|
214
|
-
|
|
213
|
+
def slice(self,
|
|
214
|
+
encoded: np.ndarray,
|
|
215
|
+
keys: Optional[np.ndarray] = None,
|
|
215
216
|
max_payload: Optional[int] = None,
|
|
216
217
|
min_payload: Optional[int] = None) -> np.ndarray:
|
|
217
218
|
"""Get list of encoded that have values in keys."""
|
|
218
219
|
# encoded_keys = encoded.view(np.uint64) >> (_64 - self.key_bits)
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
220
|
+
if keys is not None:
|
|
221
|
+
encoded_keys = self.keys(encoded)
|
|
222
|
+
idx_docs, idx_enc = intersect(keys.view(np.uint64),
|
|
223
|
+
encoded_keys.view(np.uint64),
|
|
224
|
+
drop_duplicates=False)
|
|
225
|
+
encoded = encoded[idx_enc]
|
|
222
226
|
if max_payload is None and min_payload is None:
|
|
223
|
-
return encoded
|
|
227
|
+
return encoded
|
|
224
228
|
else:
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
if
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
return
|
|
229
|
+
if min_payload is not None and min_payload % self.payload_lsb_bits != 0:
|
|
230
|
+
raise ValueError(f"min_payload must be a multiple of {self.payload_lsb_bits}")
|
|
231
|
+
if max_payload is not None and max_payload % self.payload_lsb_bits != self.payload_lsb_bits - 1:
|
|
232
|
+
raise ValueError(f"max_payload must be a multiple of {self.payload_lsb_bits} - 1")
|
|
233
|
+
min_payload = 0 if min_payload is None else min_payload
|
|
234
|
+
max_payload = 0xFFFFFFFFFFFFFFFF if max_payload is None else max_payload
|
|
235
|
+
|
|
236
|
+
def payl_slice():
|
|
237
|
+
return payload_slice(encoded, self.payload_msb_mask,
|
|
238
|
+
min_payload // self.payload_lsb_bits,
|
|
239
|
+
max_payload // self.payload_lsb_bits)
|
|
240
|
+
|
|
241
|
+
return payl_slice()
|
|
238
242
|
|
|
239
243
|
|
|
240
244
|
def convert_keys(keys) -> np.ndarray:
|