searcharray 0.0.56__tar.gz → 0.0.58__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.56 → searcharray-0.0.58}/PKG-INFO +1 -1
- {searcharray-0.0.56 → searcharray-0.0.58}/pyproject.toml +3 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/indexing.py +1 -1
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/phrase/middle_out.py +6 -2
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/postings.py +14 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/intersect.c +1875 -1342
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/intersect.pyx +17 -22
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/merge.c +1267 -665
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/merge.pyx +0 -2
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/popcount.c +1001 -403
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/popcount.pyx +1 -2
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/roaringish_ops.c +997 -422
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/roaringish_ops.pyx +0 -1
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/search.c +1323 -732
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/search.pyx +5 -6
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/spans.c +950 -375
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/spans.pyx +4 -11
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/unique.c +1222 -617
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/unique.pyx +1 -3
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.56 → searcharray-0.0.58}/setup.py +1 -1
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_extension_array.py +8 -1
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_tmdb.py +27 -2
- {searcharray-0.0.56 → searcharray-0.0.58}/README.md +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/__init__.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/phrase/bigram_freqs.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/phrase/spans.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/__init__.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/intersect.pyi +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/merge.pyi +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/popcount.pyi +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/search.pyi +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/roaringish/unique.pyi +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/similarity.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/solr.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray/utils/sort.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/setup.cfg +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_msmarco.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_search.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_similarity.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_slop_matches.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_snp_ops.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_solr.py +0 -0
- {searcharray-0.0.56 → searcharray-0.0.58}/test/test_utils.py +0 -0
|
@@ -121,7 +121,7 @@ def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate
|
|
|
121
121
|
|
|
122
122
|
# Encode posns to bit array
|
|
123
123
|
logger.info("Encoding positions to bit array")
|
|
124
|
-
posns = PosnBitArrayFromFlatBuilder(terms_w_posns)
|
|
124
|
+
posns = PosnBitArrayFromFlatBuilder(terms_w_posns, max_doc_id=(batch_beg + len(array) - 1))
|
|
125
125
|
bit_posns = posns.build()
|
|
126
126
|
|
|
127
127
|
if np.any(doc_lens > MAX_POSN):
|
|
@@ -158,8 +158,9 @@ class PosnBitArrayFromFlatBuilder:
|
|
|
158
158
|
|
|
159
159
|
"""
|
|
160
160
|
|
|
161
|
-
def __init__(self, flat_array: np.ndarray):
|
|
161
|
+
def __init__(self, flat_array: np.ndarray, max_doc_id: int) -> None:
|
|
162
162
|
self.flat_array = flat_array
|
|
163
|
+
self.max_doc_id = max_doc_id
|
|
163
164
|
|
|
164
165
|
def build(self):
|
|
165
166
|
"""Slice the flat array into a 2d array of doc ids and posns."""
|
|
@@ -179,7 +180,9 @@ class PosnBitArrayFromFlatBuilder:
|
|
|
179
180
|
sliced = encoded[beg_idx:end_idx]
|
|
180
181
|
encoded_term_posns[term_ids[into_terms]] = sliced
|
|
181
182
|
|
|
182
|
-
|
|
183
|
+
largest_doc_id_with_term = np.max(self.flat_array[1])
|
|
184
|
+
assert self.max_doc_id >= largest_doc_id_with_term
|
|
185
|
+
return PosnBitArray(encoded_term_posns, self.max_doc_id)
|
|
183
186
|
|
|
184
187
|
|
|
185
188
|
class PosnBitArrayBuilder:
|
|
@@ -390,6 +393,7 @@ class PosnBitArray:
|
|
|
390
393
|
min_posn: Optional[int] = None,
|
|
391
394
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
392
395
|
phrase_freqs = self.empty_buffer()
|
|
396
|
+
|
|
393
397
|
if len(term_ids) < 2:
|
|
394
398
|
raise ValueError("Must have at least two terms")
|
|
395
399
|
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
|
|
@@ -464,6 +464,20 @@ class SearchArray(ExtensionArray):
|
|
|
464
464
|
empties = self.doc_lens == 0
|
|
465
465
|
return empties
|
|
466
466
|
|
|
467
|
+
def unique(self):
|
|
468
|
+
"""This is a hack for colab which wants to visit every element of the array."""
|
|
469
|
+
logger.warning("Unique called on SearchArray. This is not supported.")
|
|
470
|
+
return self[:]
|
|
471
|
+
|
|
472
|
+
def __iter__(self):
|
|
473
|
+
if len(self) > 10000:
|
|
474
|
+
warning_text = """Iterating over SearchArray is very slow and not reccomended.
|
|
475
|
+
|
|
476
|
+
If you're looping a dataframe, slice out the non-SearchArray columns first."""
|
|
477
|
+
logger.warning(warning_text)
|
|
478
|
+
warnings.warn(warning_text)
|
|
479
|
+
return super().__iter__()
|
|
480
|
+
|
|
467
481
|
def take(self, indices, allow_fill=False, fill_value=None):
|
|
468
482
|
# Want to take rows of term freqs
|
|
469
483
|
row_indices = np.arange(len(self.term_mat.rows))
|