searcharray 0.0.57__tar.gz → 0.0.58__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.57 → searcharray-0.0.58}/PKG-INFO +1 -1
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/indexing.py +1 -1
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/phrase/middle_out.py +6 -2
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/postings.py +9 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/intersect.c +1666 -1113
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/merge.c +949 -347
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/popcount.c +865 -257
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/roaringish_ops.c +865 -257
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/search.c +970 -373
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/spans.c +865 -257
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/unique.c +947 -342
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.57 → searcharray-0.0.58}/setup.py +1 -1
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_tmdb.py +14 -2
- {searcharray-0.0.57 → searcharray-0.0.58}/README.md +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/pyproject.toml +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/__init__.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/phrase/bigram_freqs.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/phrase/spans.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/__init__.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/intersect.pyi +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/intersect.pyx +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/merge.pyi +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/merge.pyx +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/popcount.pyi +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/popcount.pyx +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/roaringish_ops.pyx +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/search.pyi +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/search.pyx +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/spans.pyx +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/unique.pyi +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/roaringish/unique.pyx +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/similarity.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/solr.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray/utils/sort.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/setup.cfg +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_extension_array.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_msmarco.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_search.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_similarity.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_slop_matches.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_snp_ops.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_solr.py +0 -0
- {searcharray-0.0.57 → searcharray-0.0.58}/test/test_utils.py +0 -0
|
@@ -121,7 +121,7 @@ def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate
|
|
|
121
121
|
|
|
122
122
|
# Encode posns to bit array
|
|
123
123
|
logger.info("Encoding positions to bit array")
|
|
124
|
-
posns = PosnBitArrayFromFlatBuilder(terms_w_posns)
|
|
124
|
+
posns = PosnBitArrayFromFlatBuilder(terms_w_posns, max_doc_id=(batch_beg + len(array) - 1))
|
|
125
125
|
bit_posns = posns.build()
|
|
126
126
|
|
|
127
127
|
if np.any(doc_lens > MAX_POSN):
|
|
@@ -158,8 +158,9 @@ class PosnBitArrayFromFlatBuilder:
|
|
|
158
158
|
|
|
159
159
|
"""
|
|
160
160
|
|
|
161
|
-
def __init__(self, flat_array: np.ndarray):
|
|
161
|
+
def __init__(self, flat_array: np.ndarray, max_doc_id: int) -> None:
|
|
162
162
|
self.flat_array = flat_array
|
|
163
|
+
self.max_doc_id = max_doc_id
|
|
163
164
|
|
|
164
165
|
def build(self):
|
|
165
166
|
"""Slice the flat array into a 2d array of doc ids and posns."""
|
|
@@ -179,7 +180,9 @@ class PosnBitArrayFromFlatBuilder:
|
|
|
179
180
|
sliced = encoded[beg_idx:end_idx]
|
|
180
181
|
encoded_term_posns[term_ids[into_terms]] = sliced
|
|
181
182
|
|
|
182
|
-
|
|
183
|
+
largest_doc_id_with_term = np.max(self.flat_array[1])
|
|
184
|
+
assert self.max_doc_id >= largest_doc_id_with_term
|
|
185
|
+
return PosnBitArray(encoded_term_posns, self.max_doc_id)
|
|
183
186
|
|
|
184
187
|
|
|
185
188
|
class PosnBitArrayBuilder:
|
|
@@ -390,6 +393,7 @@ class PosnBitArray:
|
|
|
390
393
|
min_posn: Optional[int] = None,
|
|
391
394
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
392
395
|
phrase_freqs = self.empty_buffer()
|
|
396
|
+
|
|
393
397
|
if len(term_ids) < 2:
|
|
394
398
|
raise ValueError("Must have at least two terms")
|
|
395
399
|
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
|
|
@@ -469,6 +469,15 @@ class SearchArray(ExtensionArray):
|
|
|
469
469
|
logger.warning("Unique called on SearchArray. This is not supported.")
|
|
470
470
|
return self[:]
|
|
471
471
|
|
|
472
|
+
def __iter__(self):
|
|
473
|
+
if len(self) > 10000:
|
|
474
|
+
warning_text = """Iterating over SearchArray is very slow and not reccomended.
|
|
475
|
+
|
|
476
|
+
If you're looping a dataframe, slice out the non-SearchArray columns first."""
|
|
477
|
+
logger.warning(warning_text)
|
|
478
|
+
warnings.warn(warning_text)
|
|
479
|
+
return super().__iter__()
|
|
480
|
+
|
|
472
481
|
def take(self, indices, allow_fill=False, fill_value=None):
|
|
473
482
|
# Want to take rows of term freqs
|
|
474
483
|
row_indices = np.arange(len(self.term_mat.rows))
|