searcharray 0.0.55__tar.gz → 0.0.57__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.55 → searcharray-0.0.57}/PKG-INFO +1 -1
- {searcharray-0.0.55 → searcharray-0.0.57}/pyproject.toml +3 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/indexing.py +83 -28
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/phrase/bigram_freqs.py +1 -1
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/postings.py +8 -1
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/intersect.c +519 -539
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/intersect.pyx +17 -22
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/merge.c +285 -285
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/merge.pyx +0 -2
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/popcount.c +253 -263
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/popcount.pyx +1 -2
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/roaringish_ops.c +249 -282
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/roaringish_ops.pyx +0 -1
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/search.c +328 -334
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/search.pyx +5 -6
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/spans.c +202 -235
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/spans.pyx +4 -11
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/unique.c +236 -236
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/unique.pyx +1 -3
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/utils/mat_set.py +11 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.55 → searcharray-0.0.57}/setup.py +1 -1
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_extension_array.py +8 -1
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_msmarco.py +18 -11
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_tmdb.py +89 -3
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_utils.py +7 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/README.md +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/__init__.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/phrase/middle_out.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/phrase/spans.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/__init__.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/intersect.pyi +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/merge.pyi +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/popcount.pyi +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/search.pyi +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/unique.pyi +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/similarity.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/solr.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/utils/sort.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/setup.cfg +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_search.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_similarity.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_slop_matches.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_snp_ops.py +0 -0
- {searcharray-0.0.55 → searcharray-0.0.57}/test/test_solr.py +0 -0
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import math
|
|
3
3
|
import gc
|
|
4
|
-
|
|
4
|
+
import sys
|
|
5
|
+
from typing import Iterable, List
|
|
6
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
5
7
|
from itertools import islice
|
|
6
8
|
from searcharray.phrase.middle_out import MAX_POSN, PosnBitArrayFromFlatBuilder, PosnBitArrayBuilder, PosnBitArrayAlreadyEncBuilder
|
|
7
9
|
from searcharray.term_dict import TermDict
|
|
@@ -10,6 +12,15 @@ from searcharray.utils.row_viewable_matrix import RowViewableMatrix
|
|
|
10
12
|
|
|
11
13
|
import logging
|
|
12
14
|
logger = logging.getLogger(__name__)
|
|
15
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
16
|
+
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
17
|
+
logger.setLevel(logging.INFO)
|
|
18
|
+
handler.setFormatter(formatter)
|
|
19
|
+
logger.addHandler(handler)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# Set to stdout for debugging
|
|
23
|
+
# logging.basicConfig(level=logging.DEBUG)
|
|
13
24
|
|
|
14
25
|
|
|
15
26
|
def _compute_doc_lens(posns: np.ndarray, doc_ids: np.ndarray, num_docs: int) -> np.ndarray:
|
|
@@ -71,7 +82,7 @@ def _gather_tokens(array, tokenizer, term_dict, term_doc, start_doc_id=0, trunc_
|
|
|
71
82
|
del all_terms, all_docs, all_posns
|
|
72
83
|
gc.collect()
|
|
73
84
|
logger.info("Tokenization -- DONE")
|
|
74
|
-
return terms_w_posns,
|
|
85
|
+
return terms_w_posns, term_doc
|
|
75
86
|
|
|
76
87
|
|
|
77
88
|
def _lex_sort(terms_w_posns):
|
|
@@ -90,14 +101,16 @@ def _invert_docs_terms(terms_w_posns):
|
|
|
90
101
|
return terms_w_posns[:, lexsort]
|
|
91
102
|
|
|
92
103
|
|
|
93
|
-
def _tokenize_batch(array, tokenizer, term_dict,
|
|
104
|
+
def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate=False):
|
|
105
|
+
logger.info(f"{batch_beg} Batch Start tokenization")
|
|
94
106
|
trunc_posn = None
|
|
95
107
|
if truncate:
|
|
96
108
|
trunc_posn = MAX_POSN
|
|
97
109
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
110
|
+
term_doc = SparseMatSetBuilder()
|
|
111
|
+
terms_w_posns, term_doc = _gather_tokens(array, tokenizer,
|
|
112
|
+
term_dict, term_doc, start_doc_id=batch_beg,
|
|
113
|
+
trunc_posn=trunc_posn)
|
|
101
114
|
|
|
102
115
|
# Use posns to compute doc lens
|
|
103
116
|
doc_lens = _compute_doc_lens(posns=terms_w_posns[2, :],
|
|
@@ -114,12 +127,11 @@ def _tokenize_batch(array, tokenizer, term_dict, term_doc, batch_size, batch_beg
|
|
|
114
127
|
if np.any(doc_lens > MAX_POSN):
|
|
115
128
|
raise ValueError(f"Document length exceeds maximum of {MAX_POSN}")
|
|
116
129
|
|
|
117
|
-
return term_doc, bit_posns,
|
|
130
|
+
return batch_beg, term_doc, bit_posns, doc_lens
|
|
118
131
|
|
|
119
132
|
|
|
120
133
|
def batch_iterator(iterator, batch_size):
|
|
121
134
|
"""
|
|
122
|
-
Slices an iterator into batches of specified size and yields each batch until the iterator is exhausted.
|
|
123
135
|
|
|
124
136
|
:param iterator: The iterator to slice into batches.
|
|
125
137
|
:param batch_size: The size of each batch.
|
|
@@ -134,44 +146,87 @@ def batch_iterator(iterator, batch_size):
|
|
|
134
146
|
batch_beg += batch_size
|
|
135
147
|
|
|
136
148
|
|
|
137
|
-
def
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
for batch_beg, batch in batch_iterator(array, batch_size):
|
|
149
|
+
def _process_batches(term_doc, batch_size,
|
|
150
|
+
futures,
|
|
151
|
+
last_batch_beg_processed,
|
|
152
|
+
bit_posns=None,
|
|
153
|
+
doc_lens=None,
|
|
154
|
+
truncate=False):
|
|
155
|
+
batch_results = [None] * len(futures)
|
|
156
|
+
batch_beg = 0
|
|
157
|
+
for future in as_completed(futures):
|
|
148
158
|
try:
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
159
|
+
batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = future.result()
|
|
160
|
+
idx = (batch_beg - last_batch_beg_processed) // batch_size
|
|
161
|
+
assert batch_results[idx] is None
|
|
162
|
+
batch_results[idx] \
|
|
163
|
+
= (batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens)
|
|
154
164
|
except ValueError as e:
|
|
155
165
|
logger.error(e)
|
|
156
166
|
logger.error(f"Batch {batch_beg} failed to tokenize")
|
|
157
167
|
raise e
|
|
158
168
|
|
|
169
|
+
for result in batch_results:
|
|
170
|
+
assert result is not None
|
|
171
|
+
batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = result
|
|
172
|
+
term_doc.concat(batch_term_doc)
|
|
159
173
|
if bit_posns is None:
|
|
160
174
|
bit_posns = batch_bit_posns
|
|
161
175
|
else:
|
|
162
176
|
bit_posns.concat(batch_bit_posns)
|
|
163
177
|
|
|
164
178
|
doc_lens.append(batch_doc_lens)
|
|
179
|
+
return bit_posns
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
|
|
183
|
+
truncate=False, workers=4):
|
|
184
|
+
"""Build index directly from tokenizing docs (array of string)."""
|
|
185
|
+
term_dict = TermDict()
|
|
186
|
+
term_doc = SparseMatSetBuilder()
|
|
187
|
+
doc_lens: List[np.ndarray] = []
|
|
188
|
+
bit_posns = None
|
|
165
189
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
190
|
+
logger.info("Indexing begins")
|
|
191
|
+
futures = []
|
|
192
|
+
last_batch_beg_processed = 0
|
|
193
|
+
|
|
194
|
+
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
195
|
+
|
|
196
|
+
for batch_beg, batch in batch_iterator(array, batch_size):
|
|
197
|
+
futures.append(executor.submit(_tokenize_batch,
|
|
198
|
+
batch,
|
|
199
|
+
tokenizer,
|
|
200
|
+
term_dict,
|
|
201
|
+
batch_size,
|
|
202
|
+
batch_beg,
|
|
203
|
+
truncate=truncate))
|
|
204
|
+
|
|
205
|
+
if len(futures) >= workers:
|
|
206
|
+
logger.info(f"Collecting {len(futures)} futures")
|
|
207
|
+
bit_posns = _process_batches(term_doc, batch_size,
|
|
208
|
+
futures,
|
|
209
|
+
last_batch_beg_processed,
|
|
210
|
+
bit_posns, doc_lens, truncate=truncate)
|
|
211
|
+
logger.info(f"{batch_beg} Batch Complete")
|
|
212
|
+
logger.info(f"Roaringish NBytes -- {convert_size(bit_posns.nbytes)}")
|
|
213
|
+
logger.info(f"Term Dict Size -- {len(term_dict)}")
|
|
214
|
+
|
|
215
|
+
last_batch_beg_processed += len(futures) * batch_size
|
|
216
|
+
futures = []
|
|
217
|
+
if len(futures) > 0:
|
|
218
|
+
bit_posns = _process_batches(term_doc, batch_size,
|
|
219
|
+
futures,
|
|
220
|
+
last_batch_beg_processed,
|
|
221
|
+
bit_posns, doc_lens, truncate=truncate)
|
|
169
222
|
|
|
170
223
|
doc_lens = np.concatenate(doc_lens)
|
|
171
224
|
|
|
172
225
|
avg_doc_length = np.mean(doc_lens)
|
|
173
226
|
|
|
174
|
-
|
|
227
|
+
term_doc_built = RowViewableMatrix(term_doc.build())
|
|
228
|
+
logger.info("Indexing from tokenization complete")
|
|
229
|
+
return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
|
|
175
230
|
|
|
176
231
|
|
|
177
232
|
def build_index_from_terms_list(postings, Terms):
|
|
@@ -206,7 +206,7 @@ def _set_adjbit_at_header(next_inner: np.ndarray, next_adj: np.ndarray,
|
|
|
206
206
|
next_inner[same_header_inner] |= _1
|
|
207
207
|
next_adj = next_adj[ignore_mask]
|
|
208
208
|
if len(same_header_adj) > 0 and cont == Continuation.LHS:
|
|
209
|
-
next_inner[
|
|
209
|
+
next_inner[same_header_inner] |= _upper_bit
|
|
210
210
|
next_adj = next_adj[ignore_mask]
|
|
211
211
|
return merge(next_inner, next_adj)
|
|
212
212
|
|
|
@@ -228,6 +228,7 @@ class SearchArray(ExtensionArray):
|
|
|
228
228
|
truncate=False,
|
|
229
229
|
batch_size=100000,
|
|
230
230
|
avoid_copies=True,
|
|
231
|
+
workers=4,
|
|
231
232
|
autowarm=True) -> 'SearchArray':
|
|
232
233
|
"""Index an array of strings into SearchArray using tokenizer.
|
|
233
234
|
|
|
@@ -255,7 +256,8 @@ class SearchArray(ExtensionArray):
|
|
|
255
256
|
|
|
256
257
|
term_mat, posns, term_dict, avg_doc_length, doc_lens =\
|
|
257
258
|
build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
|
|
258
|
-
truncate=truncate
|
|
259
|
+
truncate=truncate,
|
|
260
|
+
workers=workers)
|
|
259
261
|
|
|
260
262
|
if autowarm:
|
|
261
263
|
posns.warm()
|
|
@@ -462,6 +464,11 @@ class SearchArray(ExtensionArray):
|
|
|
462
464
|
empties = self.doc_lens == 0
|
|
463
465
|
return empties
|
|
464
466
|
|
|
467
|
+
def unique(self):
|
|
468
|
+
"""This is a hack for colab which wants to visit every element of the array."""
|
|
469
|
+
logger.warning("Unique called on SearchArray. This is not supported.")
|
|
470
|
+
return self[:]
|
|
471
|
+
|
|
465
472
|
def take(self, indices, allow_fill=False, fill_value=None):
|
|
466
473
|
# Want to take rows of term freqs
|
|
467
474
|
row_indices = np.arange(len(self.term_mat.rows))
|