searcharray 0.0.64__tar.gz → 0.0.66__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.64 → searcharray-0.0.66}/PKG-INFO +1 -1
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/indexing.py +45 -4
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/phrase/middle_out.py +6 -3
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/postings.py +2 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/intersect.c +149 -149
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/merge.c +149 -149
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/popcount.c +149 -149
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/roaringish_ops.c +149 -149
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/search.c +149 -149
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/spans.c +149 -149
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/unique.c +149 -149
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.64 → searcharray-0.0.66}/setup.py +1 -1
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_search.py +19 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_similarity.py +16 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_tmdb.py +7 -4
- {searcharray-0.0.64 → searcharray-0.0.66}/README.md +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/pyproject.toml +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/__init__.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/phrase/bigram_freqs.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/phrase/memmap_arrays.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/phrase/spans.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/__init__.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/intersect.pyi +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/intersect.pyx +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/merge.pyi +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/merge.pyx +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/popcount.pyi +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/popcount.pyx +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/roaringish.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/roaringish_ops.pyx +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/search.pyi +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/search.pyx +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/snp_ops.pxd +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/spans.pyx +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/unique.pyi +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/roaringish/unique.pyx +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/similarity.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/solr.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray/utils/sort.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/setup.cfg +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_extension_array.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_minmax_posns.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_msmarco.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_slop_matches.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_snp_ops.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_solr.py +0 -0
- {searcharray-0.0.64 → searcharray-0.0.66}/test/test_utils.py +0 -0
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import math
|
|
3
|
-
import gc
|
|
4
3
|
import os
|
|
5
4
|
import sys
|
|
6
5
|
from typing import Iterable, List, Optional
|
|
@@ -72,6 +71,7 @@ def _gather_tokens(array, tokenizer,
|
|
|
72
71
|
for idx, doc in enumerate(array):
|
|
73
72
|
doc_id = start_doc_id + idx
|
|
74
73
|
|
|
74
|
+
# Speedup - use predefined vocabulary
|
|
75
75
|
terms = np.asarray([term_dict.add_term(token)
|
|
76
76
|
for token in tokenizer(doc)], dtype=np.uint32)[:trunc_posn]
|
|
77
77
|
doc_ids = np.full(len(terms), doc_id, dtype=np.uint32)[:trunc_posn]
|
|
@@ -92,8 +92,8 @@ def _gather_tokens(array, tokenizer,
|
|
|
92
92
|
|
|
93
93
|
logger.info("Tokenization -- vstacking")
|
|
94
94
|
terms_w_posns = np.vstack([all_terms, all_docs, all_posns])
|
|
95
|
-
del all_terms, all_docs, all_posns
|
|
96
|
-
gc.collect()
|
|
95
|
+
# del all_terms, all_docs, all_posns
|
|
96
|
+
# gc.collect()
|
|
97
97
|
logger.info("Tokenization -- DONE")
|
|
98
98
|
return terms_w_posns, term_doc
|
|
99
99
|
|
|
@@ -196,10 +196,50 @@ def _process_batches(term_doc, batch_size,
|
|
|
196
196
|
return bit_posns
|
|
197
197
|
|
|
198
198
|
|
|
199
|
+
def build_index_no_workers(array: Iterable, tokenizer, batch_size=10000,
|
|
200
|
+
data_dir: Optional[str] = None,
|
|
201
|
+
cache_gt_than=25,
|
|
202
|
+
truncate=False):
|
|
203
|
+
term_dict = TermDict()
|
|
204
|
+
term_doc = SparseMatSetBuilder()
|
|
205
|
+
doc_lens: List[np.ndarray] = []
|
|
206
|
+
bit_posns = None
|
|
207
|
+
|
|
208
|
+
logger.info("Indexing begins w/ NO workers")
|
|
209
|
+
for batch_beg, batch in batch_iterator(array, batch_size):
|
|
210
|
+
batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = _tokenize_batch(batch, tokenizer, term_dict, batch_size, batch_beg, truncate=truncate)
|
|
211
|
+
term_doc.concat(batch_term_doc)
|
|
212
|
+
if bit_posns is None:
|
|
213
|
+
bit_posns = batch_bit_posns
|
|
214
|
+
else:
|
|
215
|
+
bit_posns.concat(batch_bit_posns)
|
|
216
|
+
doc_lens.append(batch_doc_lens)
|
|
217
|
+
|
|
218
|
+
doc_lens = np.concatenate(doc_lens)
|
|
219
|
+
|
|
220
|
+
avg_doc_length = np.mean(doc_lens)
|
|
221
|
+
|
|
222
|
+
term_doc_built = RowViewableMatrix(term_doc.build())
|
|
223
|
+
logger.info("Indexing from tokenization complete")
|
|
224
|
+
assert bit_posns is not None
|
|
225
|
+
# if data_dir is None:
|
|
226
|
+
# data_dir = searcharray_home()
|
|
227
|
+
if data_dir is not None:
|
|
228
|
+
logger.info(f"Memmapping bit positions to {data_dir}")
|
|
229
|
+
bit_posns.memmap(data_dir)
|
|
230
|
+
bit_posns.cache_gt_than = cache_gt_than
|
|
231
|
+
return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
|
|
232
|
+
|
|
233
|
+
|
|
199
234
|
def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
|
|
200
235
|
data_dir: Optional[str] = None,
|
|
201
|
-
truncate=False, workers=4
|
|
236
|
+
truncate=False, workers=4,
|
|
237
|
+
cache_gt_than=25):
|
|
202
238
|
"""Build index directly from tokenizing docs (array of string)."""
|
|
239
|
+
if workers == 1:
|
|
240
|
+
return build_index_no_workers(array, tokenizer, batch_size=batch_size,
|
|
241
|
+
cache_gt_than=cache_gt_than,
|
|
242
|
+
data_dir=data_dir, truncate=truncate)
|
|
203
243
|
term_dict = TermDict()
|
|
204
244
|
term_doc = SparseMatSetBuilder()
|
|
205
245
|
doc_lens: List[np.ndarray] = []
|
|
@@ -250,6 +290,7 @@ def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
|
|
|
250
290
|
if data_dir is not None:
|
|
251
291
|
logger.info(f"Memmapping bit positions to {data_dir}")
|
|
252
292
|
bit_posns.memmap(data_dir)
|
|
293
|
+
bit_posns.cache_gt_than = cache_gt_than
|
|
253
294
|
return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
|
|
254
295
|
|
|
255
296
|
|
|
@@ -295,14 +295,17 @@ class FilteredPosns(abc.Mapping):
|
|
|
295
295
|
|
|
296
296
|
class PosnBitArray:
|
|
297
297
|
|
|
298
|
-
def __init__(self, encoded_term_posns: Union[ArrayDict, FilteredPosns], max_doc_id: int
|
|
298
|
+
def __init__(self, encoded_term_posns: Union[ArrayDict, FilteredPosns], max_doc_id: int,
|
|
299
|
+
cache_gt_than=25):
|
|
299
300
|
self.encoded_term_posns = encoded_term_posns
|
|
300
301
|
self.max_doc_id = max_doc_id
|
|
301
302
|
self.docfreq_cache : Dict[int, np.uint64] = {}
|
|
302
303
|
self.termfreq_cache : Dict[int, Tuple[np.ndarray, np.ndarray]] = {}
|
|
304
|
+
self.cache_gt_than = cache_gt_than
|
|
303
305
|
|
|
304
306
|
def memmap(self, data_dir):
|
|
305
|
-
|
|
307
|
+
if self.encoded_term_posns:
|
|
308
|
+
self.encoded_term_posns = MemoryMappedArrays(data_dir, self.encoded_term_posns)
|
|
306
309
|
|
|
307
310
|
def warm(self):
|
|
308
311
|
"""Warm tf / df cache of most common terms."""
|
|
@@ -479,7 +482,7 @@ class PosnBitArray:
|
|
|
479
482
|
return self.docfreq_cache[term_id]
|
|
480
483
|
|
|
481
484
|
def _maybe_cache_docfreq(self, term_id: int, docfreq: np.uint64):
|
|
482
|
-
if self.
|
|
485
|
+
if len(self.encoded_term_posns[term_id]) > self.cache_gt_than:
|
|
483
486
|
self.docfreq_cache[term_id] = docfreq
|
|
484
487
|
|
|
485
488
|
def docfreq(self, term_id: int) -> np.uint64:
|
|
@@ -252,6 +252,7 @@ class SearchArray(ExtensionArray):
|
|
|
252
252
|
batch_size=100000,
|
|
253
253
|
avoid_copies=True,
|
|
254
254
|
workers=4,
|
|
255
|
+
cache_gt_than=25,
|
|
255
256
|
data_dir: Optional[str] = None,
|
|
256
257
|
autowarm=True) -> 'SearchArray':
|
|
257
258
|
"""Index an array of strings into SearchArray using tokenizer.
|
|
@@ -282,6 +283,7 @@ class SearchArray(ExtensionArray):
|
|
|
282
283
|
build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
|
|
283
284
|
truncate=truncate,
|
|
284
285
|
data_dir=data_dir,
|
|
286
|
+
cache_gt_than=cache_gt_than,
|
|
285
287
|
workers=workers)
|
|
286
288
|
|
|
287
289
|
if autowarm:
|