searcharray 0.0.38__tar.gz → 0.0.39__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.38 → searcharray-0.0.39}/PKG-INFO +1 -1
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/phrase/middle_out.py +17 -2
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/postings.py +40 -6
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/similarity.py +16 -2
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.38 → searcharray-0.0.39}/setup.py +1 -1
- {searcharray-0.0.38 → searcharray-0.0.39}/test/test_msmarco.py +41 -2
- {searcharray-0.0.38 → searcharray-0.0.39}/test/test_tmdb.py +9 -2
- {searcharray-0.0.38 → searcharray-0.0.39}/README.md +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/pyproject.toml +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/__init__.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/indexing.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/solr.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/utils/roaringish.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/setup.cfg +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/test/test_extension_array.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/test/test_search.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/test/test_solr.py +0 -0
- {searcharray-0.0.38 → searcharray-0.0.39}/test/test_utils.py +0 -0
|
@@ -296,6 +296,17 @@ class PosnBitArray:
|
|
|
296
296
|
self.docfreq_cache : Dict[int, np.uint64] = {}
|
|
297
297
|
self.termfreq_cache : Dict[int, Tuple[np.ndarray, np.ndarray]] = {}
|
|
298
298
|
|
|
299
|
+
def warm(self):
|
|
300
|
+
"""Warm tf / df cache of most common terms."""
|
|
301
|
+
for term_id, encoded in self.encoded_term_posns.items():
|
|
302
|
+
if len(encoded) > 255:
|
|
303
|
+
self.docfreq(term_id)
|
|
304
|
+
self.termfreqs(term_id)
|
|
305
|
+
|
|
306
|
+
def clear_cache(self):
|
|
307
|
+
self.docfreq_cache = {}
|
|
308
|
+
self.termfreq_cache = {}
|
|
309
|
+
|
|
299
310
|
def copy(self):
|
|
300
311
|
new = PosnBitArray(deepcopy(self.encoded_term_posns), self.max_doc_id)
|
|
301
312
|
return new
|
|
@@ -315,6 +326,8 @@ class PosnBitArray:
|
|
|
315
326
|
only_other_terms = set(other.encoded_term_posns.keys()).difference(set(self.encoded_term_posns.keys()))
|
|
316
327
|
for term_id in only_other_terms:
|
|
317
328
|
self.encoded_term_posns[term_id] = other.encoded_term_posns[term_id]
|
|
329
|
+
self.max_doc_id = max(self.max_doc_id, other.max_doc_id)
|
|
330
|
+
self.clear_cache()
|
|
318
331
|
|
|
319
332
|
def slice(self, key):
|
|
320
333
|
sliced_term_posns = {}
|
|
@@ -343,6 +356,8 @@ class PosnBitArray:
|
|
|
343
356
|
posns_self = self.encoded_term_posns[term_id]
|
|
344
357
|
posns_other = other.encoded_term_posns[term_id]
|
|
345
358
|
self.encoded_term_posns[term_id] = snp.merge(posns_self, posns_other)
|
|
359
|
+
self.max_doc_id = self.max_doc_id + other.max_doc_id
|
|
360
|
+
self.clear_cache()
|
|
346
361
|
|
|
347
362
|
def doc_encoded_posns(self, term_id: int, doc_id: int) -> np.ndarray:
|
|
348
363
|
term_posns = encoder.slice(self.encoded_term_posns[term_id],
|
|
@@ -411,7 +426,7 @@ class PosnBitArray:
|
|
|
411
426
|
posns = term_posns & encoder.payload_lsb_mask
|
|
412
427
|
bit_counts = bit_count64(posns)
|
|
413
428
|
|
|
414
|
-
term_freqs = np.add.reduceat(bit_counts, change_indices)
|
|
429
|
+
term_freqs = np.add.reduceat(bit_counts, change_indices, dtype=np.float32)
|
|
415
430
|
return sorted_unique(doc_ids), term_freqs
|
|
416
431
|
|
|
417
432
|
def _termfreqs_with_cache(self, term_id: int) -> Tuple[np.ndarray, np.ndarray]:
|
|
@@ -431,7 +446,7 @@ class PosnBitArray:
|
|
|
431
446
|
return self.docfreq_cache[term_id]
|
|
432
447
|
|
|
433
448
|
def _maybe_cache_docfreq(self, term_id: int, docfreq: np.uint64):
|
|
434
|
-
if docfreq > self.max_doc_id //
|
|
449
|
+
if self.max_doc_id >= 100000 and docfreq > (self.max_doc_id // 100):
|
|
435
450
|
self.docfreq_cache[term_id] = docfreq
|
|
436
451
|
|
|
437
452
|
def docfreq(self, term_id: int) -> np.uint64:
|
|
@@ -203,7 +203,11 @@ def _row_to_postings_row(doc_id, row, doc_len, term_dict, posns: PosnBitArray):
|
|
|
203
203
|
|
|
204
204
|
|
|
205
205
|
class SearchArray(ExtensionArray):
|
|
206
|
-
"""An array of tokenized text (
|
|
206
|
+
"""An array of tokenized text (Terms).
|
|
207
|
+
|
|
208
|
+
Use the class method `index` to create a SearchArray from a list of strings and tokenizer
|
|
209
|
+
|
|
210
|
+
"""
|
|
207
211
|
|
|
208
212
|
dtype = TermsDtype()
|
|
209
213
|
|
|
@@ -219,9 +223,33 @@ class SearchArray(ExtensionArray):
|
|
|
219
223
|
self.doc_lens = build_index_from_terms_list(postings, Terms)
|
|
220
224
|
|
|
221
225
|
@classmethod
|
|
222
|
-
def index(cls, array: Iterable,
|
|
223
|
-
|
|
224
|
-
|
|
226
|
+
def index(cls, array: Iterable,
|
|
227
|
+
tokenizer=ws_tokenizer,
|
|
228
|
+
truncate=False,
|
|
229
|
+
batch_size=100000,
|
|
230
|
+
avoid_copies=True,
|
|
231
|
+
autowarm=True) -> 'SearchArray':
|
|
232
|
+
"""Index an array of strings into SearchArray using tokenizer.
|
|
233
|
+
|
|
234
|
+
Parameters
|
|
235
|
+
----------
|
|
236
|
+
array : Iterable
|
|
237
|
+
A list of strings to index
|
|
238
|
+
tokenizer : Callable
|
|
239
|
+
A function that takes a string and returns an iterable of tokens
|
|
240
|
+
truncate : bool
|
|
241
|
+
If true, truncate the tokenized strings to a maximum length (2^18-1)
|
|
242
|
+
batch_size : int
|
|
243
|
+
For tuning memory usage, the number of strings to encode at once.
|
|
244
|
+
(which will be merged into one index at the end). Higher the more memory
|
|
245
|
+
will be used.
|
|
246
|
+
avoid_copies : bool
|
|
247
|
+
If true, avoid copying the term dictionary and positions during normal
|
|
248
|
+
pandas copy operations
|
|
249
|
+
autowarm : bool
|
|
250
|
+
If true, precompute docfreq / term freqs for most common terms
|
|
251
|
+
after indexing
|
|
252
|
+
"""
|
|
225
253
|
if not is_list_like(array):
|
|
226
254
|
raise TypeError("Expected list-like object, got {}".format(type(array)))
|
|
227
255
|
|
|
@@ -229,6 +257,9 @@ class SearchArray(ExtensionArray):
|
|
|
229
257
|
build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
|
|
230
258
|
truncate=truncate)
|
|
231
259
|
|
|
260
|
+
if autowarm:
|
|
261
|
+
posns.warm()
|
|
262
|
+
|
|
232
263
|
postings = cls([], tokenizer=tokenizer, avoid_copies=avoid_copies)
|
|
233
264
|
postings.term_mat = term_mat
|
|
234
265
|
postings.posns = posns
|
|
@@ -237,6 +268,9 @@ class SearchArray(ExtensionArray):
|
|
|
237
268
|
postings.doc_lens = doc_lens
|
|
238
269
|
return postings
|
|
239
270
|
|
|
271
|
+
def warm(self):
|
|
272
|
+
self.posns.warm()
|
|
273
|
+
|
|
240
274
|
@classmethod
|
|
241
275
|
def _from_sequence(cls, scalars, dtype=None, copy=False):
|
|
242
276
|
"""Construct a new SearchArray from a sequence of scalars (PostingRow or convertible into)."""
|
|
@@ -492,7 +526,7 @@ class SearchArray(ExtensionArray):
|
|
|
492
526
|
|
|
493
527
|
try:
|
|
494
528
|
term_id = self.term_dict.get_term_id(token)
|
|
495
|
-
matches = np.zeros(len(self), dtype=
|
|
529
|
+
matches = np.zeros(len(self), dtype=np.float32)
|
|
496
530
|
slice_of_rows = None
|
|
497
531
|
if self.term_mat.subset:
|
|
498
532
|
slice_of_rows = self.term_mat.rows
|
|
@@ -507,7 +541,7 @@ class SearchArray(ExtensionArray):
|
|
|
507
541
|
matches[doc_ids] = termfreqs
|
|
508
542
|
return matches
|
|
509
543
|
except TermMissingError:
|
|
510
|
-
return np.zeros(len(self), dtype=
|
|
544
|
+
return np.zeros(len(self), dtype=np.float32)
|
|
511
545
|
|
|
512
546
|
def docfreq(self, token: str) -> int:
|
|
513
547
|
if not isinstance(token, str):
|
|
@@ -12,6 +12,19 @@ class Similarity(Protocol):
|
|
|
12
12
|
...
|
|
13
13
|
|
|
14
14
|
|
|
15
|
+
def compute_idf(num_docs, sum_dfs):
|
|
16
|
+
"""Calculate idf."""
|
|
17
|
+
return np.log(1 + (num_docs - sum_dfs + 0.5) / (sum_dfs + 0.5))
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def compute_tfs(term_freqs: np.ndarray, doc_lens, avg_doc_lens, k1, b):
|
|
21
|
+
adj_doc_lens = k1 * (1 - b + b * doc_lens / avg_doc_lens)
|
|
22
|
+
# Divide tf in place for perf, but this means
|
|
23
|
+
# we can't use the same term_freqs for different k1, b
|
|
24
|
+
term_freqs /= (term_freqs + adj_doc_lens)
|
|
25
|
+
return term_freqs
|
|
26
|
+
|
|
27
|
+
|
|
15
28
|
def bm25_similarity(k1: float = 1.2, b: float = 0.75) -> Similarity:
|
|
16
29
|
"""BM25 similarity function, as in Lucene 9."""
|
|
17
30
|
def bm25(term_freqs: np.ndarray, doc_freqs: np.ndarray,
|
|
@@ -21,9 +34,10 @@ def bm25_similarity(k1: float = 1.2, b: float = 0.75) -> Similarity:
|
|
|
21
34
|
# Sum doc freqs
|
|
22
35
|
sum_dfs = np.sum(doc_freqs, axis=0)
|
|
23
36
|
# Calculate idf
|
|
24
|
-
idf =
|
|
37
|
+
idf = compute_idf(num_docs, sum_dfs)
|
|
25
38
|
# Calculate tf
|
|
26
|
-
tf = term_freqs / (term_freqs + k1 * (1 - b + b * doc_lens / avg_doc_lens))
|
|
39
|
+
# tf = term_freqs / (term_freqs + k1 * (1 - b + b * doc_lens / avg_doc_lens))
|
|
40
|
+
tf = compute_tfs(term_freqs, doc_lens, avg_doc_lens, k1, b)
|
|
27
41
|
return idf * tf
|
|
28
42
|
return bm25
|
|
29
43
|
|
|
@@ -36,7 +36,7 @@ setup(
|
|
|
36
36
|
# For a discussion on single-sourcing the version across setup.py and the
|
|
37
37
|
# project code, see
|
|
38
38
|
# https://packaging.python.org/guides/single-sourcing-package-version/
|
|
39
|
-
version="0.0.
|
|
39
|
+
version="0.0.39", # Required
|
|
40
40
|
# This is a one-line description or tagline of what your project does. This
|
|
41
41
|
# corresponds to the "Summary" metadata field:
|
|
42
42
|
# https://packaging.python.org/specifications/core-metadata/#summary
|
|
@@ -271,7 +271,16 @@ def test_msmarco10k_indexing(msmarco100k_raw, benchmark):
|
|
|
271
271
|
profiler = Profiler(benchmark)
|
|
272
272
|
# Random 10k
|
|
273
273
|
tenk = msmarco100k_raw['body'].sample(10000)
|
|
274
|
-
results = profiler.run(SearchArray.index, tenk)
|
|
274
|
+
results = profiler.run(SearchArray.index, tenk, autowarm=False)
|
|
275
|
+
assert len(results) == 10000
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
279
|
+
def test_msmarco10k_indexing_warm(msmarco100k_raw, benchmark):
|
|
280
|
+
profiler = Profiler(benchmark)
|
|
281
|
+
# Random 10k
|
|
282
|
+
tenk = msmarco100k_raw['body'].sample(10000)
|
|
283
|
+
results = profiler.run(SearchArray.index, tenk, autowarm=True)
|
|
275
284
|
assert len(results) == 10000
|
|
276
285
|
|
|
277
286
|
|
|
@@ -306,7 +315,7 @@ def test_msmarco_indexall(msmarco_unzipped, benchmark, caplog):
|
|
|
306
315
|
|
|
307
316
|
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
308
317
|
@pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
|
|
309
|
-
def
|
|
318
|
+
def test_msmarco1m_or_search_unwarmed(query, msmarco1m, benchmark, caplog):
|
|
310
319
|
profiler = Profiler(benchmark)
|
|
311
320
|
|
|
312
321
|
caplog.set_level(logging.DEBUG)
|
|
@@ -318,6 +327,36 @@ def test_msmarco1m_or_search(query, msmarco1m, benchmark, caplog):
|
|
|
318
327
|
assert np.any(scores > 0)
|
|
319
328
|
|
|
320
329
|
|
|
330
|
+
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
331
|
+
@pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
|
|
332
|
+
def test_msmarco1m_or_search_warmed(query, msmarco1m, benchmark, caplog):
|
|
333
|
+
profiler = Profiler(benchmark)
|
|
334
|
+
|
|
335
|
+
caplog.set_level(logging.DEBUG)
|
|
336
|
+
|
|
337
|
+
def sum_scores(query):
|
|
338
|
+
return np.sum([msmarco1m['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
|
|
339
|
+
sum_scores(query) # Warmup
|
|
340
|
+
scores = profiler.run(sum_scores, query)
|
|
341
|
+
assert len(scores) == len(msmarco1m['body_ws'].array)
|
|
342
|
+
assert np.any(scores > 0)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
346
|
+
@pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
|
|
347
|
+
def test_msmarco1m_or_search_warming(query, msmarco1m, benchmark, caplog):
|
|
348
|
+
profiler = Profiler(benchmark)
|
|
349
|
+
|
|
350
|
+
caplog.set_level(logging.DEBUG)
|
|
351
|
+
|
|
352
|
+
def sum_scores(query):
|
|
353
|
+
return np.sum([msmarco1m['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
|
|
354
|
+
sum_scores(query) # Warmup
|
|
355
|
+
scores = profiler.run(sum_scores, query)
|
|
356
|
+
assert len(scores) == len(msmarco1m['body_ws'].array)
|
|
357
|
+
assert np.any(scores > 0)
|
|
358
|
+
|
|
359
|
+
|
|
321
360
|
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
322
361
|
@pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
|
|
323
362
|
def test_msmarco100k_or_search_unwarmed(query, msmarco100k, benchmark, caplog):
|
|
@@ -134,7 +134,14 @@ def test_phrase_match_tmdb(phrase, expected_matches, tmdb_data, benchmark):
|
|
|
134
134
|
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
135
135
|
def test_index_benchmark(benchmark, tmdb_pd_data):
|
|
136
136
|
prof = Profiler(benchmark)
|
|
137
|
-
results = prof.run(SearchArray.index, tmdb_pd_data['overview'])
|
|
137
|
+
results = prof.run(SearchArray.index, tmdb_pd_data['overview'], autowarm=False)
|
|
138
|
+
assert len(results) == len(tmdb_pd_data)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
|
|
142
|
+
def test_index_benchmark_warmed(benchmark, tmdb_pd_data):
|
|
143
|
+
prof = Profiler(benchmark)
|
|
144
|
+
results = prof.run(SearchArray.index, tmdb_pd_data['overview'], autowarm=True)
|
|
138
145
|
assert len(results) == len(tmdb_pd_data)
|
|
139
146
|
|
|
140
147
|
|
|
@@ -142,7 +149,7 @@ def test_index_benchmark(benchmark, tmdb_pd_data):
|
|
|
142
149
|
def test_index_benchmark_1k_random(benchmark, tmdb_pd_data):
|
|
143
150
|
prof = Profiler(benchmark)
|
|
144
151
|
thousand_random = np.random.choice(tmdb_pd_data['overview'], size=1000)
|
|
145
|
-
results = prof.run(SearchArray.index, thousand_random)
|
|
152
|
+
results = prof.run(SearchArray.index, thousand_random, autowarm=False)
|
|
146
153
|
assert len(results) == 1000
|
|
147
154
|
|
|
148
155
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|