searcharray 0.0.38__tar.gz → 0.0.39__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {searcharray-0.0.38 → searcharray-0.0.39}/PKG-INFO +1 -1
  2. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/phrase/middle_out.py +17 -2
  3. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/postings.py +40 -6
  4. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/similarity.py +16 -2
  5. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/PKG-INFO +1 -1
  6. {searcharray-0.0.38 → searcharray-0.0.39}/setup.py +1 -1
  7. {searcharray-0.0.38 → searcharray-0.0.39}/test/test_msmarco.py +41 -2
  8. {searcharray-0.0.38 → searcharray-0.0.39}/test/test_tmdb.py +9 -2
  9. {searcharray-0.0.38 → searcharray-0.0.39}/README.md +0 -0
  10. {searcharray-0.0.38 → searcharray-0.0.39}/pyproject.toml +0 -0
  11. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/__init__.py +0 -0
  12. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/indexing.py +0 -0
  13. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/phrase/__init__.py +0 -0
  14. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/phrase/posn_diffs.py +0 -0
  15. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/phrase/scan_merge.py +0 -0
  16. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/solr.py +0 -0
  17. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/term_dict.py +0 -0
  18. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/utils/__init__.py +0 -0
  19. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/utils/bitcount.py +0 -0
  20. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/utils/mat_set.py +0 -0
  21. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/utils/roaringish.py +0 -0
  22. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray/utils/row_viewable_matrix.py +0 -0
  23. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/SOURCES.txt +0 -0
  24. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/dependency_links.txt +0 -0
  25. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/entry_points.txt +0 -0
  26. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/requires.txt +0 -0
  27. {searcharray-0.0.38 → searcharray-0.0.39}/searcharray.egg-info/top_level.txt +0 -0
  28. {searcharray-0.0.38 → searcharray-0.0.39}/setup.cfg +0 -0
  29. {searcharray-0.0.38 → searcharray-0.0.39}/test/test_bitcount64.py +0 -0
  30. {searcharray-0.0.38 → searcharray-0.0.39}/test/test_extension_array.py +0 -0
  31. {searcharray-0.0.38 → searcharray-0.0.39}/test/test_phrase_matches.py +0 -0
  32. {searcharray-0.0.38 → searcharray-0.0.39}/test/test_search.py +0 -0
  33. {searcharray-0.0.38 → searcharray-0.0.39}/test/test_solr.py +0 -0
  34. {searcharray-0.0.38 → searcharray-0.0.39}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.38
3
+ Version: 0.0.39
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -296,6 +296,17 @@ class PosnBitArray:
296
296
  self.docfreq_cache : Dict[int, np.uint64] = {}
297
297
  self.termfreq_cache : Dict[int, Tuple[np.ndarray, np.ndarray]] = {}
298
298
 
299
+ def warm(self):
300
+ """Warm tf / df cache of most common terms."""
301
+ for term_id, encoded in self.encoded_term_posns.items():
302
+ if len(encoded) > 255:
303
+ self.docfreq(term_id)
304
+ self.termfreqs(term_id)
305
+
306
+ def clear_cache(self):
307
+ self.docfreq_cache = {}
308
+ self.termfreq_cache = {}
309
+
299
310
  def copy(self):
300
311
  new = PosnBitArray(deepcopy(self.encoded_term_posns), self.max_doc_id)
301
312
  return new
@@ -315,6 +326,8 @@ class PosnBitArray:
315
326
  only_other_terms = set(other.encoded_term_posns.keys()).difference(set(self.encoded_term_posns.keys()))
316
327
  for term_id in only_other_terms:
317
328
  self.encoded_term_posns[term_id] = other.encoded_term_posns[term_id]
329
+ self.max_doc_id = max(self.max_doc_id, other.max_doc_id)
330
+ self.clear_cache()
318
331
 
319
332
  def slice(self, key):
320
333
  sliced_term_posns = {}
@@ -343,6 +356,8 @@ class PosnBitArray:
343
356
  posns_self = self.encoded_term_posns[term_id]
344
357
  posns_other = other.encoded_term_posns[term_id]
345
358
  self.encoded_term_posns[term_id] = snp.merge(posns_self, posns_other)
359
+ self.max_doc_id = self.max_doc_id + other.max_doc_id
360
+ self.clear_cache()
346
361
 
347
362
  def doc_encoded_posns(self, term_id: int, doc_id: int) -> np.ndarray:
348
363
  term_posns = encoder.slice(self.encoded_term_posns[term_id],
@@ -411,7 +426,7 @@ class PosnBitArray:
411
426
  posns = term_posns & encoder.payload_lsb_mask
412
427
  bit_counts = bit_count64(posns)
413
428
 
414
- term_freqs = np.add.reduceat(bit_counts, change_indices)
429
+ term_freqs = np.add.reduceat(bit_counts, change_indices, dtype=np.float32)
415
430
  return sorted_unique(doc_ids), term_freqs
416
431
 
417
432
  def _termfreqs_with_cache(self, term_id: int) -> Tuple[np.ndarray, np.ndarray]:
@@ -431,7 +446,7 @@ class PosnBitArray:
431
446
  return self.docfreq_cache[term_id]
432
447
 
433
448
  def _maybe_cache_docfreq(self, term_id: int, docfreq: np.uint64):
434
- if docfreq > self.max_doc_id // 10:
449
+ if self.max_doc_id >= 100000 and docfreq > (self.max_doc_id // 100):
435
450
  self.docfreq_cache[term_id] = docfreq
436
451
 
437
452
  def docfreq(self, term_id: int) -> np.uint64:
@@ -203,7 +203,11 @@ def _row_to_postings_row(doc_id, row, doc_len, term_dict, posns: PosnBitArray):
203
203
 
204
204
 
205
205
  class SearchArray(ExtensionArray):
206
- """An array of tokenized text (Termss)."""
206
+ """An array of tokenized text (Terms).
207
+
208
+ Use the class method `index` to create a SearchArray from a list of strings and tokenizer
209
+
210
+ """
207
211
 
208
212
  dtype = TermsDtype()
209
213
 
@@ -219,9 +223,33 @@ class SearchArray(ExtensionArray):
219
223
  self.doc_lens = build_index_from_terms_list(postings, Terms)
220
224
 
221
225
  @classmethod
222
- def index(cls, array: Iterable, tokenizer=ws_tokenizer,
223
- truncate=False, batch_size=100000, avoid_copies=True) -> 'SearchArray':
224
- """Index an array of strings using tokenizer."""
226
+ def index(cls, array: Iterable,
227
+ tokenizer=ws_tokenizer,
228
+ truncate=False,
229
+ batch_size=100000,
230
+ avoid_copies=True,
231
+ autowarm=True) -> 'SearchArray':
232
+ """Index an array of strings into SearchArray using tokenizer.
233
+
234
+ Parameters
235
+ ----------
236
+ array : Iterable
237
+ A list of strings to index
238
+ tokenizer : Callable
239
+ A function that takes a string and returns an iterable of tokens
240
+ truncate : bool
241
+ If true, truncate the tokenized strings to a maximum length (2^18-1)
242
+ batch_size : int
243
+ For tuning memory usage, the number of strings to encode at once.
244
+ (which will be merged into one index at the end). Higher the more memory
245
+ will be used.
246
+ avoid_copies : bool
247
+ If true, avoid copying the term dictionary and positions during normal
248
+ pandas copy operations
249
+ autowarm : bool
250
+ If true, precompute docfreq / term freqs for most common terms
251
+ after indexing
252
+ """
225
253
  if not is_list_like(array):
226
254
  raise TypeError("Expected list-like object, got {}".format(type(array)))
227
255
 
@@ -229,6 +257,9 @@ class SearchArray(ExtensionArray):
229
257
  build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
230
258
  truncate=truncate)
231
259
 
260
+ if autowarm:
261
+ posns.warm()
262
+
232
263
  postings = cls([], tokenizer=tokenizer, avoid_copies=avoid_copies)
233
264
  postings.term_mat = term_mat
234
265
  postings.posns = posns
@@ -237,6 +268,9 @@ class SearchArray(ExtensionArray):
237
268
  postings.doc_lens = doc_lens
238
269
  return postings
239
270
 
271
+ def warm(self):
272
+ self.posns.warm()
273
+
240
274
  @classmethod
241
275
  def _from_sequence(cls, scalars, dtype=None, copy=False):
242
276
  """Construct a new SearchArray from a sequence of scalars (PostingRow or convertible into)."""
@@ -492,7 +526,7 @@ class SearchArray(ExtensionArray):
492
526
 
493
527
  try:
494
528
  term_id = self.term_dict.get_term_id(token)
495
- matches = np.zeros(len(self), dtype=int)
529
+ matches = np.zeros(len(self), dtype=np.float32)
496
530
  slice_of_rows = None
497
531
  if self.term_mat.subset:
498
532
  slice_of_rows = self.term_mat.rows
@@ -507,7 +541,7 @@ class SearchArray(ExtensionArray):
507
541
  matches[doc_ids] = termfreqs
508
542
  return matches
509
543
  except TermMissingError:
510
- return np.zeros(len(self), dtype=int)
544
+ return np.zeros(len(self), dtype=np.float32)
511
545
 
512
546
  def docfreq(self, token: str) -> int:
513
547
  if not isinstance(token, str):
@@ -12,6 +12,19 @@ class Similarity(Protocol):
12
12
  ...
13
13
 
14
14
 
15
+ def compute_idf(num_docs, sum_dfs):
16
+ """Calculate idf."""
17
+ return np.log(1 + (num_docs - sum_dfs + 0.5) / (sum_dfs + 0.5))
18
+
19
+
20
+ def compute_tfs(term_freqs: np.ndarray, doc_lens, avg_doc_lens, k1, b):
21
+ adj_doc_lens = k1 * (1 - b + b * doc_lens / avg_doc_lens)
22
+ # Divide tf in place for perf, but this means
23
+ # we can't use the same term_freqs for different k1, b
24
+ term_freqs /= (term_freqs + adj_doc_lens)
25
+ return term_freqs
26
+
27
+
15
28
  def bm25_similarity(k1: float = 1.2, b: float = 0.75) -> Similarity:
16
29
  """BM25 similarity function, as in Lucene 9."""
17
30
  def bm25(term_freqs: np.ndarray, doc_freqs: np.ndarray,
@@ -21,9 +34,10 @@ def bm25_similarity(k1: float = 1.2, b: float = 0.75) -> Similarity:
21
34
  # Sum doc freqs
22
35
  sum_dfs = np.sum(doc_freqs, axis=0)
23
36
  # Calculate idf
24
- idf = np.log(1 + (num_docs - sum_dfs + 0.5) / (sum_dfs + 0.5))
37
+ idf = compute_idf(num_docs, sum_dfs)
25
38
  # Calculate tf
26
- tf = term_freqs / (term_freqs + k1 * (1 - b + b * doc_lens / avg_doc_lens))
39
+ # tf = term_freqs / (term_freqs + k1 * (1 - b + b * doc_lens / avg_doc_lens))
40
+ tf = compute_tfs(term_freqs, doc_lens, avg_doc_lens, k1, b)
27
41
  return idf * tf
28
42
  return bm25
29
43
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.38
3
+ Version: 0.0.39
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -36,7 +36,7 @@ setup(
36
36
  # For a discussion on single-sourcing the version across setup.py and the
37
37
  # project code, see
38
38
  # https://packaging.python.org/guides/single-sourcing-package-version/
39
- version="0.0.38", # Required
39
+ version="0.0.39", # Required
40
40
  # This is a one-line description or tagline of what your project does. This
41
41
  # corresponds to the "Summary" metadata field:
42
42
  # https://packaging.python.org/specifications/core-metadata/#summary
@@ -271,7 +271,16 @@ def test_msmarco10k_indexing(msmarco100k_raw, benchmark):
271
271
  profiler = Profiler(benchmark)
272
272
  # Random 10k
273
273
  tenk = msmarco100k_raw['body'].sample(10000)
274
- results = profiler.run(SearchArray.index, tenk)
274
+ results = profiler.run(SearchArray.index, tenk, autowarm=False)
275
+ assert len(results) == 10000
276
+
277
+
278
+ @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
279
+ def test_msmarco10k_indexing_warm(msmarco100k_raw, benchmark):
280
+ profiler = Profiler(benchmark)
281
+ # Random 10k
282
+ tenk = msmarco100k_raw['body'].sample(10000)
283
+ results = profiler.run(SearchArray.index, tenk, autowarm=True)
275
284
  assert len(results) == 10000
276
285
 
277
286
 
@@ -306,7 +315,7 @@ def test_msmarco_indexall(msmarco_unzipped, benchmark, caplog):
306
315
 
307
316
  @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
308
317
  @pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
309
- def test_msmarco1m_or_search(query, msmarco1m, benchmark, caplog):
318
+ def test_msmarco1m_or_search_unwarmed(query, msmarco1m, benchmark, caplog):
310
319
  profiler = Profiler(benchmark)
311
320
 
312
321
  caplog.set_level(logging.DEBUG)
@@ -318,6 +327,36 @@ def test_msmarco1m_or_search(query, msmarco1m, benchmark, caplog):
318
327
  assert np.any(scores > 0)
319
328
 
320
329
 
330
+ @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
331
+ @pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
332
+ def test_msmarco1m_or_search_warmed(query, msmarco1m, benchmark, caplog):
333
+ profiler = Profiler(benchmark)
334
+
335
+ caplog.set_level(logging.DEBUG)
336
+
337
+ def sum_scores(query):
338
+ return np.sum([msmarco1m['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
339
+ sum_scores(query) # Warmup
340
+ scores = profiler.run(sum_scores, query)
341
+ assert len(scores) == len(msmarco1m['body_ws'].array)
342
+ assert np.any(scores > 0)
343
+
344
+
345
+ @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
346
+ @pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
347
+ def test_msmarco1m_or_search_warming(query, msmarco1m, benchmark, caplog):
348
+ profiler = Profiler(benchmark)
349
+
350
+ caplog.set_level(logging.DEBUG)
351
+
352
+ def sum_scores(query):
353
+ return np.sum([msmarco1m['body_ws'].array.score(query_term) for query_term in query.split()], axis=0)
354
+ sum_scores(query) # Warmup
355
+ scores = profiler.run(sum_scores, query)
356
+ assert len(scores) == len(msmarco1m['body_ws'].array)
357
+ assert np.any(scores > 0)
358
+
359
+
321
360
  @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
322
361
  @pytest.mark.parametrize("query", ["what is", "what is the", "what is the purpose", "what is the purpose of", "what is the purpose of cats", "star trek", "star trek the next generation", "what what what"])
323
362
  def test_msmarco100k_or_search_unwarmed(query, msmarco100k, benchmark, caplog):
@@ -134,7 +134,14 @@ def test_phrase_match_tmdb(phrase, expected_matches, tmdb_data, benchmark):
134
134
  @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
135
135
  def test_index_benchmark(benchmark, tmdb_pd_data):
136
136
  prof = Profiler(benchmark)
137
- results = prof.run(SearchArray.index, tmdb_pd_data['overview'])
137
+ results = prof.run(SearchArray.index, tmdb_pd_data['overview'], autowarm=False)
138
+ assert len(results) == len(tmdb_pd_data)
139
+
140
+
141
+ @pytest.mark.skipif(not profile_enabled, reason="Profiling disabled")
142
+ def test_index_benchmark_warmed(benchmark, tmdb_pd_data):
143
+ prof = Profiler(benchmark)
144
+ results = prof.run(SearchArray.index, tmdb_pd_data['overview'], autowarm=True)
138
145
  assert len(results) == len(tmdb_pd_data)
139
146
 
140
147
 
@@ -142,7 +149,7 @@ def test_index_benchmark(benchmark, tmdb_pd_data):
142
149
  def test_index_benchmark_1k_random(benchmark, tmdb_pd_data):
143
150
  prof = Profiler(benchmark)
144
151
  thousand_random = np.random.choice(tmdb_pd_data['overview'], size=1000)
145
- results = prof.run(SearchArray.index, thousand_random)
152
+ results = prof.run(SearchArray.index, thousand_random, autowarm=False)
146
153
  assert len(results) == 1000
147
154
 
148
155
 
File without changes
File without changes