searcharray 0.0.65__tar.gz → 0.0.67__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. {searcharray-0.0.65 → searcharray-0.0.67}/PKG-INFO +1 -1
  2. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/indexing.py +45 -4
  3. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/phrase/middle_out.py +4 -2
  4. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/postings.py +2 -0
  5. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/intersect.c +149 -149
  6. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/merge.c +149 -149
  7. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/popcount.c +464 -340
  8. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/popcount.pyx +12 -6
  9. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/roaringish_ops.c +149 -149
  10. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/search.c +149 -149
  11. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/spans.c +243 -241
  12. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/unique.c +149 -149
  13. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray.egg-info/PKG-INFO +1 -1
  14. {searcharray-0.0.65 → searcharray-0.0.67}/setup.py +1 -1
  15. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_search.py +21 -0
  16. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_similarity.py +16 -0
  17. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_tmdb.py +7 -4
  18. {searcharray-0.0.65 → searcharray-0.0.67}/README.md +0 -0
  19. {searcharray-0.0.65 → searcharray-0.0.67}/pyproject.toml +0 -0
  20. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/__init__.py +0 -0
  21. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/phrase/__init__.py +0 -0
  22. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/phrase/bigram_freqs.py +0 -0
  23. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/phrase/memmap_arrays.py +0 -0
  24. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/phrase/spans.py +0 -0
  25. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/__init__.py +0 -0
  26. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/intersect.pyi +0 -0
  27. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/intersect.pyx +0 -0
  28. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/merge.pyi +0 -0
  29. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/merge.pyx +0 -0
  30. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/popcount.pyi +0 -0
  31. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/roaringish.py +0 -0
  32. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/roaringish_ops.pyx +0 -0
  33. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/search.pyi +0 -0
  34. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/search.pyx +0 -0
  35. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/snp_ops.pxd +0 -0
  36. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/spans.pyx +0 -0
  37. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/unique.pyi +0 -0
  38. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/roaringish/unique.pyx +0 -0
  39. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/similarity.py +0 -0
  40. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/solr.py +0 -0
  41. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/term_dict.py +0 -0
  42. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/utils/__init__.py +0 -0
  43. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/utils/bitcount.py +0 -0
  44. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/utils/mat_set.py +0 -0
  45. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/utils/row_viewable_matrix.py +0 -0
  46. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray/utils/sort.py +0 -0
  47. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray.egg-info/SOURCES.txt +0 -0
  48. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray.egg-info/dependency_links.txt +0 -0
  49. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray.egg-info/entry_points.txt +0 -0
  50. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray.egg-info/requires.txt +0 -0
  51. {searcharray-0.0.65 → searcharray-0.0.67}/searcharray.egg-info/top_level.txt +0 -0
  52. {searcharray-0.0.65 → searcharray-0.0.67}/setup.cfg +0 -0
  53. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_bitcount64.py +0 -0
  54. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_extension_array.py +0 -0
  55. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_minmax_posns.py +0 -0
  56. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_msmarco.py +0 -0
  57. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_phrase_matches.py +0 -0
  58. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_slop_matches.py +0 -0
  59. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_snp_ops.py +0 -0
  60. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_solr.py +0 -0
  61. {searcharray-0.0.65 → searcharray-0.0.67}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.65
3
+ Version: 0.0.67
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -1,6 +1,5 @@
1
1
  import numpy as np
2
2
  import math
3
- import gc
4
3
  import os
5
4
  import sys
6
5
  from typing import Iterable, List, Optional
@@ -72,6 +71,7 @@ def _gather_tokens(array, tokenizer,
72
71
  for idx, doc in enumerate(array):
73
72
  doc_id = start_doc_id + idx
74
73
 
74
+ # Speedup - use predefined vocabulary
75
75
  terms = np.asarray([term_dict.add_term(token)
76
76
  for token in tokenizer(doc)], dtype=np.uint32)[:trunc_posn]
77
77
  doc_ids = np.full(len(terms), doc_id, dtype=np.uint32)[:trunc_posn]
@@ -92,8 +92,8 @@ def _gather_tokens(array, tokenizer,
92
92
 
93
93
  logger.info("Tokenization -- vstacking")
94
94
  terms_w_posns = np.vstack([all_terms, all_docs, all_posns])
95
- del all_terms, all_docs, all_posns
96
- gc.collect()
95
+ # del all_terms, all_docs, all_posns
96
+ # gc.collect()
97
97
  logger.info("Tokenization -- DONE")
98
98
  return terms_w_posns, term_doc
99
99
 
@@ -196,10 +196,50 @@ def _process_batches(term_doc, batch_size,
196
196
  return bit_posns
197
197
 
198
198
 
199
+ def build_index_no_workers(array: Iterable, tokenizer, batch_size=10000,
200
+ data_dir: Optional[str] = None,
201
+ cache_gt_than=25,
202
+ truncate=False):
203
+ term_dict = TermDict()
204
+ term_doc = SparseMatSetBuilder()
205
+ doc_lens: List[np.ndarray] = []
206
+ bit_posns = None
207
+
208
+ logger.info("Indexing begins w/ NO workers")
209
+ for batch_beg, batch in batch_iterator(array, batch_size):
210
+ batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = _tokenize_batch(batch, tokenizer, term_dict, batch_size, batch_beg, truncate=truncate)
211
+ term_doc.concat(batch_term_doc)
212
+ if bit_posns is None:
213
+ bit_posns = batch_bit_posns
214
+ else:
215
+ bit_posns.concat(batch_bit_posns)
216
+ doc_lens.append(batch_doc_lens)
217
+
218
+ doc_lens = np.concatenate(doc_lens)
219
+
220
+ avg_doc_length = np.mean(doc_lens)
221
+
222
+ term_doc_built = RowViewableMatrix(term_doc.build())
223
+ logger.info("Indexing from tokenization complete")
224
+ assert bit_posns is not None
225
+ # if data_dir is None:
226
+ # data_dir = searcharray_home()
227
+ if data_dir is not None:
228
+ logger.info(f"Memmapping bit positions to {data_dir}")
229
+ bit_posns.memmap(data_dir)
230
+ bit_posns.cache_gt_than = cache_gt_than
231
+ return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
232
+
233
+
199
234
  def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
200
235
  data_dir: Optional[str] = None,
201
- truncate=False, workers=4):
236
+ truncate=False, workers=4,
237
+ cache_gt_than=25):
202
238
  """Build index directly from tokenizing docs (array of string)."""
239
+ if workers == 1:
240
+ return build_index_no_workers(array, tokenizer, batch_size=batch_size,
241
+ cache_gt_than=cache_gt_than,
242
+ data_dir=data_dir, truncate=truncate)
203
243
  term_dict = TermDict()
204
244
  term_doc = SparseMatSetBuilder()
205
245
  doc_lens: List[np.ndarray] = []
@@ -250,6 +290,7 @@ def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
250
290
  if data_dir is not None:
251
291
  logger.info(f"Memmapping bit positions to {data_dir}")
252
292
  bit_posns.memmap(data_dir)
293
+ bit_posns.cache_gt_than = cache_gt_than
253
294
  return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
254
295
 
255
296
 
@@ -295,11 +295,13 @@ class FilteredPosns(abc.Mapping):
295
295
 
296
296
  class PosnBitArray:
297
297
 
298
- def __init__(self, encoded_term_posns: Union[ArrayDict, FilteredPosns], max_doc_id: int):
298
+ def __init__(self, encoded_term_posns: Union[ArrayDict, FilteredPosns], max_doc_id: int,
299
+ cache_gt_than=25):
299
300
  self.encoded_term_posns = encoded_term_posns
300
301
  self.max_doc_id = max_doc_id
301
302
  self.docfreq_cache : Dict[int, np.uint64] = {}
302
303
  self.termfreq_cache : Dict[int, Tuple[np.ndarray, np.ndarray]] = {}
304
+ self.cache_gt_than = cache_gt_than
303
305
 
304
306
  def memmap(self, data_dir):
305
307
  if self.encoded_term_posns:
@@ -480,7 +482,7 @@ class PosnBitArray:
480
482
  return self.docfreq_cache[term_id]
481
483
 
482
484
  def _maybe_cache_docfreq(self, term_id: int, docfreq: np.uint64):
483
- if self.max_doc_id >= 99999 and docfreq > (self.max_doc_id // 100):
485
+ if len(self.encoded_term_posns[term_id]) > self.cache_gt_than:
484
486
  self.docfreq_cache[term_id] = docfreq
485
487
 
486
488
  def docfreq(self, term_id: int) -> np.uint64:
@@ -252,6 +252,7 @@ class SearchArray(ExtensionArray):
252
252
  batch_size=100000,
253
253
  avoid_copies=True,
254
254
  workers=4,
255
+ cache_gt_than=25,
255
256
  data_dir: Optional[str] = None,
256
257
  autowarm=True) -> 'SearchArray':
257
258
  """Index an array of strings into SearchArray using tokenizer.
@@ -282,6 +283,7 @@ class SearchArray(ExtensionArray):
282
283
  build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
283
284
  truncate=truncate,
284
285
  data_dir=data_dir,
286
+ cache_gt_than=cache_gt_than,
285
287
  workers=workers)
286
288
 
287
289
  if autowarm: