searcharray 0.0.55__tar.gz → 0.0.57__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {searcharray-0.0.55 → searcharray-0.0.57}/PKG-INFO +1 -1
  2. {searcharray-0.0.55 → searcharray-0.0.57}/pyproject.toml +3 -0
  3. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/indexing.py +83 -28
  4. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/phrase/bigram_freqs.py +1 -1
  5. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/postings.py +8 -1
  6. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/intersect.c +519 -539
  7. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/intersect.pyx +17 -22
  8. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/merge.c +285 -285
  9. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/merge.pyx +0 -2
  10. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/popcount.c +253 -263
  11. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/popcount.pyx +1 -2
  12. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/roaringish_ops.c +249 -282
  13. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/roaringish_ops.pyx +0 -1
  14. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/search.c +328 -334
  15. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/search.pyx +5 -6
  16. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/spans.c +202 -235
  17. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/spans.pyx +4 -11
  18. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/unique.c +236 -236
  19. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/unique.pyx +1 -3
  20. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/utils/mat_set.py +11 -0
  21. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/PKG-INFO +1 -1
  22. {searcharray-0.0.55 → searcharray-0.0.57}/setup.py +1 -1
  23. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_extension_array.py +8 -1
  24. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_msmarco.py +18 -11
  25. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_tmdb.py +89 -3
  26. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_utils.py +7 -0
  27. {searcharray-0.0.55 → searcharray-0.0.57}/README.md +0 -0
  28. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/__init__.py +0 -0
  29. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/phrase/__init__.py +0 -0
  30. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/phrase/middle_out.py +0 -0
  31. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/phrase/spans.py +0 -0
  32. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/__init__.py +0 -0
  33. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/intersect.pyi +0 -0
  34. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/merge.pyi +0 -0
  35. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/popcount.pyi +0 -0
  36. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/roaringish.py +0 -0
  37. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/search.pyi +0 -0
  38. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/snp_ops.pxd +0 -0
  39. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/roaringish/unique.pyi +0 -0
  40. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/similarity.py +0 -0
  41. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/solr.py +0 -0
  42. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/term_dict.py +0 -0
  43. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/utils/__init__.py +0 -0
  44. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/utils/bitcount.py +0 -0
  45. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/utils/row_viewable_matrix.py +0 -0
  46. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray/utils/sort.py +0 -0
  47. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/SOURCES.txt +0 -0
  48. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/dependency_links.txt +0 -0
  49. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/entry_points.txt +0 -0
  50. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/requires.txt +0 -0
  51. {searcharray-0.0.55 → searcharray-0.0.57}/searcharray.egg-info/top_level.txt +0 -0
  52. {searcharray-0.0.55 → searcharray-0.0.57}/setup.cfg +0 -0
  53. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_bitcount64.py +0 -0
  54. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_minmax_posns.py +0 -0
  55. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_phrase_matches.py +0 -0
  56. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_search.py +0 -0
  57. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_similarity.py +0 -0
  58. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_slop_matches.py +0 -0
  59. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_snp_ops.py +0 -0
  60. {searcharray-0.0.55 → searcharray-0.0.57}/test/test_solr.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.55
3
+ Version: 0.0.57
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -3,3 +3,6 @@
3
3
  # https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support
4
4
  requires = ["setuptools>=43.0.0", "wheel", "numpy", "cython"]
5
5
  build-backend = "setuptools.build_meta"
6
+
7
+ [tool.cython-lint]
8
+ max-line-length = 120
@@ -1,7 +1,9 @@
1
1
  import numpy as np
2
2
  import math
3
3
  import gc
4
- from typing import Iterable
4
+ import sys
5
+ from typing import Iterable, List
6
+ from concurrent.futures import ThreadPoolExecutor, as_completed
5
7
  from itertools import islice
6
8
  from searcharray.phrase.middle_out import MAX_POSN, PosnBitArrayFromFlatBuilder, PosnBitArrayBuilder, PosnBitArrayAlreadyEncBuilder
7
9
  from searcharray.term_dict import TermDict
@@ -10,6 +12,15 @@ from searcharray.utils.row_viewable_matrix import RowViewableMatrix
10
12
 
11
13
  import logging
12
14
  logger = logging.getLogger(__name__)
15
+ handler = logging.StreamHandler(sys.stdout)
16
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
17
+ logger.setLevel(logging.INFO)
18
+ handler.setFormatter(formatter)
19
+ logger.addHandler(handler)
20
+
21
+
22
+ # Set to stdout for debugging
23
+ # logging.basicConfig(level=logging.DEBUG)
13
24
 
14
25
 
15
26
  def _compute_doc_lens(posns: np.ndarray, doc_ids: np.ndarray, num_docs: int) -> np.ndarray:
@@ -71,7 +82,7 @@ def _gather_tokens(array, tokenizer, term_dict, term_doc, start_doc_id=0, trunc_
71
82
  del all_terms, all_docs, all_posns
72
83
  gc.collect()
73
84
  logger.info("Tokenization -- DONE")
74
- return terms_w_posns, term_dict, term_doc
85
+ return terms_w_posns, term_doc
75
86
 
76
87
 
77
88
  def _lex_sort(terms_w_posns):
@@ -90,14 +101,16 @@ def _invert_docs_terms(terms_w_posns):
90
101
  return terms_w_posns[:, lexsort]
91
102
 
92
103
 
93
- def _tokenize_batch(array, tokenizer, term_dict, term_doc, batch_size, batch_beg, truncate=False):
104
+ def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate=False):
105
+ logger.info(f"{batch_beg} Batch Start tokenization")
94
106
  trunc_posn = None
95
107
  if truncate:
96
108
  trunc_posn = MAX_POSN
97
109
 
98
- terms_w_posns, term_dict, term_doc = _gather_tokens(array, tokenizer,
99
- term_dict, term_doc, start_doc_id=batch_beg,
100
- trunc_posn=trunc_posn)
110
+ term_doc = SparseMatSetBuilder()
111
+ terms_w_posns, term_doc = _gather_tokens(array, tokenizer,
112
+ term_dict, term_doc, start_doc_id=batch_beg,
113
+ trunc_posn=trunc_posn)
101
114
 
102
115
  # Use posns to compute doc lens
103
116
  doc_lens = _compute_doc_lens(posns=terms_w_posns[2, :],
@@ -114,12 +127,11 @@ def _tokenize_batch(array, tokenizer, term_dict, term_doc, batch_size, batch_beg
114
127
  if np.any(doc_lens > MAX_POSN):
115
128
  raise ValueError(f"Document length exceeds maximum of {MAX_POSN}")
116
129
 
117
- return term_doc, bit_posns, term_dict, doc_lens
130
+ return batch_beg, term_doc, bit_posns, doc_lens
118
131
 
119
132
 
120
133
  def batch_iterator(iterator, batch_size):
121
134
  """
122
- Slices an iterator into batches of specified size and yields each batch until the iterator is exhausted.
123
135
 
124
136
  :param iterator: The iterator to slice into batches.
125
137
  :param batch_size: The size of each batch.
@@ -134,44 +146,87 @@ def batch_iterator(iterator, batch_size):
134
146
  batch_beg += batch_size
135
147
 
136
148
 
137
- def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000, truncate=False):
138
- """Build index directly from tokenizing docs (array of string)."""
139
- term_dict = TermDict()
140
- term_doc = SparseMatSetBuilder()
141
- doc_lens = []
142
- bit_posns = None
143
- batch_bit_posns = None
144
-
145
- logger.info("Indexing begins")
146
-
147
- for batch_beg, batch in batch_iterator(array, batch_size):
149
+ def _process_batches(term_doc, batch_size,
150
+ futures,
151
+ last_batch_beg_processed,
152
+ bit_posns=None,
153
+ doc_lens=None,
154
+ truncate=False):
155
+ batch_results = [None] * len(futures)
156
+ batch_beg = 0
157
+ for future in as_completed(futures):
148
158
  try:
149
- logger.info(f"{batch_beg} Batch Start")
150
- term_doc, batch_bit_posns, term_dict, batch_doc_lens = _tokenize_batch(batch, tokenizer,
151
- term_dict, term_doc,
152
- batch_size, batch_beg,
153
- truncate=truncate)
159
+ batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = future.result()
160
+ idx = (batch_beg - last_batch_beg_processed) // batch_size
161
+ assert batch_results[idx] is None
162
+ batch_results[idx] \
163
+ = (batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens)
154
164
  except ValueError as e:
155
165
  logger.error(e)
156
166
  logger.error(f"Batch {batch_beg} failed to tokenize")
157
167
  raise e
158
168
 
169
+ for result in batch_results:
170
+ assert result is not None
171
+ batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = result
172
+ term_doc.concat(batch_term_doc)
159
173
  if bit_posns is None:
160
174
  bit_posns = batch_bit_posns
161
175
  else:
162
176
  bit_posns.concat(batch_bit_posns)
163
177
 
164
178
  doc_lens.append(batch_doc_lens)
179
+ return bit_posns
180
+
181
+
182
+ def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
183
+ truncate=False, workers=4):
184
+ """Build index directly from tokenizing docs (array of string)."""
185
+ term_dict = TermDict()
186
+ term_doc = SparseMatSetBuilder()
187
+ doc_lens: List[np.ndarray] = []
188
+ bit_posns = None
165
189
 
166
- logger.info(f"{batch_beg} Batch Complete")
167
- logger.info(f"Roaringish NBytes -- {convert_size(bit_posns.nbytes)}")
168
- logger.info(f"Term Dict Size -- {len(term_dict)}")
190
+ logger.info("Indexing begins")
191
+ futures = []
192
+ last_batch_beg_processed = 0
193
+
194
+ with ThreadPoolExecutor(max_workers=workers) as executor:
195
+
196
+ for batch_beg, batch in batch_iterator(array, batch_size):
197
+ futures.append(executor.submit(_tokenize_batch,
198
+ batch,
199
+ tokenizer,
200
+ term_dict,
201
+ batch_size,
202
+ batch_beg,
203
+ truncate=truncate))
204
+
205
+ if len(futures) >= workers:
206
+ logger.info(f"Collecting {len(futures)} futures")
207
+ bit_posns = _process_batches(term_doc, batch_size,
208
+ futures,
209
+ last_batch_beg_processed,
210
+ bit_posns, doc_lens, truncate=truncate)
211
+ logger.info(f"{batch_beg} Batch Complete")
212
+ logger.info(f"Roaringish NBytes -- {convert_size(bit_posns.nbytes)}")
213
+ logger.info(f"Term Dict Size -- {len(term_dict)}")
214
+
215
+ last_batch_beg_processed += len(futures) * batch_size
216
+ futures = []
217
+ if len(futures) > 0:
218
+ bit_posns = _process_batches(term_doc, batch_size,
219
+ futures,
220
+ last_batch_beg_processed,
221
+ bit_posns, doc_lens, truncate=truncate)
169
222
 
170
223
  doc_lens = np.concatenate(doc_lens)
171
224
 
172
225
  avg_doc_length = np.mean(doc_lens)
173
226
 
174
- return RowViewableMatrix(term_doc.build()), bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
227
+ term_doc_built = RowViewableMatrix(term_doc.build())
228
+ logger.info("Indexing from tokenization complete")
229
+ return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
175
230
 
176
231
 
177
232
  def build_index_from_terms_list(postings, Terms):
@@ -206,7 +206,7 @@ def _set_adjbit_at_header(next_inner: np.ndarray, next_adj: np.ndarray,
206
206
  next_inner[same_header_inner] |= _1
207
207
  next_adj = next_adj[ignore_mask]
208
208
  if len(same_header_adj) > 0 and cont == Continuation.LHS:
209
- next_inner[same_header_adj] |= _upper_bit
209
+ next_inner[same_header_inner] |= _upper_bit
210
210
  next_adj = next_adj[ignore_mask]
211
211
  return merge(next_inner, next_adj)
212
212
 
@@ -228,6 +228,7 @@ class SearchArray(ExtensionArray):
228
228
  truncate=False,
229
229
  batch_size=100000,
230
230
  avoid_copies=True,
231
+ workers=4,
231
232
  autowarm=True) -> 'SearchArray':
232
233
  """Index an array of strings into SearchArray using tokenizer.
233
234
 
@@ -255,7 +256,8 @@ class SearchArray(ExtensionArray):
255
256
 
256
257
  term_mat, posns, term_dict, avg_doc_length, doc_lens =\
257
258
  build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
258
- truncate=truncate)
259
+ truncate=truncate,
260
+ workers=workers)
259
261
 
260
262
  if autowarm:
261
263
  posns.warm()
@@ -462,6 +464,11 @@ class SearchArray(ExtensionArray):
462
464
  empties = self.doc_lens == 0
463
465
  return empties
464
466
 
467
+ def unique(self):
468
+ """This is a hack for colab which wants to visit every element of the array."""
469
+ logger.warning("Unique called on SearchArray. This is not supported.")
470
+ return self[:]
471
+
465
472
  def take(self, indices, allow_fill=False, fill_value=None):
466
473
  # Want to take rows of term freqs
467
474
  row_indices = np.arange(len(self.term_mat.rows))