searcharray 0.0.54__tar.gz → 0.0.56__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. {searcharray-0.0.54 → searcharray-0.0.56}/PKG-INFO +1 -1
  2. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/indexing.py +83 -28
  3. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/phrase/bigram_freqs.py +41 -30
  4. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/phrase/middle_out.py +62 -12
  5. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/postings.py +32 -82
  6. searcharray-0.0.56/searcharray/roaringish/__init__.py +7 -0
  7. searcharray-0.0.54/searcharray/roaringish/snp_ops.c → searcharray-0.0.56/searcharray/roaringish/intersect.c +4203 -10360
  8. searcharray-0.0.56/searcharray/roaringish/intersect.pyi +25 -0
  9. searcharray-0.0.56/searcharray/roaringish/intersect.pyx +395 -0
  10. searcharray-0.0.56/searcharray/roaringish/merge.c +31524 -0
  11. searcharray-0.0.56/searcharray/roaringish/merge.pyi +8 -0
  12. searcharray-0.0.56/searcharray/roaringish/merge.pyx +160 -0
  13. searcharray-0.0.56/searcharray/roaringish/popcount.c +31835 -0
  14. searcharray-0.0.56/searcharray/roaringish/popcount.pyi +20 -0
  15. searcharray-0.0.56/searcharray/roaringish/popcount.pyx +113 -0
  16. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/roaringish.py +34 -3
  17. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/roaringish_ops.c +300 -1076
  18. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/roaringish_ops.pyx +1 -33
  19. searcharray-0.0.56/searcharray/roaringish/search.c +31761 -0
  20. searcharray-0.0.56/searcharray/roaringish/search.pyi +25 -0
  21. searcharray-0.0.56/searcharray/roaringish/search.pyx +141 -0
  22. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/snp_ops.pxd +0 -10
  23. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/spans.c +242 -370
  24. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/spans.pyx +7 -1
  25. searcharray-0.0.56/searcharray/roaringish/unique.c +31772 -0
  26. searcharray-0.0.56/searcharray/roaringish/unique.pyi +6 -0
  27. searcharray-0.0.56/searcharray/roaringish/unique.pyx +147 -0
  28. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/similarity.py +15 -6
  29. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/solr.py +86 -42
  30. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/utils/mat_set.py +11 -0
  31. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/PKG-INFO +1 -1
  32. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/SOURCES.txt +15 -4
  33. {searcharray-0.0.54 → searcharray-0.0.56}/setup.py +3 -3
  34. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_minmax_posns.py +1 -5
  35. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_msmarco.py +105 -25
  36. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_phrase_matches.py +76 -19
  37. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_search.py +9 -6
  38. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_snp_ops.py +65 -3
  39. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_tmdb.py +77 -4
  40. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_utils.py +12 -2
  41. searcharray-0.0.54/searcharray/phrase/posn_diffs.py +0 -147
  42. searcharray-0.0.54/searcharray/phrase/scan_merge.py +0 -84
  43. searcharray-0.0.54/searcharray/roaringish/__init__.py +0 -4
  44. searcharray-0.0.54/searcharray/roaringish/snp_ops.pyx +0 -664
  45. {searcharray-0.0.54 → searcharray-0.0.56}/README.md +0 -0
  46. {searcharray-0.0.54 → searcharray-0.0.56}/pyproject.toml +0 -0
  47. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/__init__.py +0 -0
  48. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/phrase/__init__.py +0 -0
  49. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/phrase/spans.py +0 -0
  50. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/term_dict.py +0 -0
  51. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/utils/__init__.py +0 -0
  52. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/utils/bitcount.py +0 -0
  53. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/utils/row_viewable_matrix.py +0 -0
  54. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/utils/sort.py +0 -0
  55. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/dependency_links.txt +0 -0
  56. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/entry_points.txt +0 -0
  57. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/requires.txt +0 -0
  58. {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/top_level.txt +0 -0
  59. {searcharray-0.0.54 → searcharray-0.0.56}/setup.cfg +0 -0
  60. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_bitcount64.py +0 -0
  61. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_extension_array.py +0 -0
  62. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_similarity.py +0 -0
  63. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_slop_matches.py +0 -0
  64. {searcharray-0.0.54 → searcharray-0.0.56}/test/test_solr.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.54
3
+ Version: 0.0.56
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -1,7 +1,9 @@
1
1
  import numpy as np
2
2
  import math
3
3
  import gc
4
- from typing import Iterable
4
+ import sys
5
+ from typing import Iterable, List
6
+ from concurrent.futures import ThreadPoolExecutor, as_completed
5
7
  from itertools import islice
6
8
  from searcharray.phrase.middle_out import MAX_POSN, PosnBitArrayFromFlatBuilder, PosnBitArrayBuilder, PosnBitArrayAlreadyEncBuilder
7
9
  from searcharray.term_dict import TermDict
@@ -10,6 +12,15 @@ from searcharray.utils.row_viewable_matrix import RowViewableMatrix
10
12
 
11
13
  import logging
12
14
  logger = logging.getLogger(__name__)
15
+ handler = logging.StreamHandler(sys.stdout)
16
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
17
+ logger.setLevel(logging.INFO)
18
+ handler.setFormatter(formatter)
19
+ logger.addHandler(handler)
20
+
21
+
22
+ # Set to stdout for debugging
23
+ # logging.basicConfig(level=logging.DEBUG)
13
24
 
14
25
 
15
26
  def _compute_doc_lens(posns: np.ndarray, doc_ids: np.ndarray, num_docs: int) -> np.ndarray:
@@ -71,7 +82,7 @@ def _gather_tokens(array, tokenizer, term_dict, term_doc, start_doc_id=0, trunc_
71
82
  del all_terms, all_docs, all_posns
72
83
  gc.collect()
73
84
  logger.info("Tokenization -- DONE")
74
- return terms_w_posns, term_dict, term_doc
85
+ return terms_w_posns, term_doc
75
86
 
76
87
 
77
88
  def _lex_sort(terms_w_posns):
@@ -90,14 +101,16 @@ def _invert_docs_terms(terms_w_posns):
90
101
  return terms_w_posns[:, lexsort]
91
102
 
92
103
 
93
- def _tokenize_batch(array, tokenizer, term_dict, term_doc, batch_size, batch_beg, truncate=False):
104
+ def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate=False):
105
+ logger.info(f"{batch_beg} Batch Start tokenization")
94
106
  trunc_posn = None
95
107
  if truncate:
96
108
  trunc_posn = MAX_POSN
97
109
 
98
- terms_w_posns, term_dict, term_doc = _gather_tokens(array, tokenizer,
99
- term_dict, term_doc, start_doc_id=batch_beg,
100
- trunc_posn=trunc_posn)
110
+ term_doc = SparseMatSetBuilder()
111
+ terms_w_posns, term_doc = _gather_tokens(array, tokenizer,
112
+ term_dict, term_doc, start_doc_id=batch_beg,
113
+ trunc_posn=trunc_posn)
101
114
 
102
115
  # Use posns to compute doc lens
103
116
  doc_lens = _compute_doc_lens(posns=terms_w_posns[2, :],
@@ -114,12 +127,11 @@ def _tokenize_batch(array, tokenizer, term_dict, term_doc, batch_size, batch_beg
114
127
  if np.any(doc_lens > MAX_POSN):
115
128
  raise ValueError(f"Document length exceeds maximum of {MAX_POSN}")
116
129
 
117
- return term_doc, bit_posns, term_dict, doc_lens
130
+ return batch_beg, term_doc, bit_posns, doc_lens
118
131
 
119
132
 
120
133
  def batch_iterator(iterator, batch_size):
121
134
  """
122
- Slices an iterator into batches of specified size and yields each batch until the iterator is exhausted.
123
135
 
124
136
  :param iterator: The iterator to slice into batches.
125
137
  :param batch_size: The size of each batch.
@@ -134,44 +146,87 @@ def batch_iterator(iterator, batch_size):
134
146
  batch_beg += batch_size
135
147
 
136
148
 
137
- def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000, truncate=False):
138
- """Build index directly from tokenizing docs (array of string)."""
139
- term_dict = TermDict()
140
- term_doc = SparseMatSetBuilder()
141
- doc_lens = []
142
- bit_posns = None
143
- batch_bit_posns = None
144
-
145
- logger.info("Indexing begins")
146
-
147
- for batch_beg, batch in batch_iterator(array, batch_size):
149
+ def _process_batches(term_doc, batch_size,
150
+ futures,
151
+ last_batch_beg_processed,
152
+ bit_posns=None,
153
+ doc_lens=None,
154
+ truncate=False):
155
+ batch_results = [None] * len(futures)
156
+ batch_beg = 0
157
+ for future in as_completed(futures):
148
158
  try:
149
- logger.info(f"{batch_beg} Batch Start")
150
- term_doc, batch_bit_posns, term_dict, batch_doc_lens = _tokenize_batch(batch, tokenizer,
151
- term_dict, term_doc,
152
- batch_size, batch_beg,
153
- truncate=truncate)
159
+ batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = future.result()
160
+ idx = (batch_beg - last_batch_beg_processed) // batch_size
161
+ assert batch_results[idx] is None
162
+ batch_results[idx] \
163
+ = (batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens)
154
164
  except ValueError as e:
155
165
  logger.error(e)
156
166
  logger.error(f"Batch {batch_beg} failed to tokenize")
157
167
  raise e
158
168
 
169
+ for result in batch_results:
170
+ assert result is not None
171
+ batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = result
172
+ term_doc.concat(batch_term_doc)
159
173
  if bit_posns is None:
160
174
  bit_posns = batch_bit_posns
161
175
  else:
162
176
  bit_posns.concat(batch_bit_posns)
163
177
 
164
178
  doc_lens.append(batch_doc_lens)
179
+ return bit_posns
180
+
181
+
182
+ def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
183
+ truncate=False, workers=4):
184
+ """Build index directly from tokenizing docs (array of string)."""
185
+ term_dict = TermDict()
186
+ term_doc = SparseMatSetBuilder()
187
+ doc_lens: List[np.ndarray] = []
188
+ bit_posns = None
165
189
 
166
- logger.info(f"{batch_beg} Batch Complete")
167
- logger.info(f"Roaringish NBytes -- {convert_size(bit_posns.nbytes)}")
168
- logger.info(f"Term Dict Size -- {len(term_dict)}")
190
+ logger.info("Indexing begins")
191
+ futures = []
192
+ last_batch_beg_processed = 0
193
+
194
+ with ThreadPoolExecutor(max_workers=workers) as executor:
195
+
196
+ for batch_beg, batch in batch_iterator(array, batch_size):
197
+ futures.append(executor.submit(_tokenize_batch,
198
+ batch,
199
+ tokenizer,
200
+ term_dict,
201
+ batch_size,
202
+ batch_beg,
203
+ truncate=truncate))
204
+
205
+ if len(futures) >= workers:
206
+ logger.info(f"Collecting {len(futures)} futures")
207
+ bit_posns = _process_batches(term_doc, batch_size,
208
+ futures,
209
+ last_batch_beg_processed,
210
+ bit_posns, doc_lens, truncate=truncate)
211
+ logger.info(f"{batch_beg} Batch Complete")
212
+ logger.info(f"Roaringish NBytes -- {convert_size(bit_posns.nbytes)}")
213
+ logger.info(f"Term Dict Size -- {len(term_dict)}")
214
+
215
+ last_batch_beg_processed += len(futures) * batch_size
216
+ futures = []
217
+ if len(futures) > 0:
218
+ bit_posns = _process_batches(term_doc, batch_size,
219
+ futures,
220
+ last_batch_beg_processed,
221
+ bit_posns, doc_lens, truncate=truncate)
169
222
 
170
223
  doc_lens = np.concatenate(doc_lens)
171
224
 
172
225
  avg_doc_length = np.mean(doc_lens)
173
226
 
174
- return RowViewableMatrix(term_doc.build()), bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
227
+ term_doc_built = RowViewableMatrix(term_doc.build())
228
+ logger.info("Indexing from tokenization complete")
229
+ return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
175
230
 
176
231
 
177
232
  def build_index_from_terms_list(postings, Terms):
@@ -99,7 +99,10 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
99
99
  return phrase_freqs, (lhs_cont, rhs_cont)
100
100
 
101
101
 
102
- def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
102
+ def _inner_bigram_freqs(lhs: np.ndarray,
103
+ rhs: np.ndarray,
104
+ lhs_int: np.ndarray,
105
+ rhs_int: np.ndarray,
103
106
  phrase_freqs: np.ndarray,
104
107
  cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
105
108
  """Count bigram matches between two encoded arrays, within a 64 bit word with same MSBs.
@@ -108,6 +111,8 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
108
111
  -----------
109
112
  lhs: roaringish encoded posn array of left term
110
113
  rhs: roaringish encoded posn array of right term
114
+ lhs_int: np.ndarray, intersected lhs
115
+ rhs_int: np.ndarray, intersected rhs
111
116
  phrase_freqs: np.ndarray, preallocated phrase freqs for output
112
117
  cont_rhs: bool, whether to continue matching on the rhs or lhs
113
118
 
@@ -117,7 +122,7 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
117
122
  cont_next: the next (lhs or rhs) array to continue matching
118
123
 
119
124
  """
120
- lhs_int, rhs_int = encoder.intersect(lhs, rhs)
125
+ # lhs_int, rhs_int = encoder.intersect(lhs, rhs)
121
126
  lhs_doc_ids = encoder.keys(lhs_int)
122
127
  if len(lhs_int) != len(rhs_int):
123
128
  raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
@@ -149,6 +154,7 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
149
154
 
150
155
 
151
156
  def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
157
+ lhs_adj: np.ndarray, rhs_adj: np.ndarray,
152
158
  phrase_freqs: np.ndarray,
153
159
  cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
154
160
  """Count bigram matches between two encoded arrays where they occur in adjacent 64 bit words.
@@ -159,10 +165,9 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
159
165
  rhs_next: the next rhs array to continue matching
160
166
 
161
167
  """
162
- lhs_int, rhs_int = encoder.intersect_rshift(lhs, rhs, rshift=_neg1)
163
- lhs_doc_ids = encoder.keys(lhs_int)
168
+ lhs_doc_ids = encoder.keys(lhs_adj)
164
169
  # lhs lsb set and rhs lsb's most significant bit set
165
- matches = ((lhs_int & _upper_bit) != 0) & ((rhs_int & _1) != 0)
170
+ matches = ((lhs_adj & _upper_bit) != 0) & ((rhs_adj & _1) != 0)
166
171
  unique, counts = np.unique(lhs_doc_ids[matches], return_counts=True)
167
172
  phrase_freqs[unique] += counts
168
173
  # Set lsb to 0 where no match, lsb to 1 where match
@@ -170,40 +175,40 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
170
175
  lhs_next = None if cont == Continuation.RHS else np.asarray([], dtype=np.uint64)
171
176
  if np.any(matches):
172
177
  if cont in [Continuation.RHS, Continuation.BOTH]:
173
- rhs_next = rhs_int[matches]
178
+ rhs_next = rhs_adj[matches]
174
179
  assert rhs_next is not None
175
- rhs_next = encoder.header(rhs_int) | _1
180
+ rhs_next = encoder.header(rhs_adj) | _1
176
181
  elif cont in [Continuation.LHS, Continuation.BOTH]:
177
- rhs_next = rhs_int[matches]
182
+ rhs_next = rhs_adj[matches]
178
183
  assert rhs_next is not None
179
184
  rhs_next |= _1
180
- rhs_next = encoder.header(rhs_int) | _1
181
- lhs_next = lhs_int[matches]
182
- lhs_next = encoder.header(lhs_int) | _upper_bit
185
+ rhs_next = encoder.header(rhs_adj) | _1
186
+ lhs_next = lhs_adj[matches]
187
+ lhs_next = encoder.header(lhs_adj) | _upper_bit
183
188
  rhs_next = None
184
189
  return phrase_freqs, (lhs_next, rhs_next)
185
190
 
186
191
 
187
- def _set_adjbit_at_header(enc1: np.ndarray, just_lsb_enc: np.ndarray,
192
+ def _set_adjbit_at_header(next_inner: np.ndarray, next_adj: np.ndarray,
188
193
  cont: Continuation = Continuation.RHS) -> np.ndarray:
189
194
  """Merge two encoded arrays on their headers."""
190
- if len(enc1) == 0:
191
- return just_lsb_enc
192
- if len(just_lsb_enc) == 0:
193
- return enc1
195
+ if len(next_inner) == 0:
196
+ return next_adj
197
+ if len(next_adj) == 0:
198
+ return next_inner
194
199
 
195
- same_header_enc1, same_header_just_lsb = intersect(enc1, just_lsb_enc,
196
- mask=encoder.header_mask)
200
+ same_header_inner, same_header_adj = intersect(next_inner, next_adj,
201
+ mask=encoder.header_mask)
197
202
  # Set _1 on intersection
198
- ignore_mask = np.ones(len(just_lsb_enc), dtype=bool)
199
- ignore_mask[same_header_just_lsb] = False
200
- if len(same_header_enc1) > 0 and cont == Continuation.RHS:
201
- enc1[same_header_enc1] |= _1
202
- just_lsb_enc = just_lsb_enc[ignore_mask]
203
- if len(same_header_just_lsb) > 0 and cont == Continuation.LHS:
204
- enc1[same_header_enc1] |= _upper_bit
205
- just_lsb_enc = just_lsb_enc[ignore_mask]
206
- return merge(enc1, just_lsb_enc)
203
+ ignore_mask = np.ones(len(next_adj), dtype=bool)
204
+ ignore_mask[same_header_adj] = False
205
+ if len(same_header_inner) > 0 and cont == Continuation.RHS:
206
+ next_inner[same_header_inner] |= _1
207
+ next_adj = next_adj[ignore_mask]
208
+ if len(same_header_adj) > 0 and cont == Continuation.LHS:
209
+ next_inner[same_header_inner] |= _upper_bit
210
+ next_adj = next_adj[ignore_mask]
211
+ return merge(next_inner, next_adj)
207
212
 
208
213
 
209
214
  def bigram_freqs(lhs: np.ndarray,
@@ -242,9 +247,13 @@ def bigram_freqs(lhs: np.ndarray,
242
247
  # rhs_term: 1234 101 0101000100
243
248
  # ^^ ^^ <- inner matches, terms next to each other
244
249
  #
245
- phrase_freqs, (lhs_next_inner, rhs_next_inner)\
246
- = _inner_bigram_freqs(lhs, rhs, phrase_freqs, cont)
250
+ lhs_int, rhs_int, lhs_adj, rhs_adj = encoder.intersect_candidates(lhs, rhs)
247
251
 
252
+ phrase_freqs, (lhs_next_inner, rhs_next_inner)\
253
+ = _inner_bigram_freqs(lhs, rhs,
254
+ lhs_int, rhs_int,
255
+ phrase_freqs,
256
+ cont)
248
257
  # ***************************************************************************
249
258
  # "Adjacent" matches:
250
259
  # Of course we also need to check for matches that span two 64 bit words,
@@ -259,7 +268,9 @@ def bigram_freqs(lhs: np.ndarray,
259
268
  # 1 less than the posn_msb of rhs
260
269
  # so we detect these differently
261
270
  phrase_freqs, (lhs_next_adj, rhs_next_adj)\
262
- = _adjacent_bigram_freqs(lhs, rhs, phrase_freqs, cont)
271
+ = _adjacent_bigram_freqs(lhs, rhs,
272
+ lhs_adj, rhs_adj,
273
+ phrase_freqs, cont)
263
274
 
264
275
  # ***************************************************************************
265
276
  # rhs_next is where the bigram ends on the rhs side, we can use this
@@ -13,7 +13,7 @@ from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
13
13
  from searcharray.phrase.spans import span_search
14
14
  import numbers
15
15
  import logging
16
- from collections import defaultdict
16
+ from collections import defaultdict, abc
17
17
 
18
18
 
19
19
  logger = logging.getLogger(__name__)
@@ -72,6 +72,7 @@ def trim_phrase_search(encoded_posns: List[np.ndarray],
72
72
 
73
73
  def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
74
74
  phrase_freqs: np.ndarray,
75
+ max_doc_id: np.uint64 = _0,
75
76
  trim: bool = True) -> np.ndarray:
76
77
  """Compute phrase freqs from a set of encoded positions."""
77
78
  if len(encoded_posns) < 2:
@@ -86,7 +87,8 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
86
87
  for rhs in encoded_posns[1:]:
87
88
  # Only count the count of the last bigram (ignoring the ones where priors did not match)
88
89
  phrase_freqs[mask] = 0
89
- phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs, cont=Continuation.RHS)
90
+ phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
91
+ cont=Continuation.RHS)
90
92
  assert conts[1] is not None
91
93
  lhs = conts[1]
92
94
  mask &= (phrase_freqs > 0)
@@ -99,6 +101,7 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
99
101
 
100
102
  def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
101
103
  phrase_freqs: np.ndarray,
104
+ max_doc_id: np.uint64 = _0,
102
105
  trim: bool = True) -> np.ndarray:
103
106
  """Compute phrase freqs from a set of encoded positions."""
104
107
  if len(encoded_posns) < 2:
@@ -113,7 +116,8 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
113
116
  for lhs in encoded_posns[-2::-1]:
114
117
  # Only count the count of the last bigram (ignoring the ones where priors did not match)
115
118
  phrase_freqs[mask] = 0
116
- phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs, cont=Continuation.LHS)
119
+ phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
120
+ cont=Continuation.LHS)
117
121
  assert conts[0] is not None
118
122
  rhs = conts[0]
119
123
  mask &= (phrase_freqs > 0)
@@ -126,18 +130,19 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
126
130
 
127
131
  def compute_phrase_freqs(encoded_posns: List[np.ndarray],
128
132
  phrase_freqs: np.ndarray,
133
+ max_doc_id: np.uint64 = _0,
129
134
  trim: bool = False) -> np.ndarray:
130
135
  """Compute phrase freqs from a set of encoded positions."""
131
136
  shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
132
137
  if shortest_len_index <= 1:
133
- return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim)
138
+ return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim, max_doc_id=max_doc_id)
134
139
  elif shortest_len_index >= len(encoded_posns) - 2:
135
- return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim)
140
+ return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim, max_doc_id=max_doc_id)
136
141
  else:
137
142
  # We optimize this case by going middle-out
138
143
  # We can take the min of both directions phrase freqs
139
- lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim)
140
- rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim)
144
+ lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim, max_doc_id=max_doc_id)
145
+ rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim, max_doc_id=max_doc_id)
141
146
  phrase_freqs = np.minimum(lhs, rhs)
142
147
  return phrase_freqs
143
148
 
@@ -259,6 +264,35 @@ def index_range(rng, key):
259
264
  return np.asarray(list(rng))[key]
260
265
 
261
266
 
267
+ class FilteredPosns(abc.Mapping):
268
+ """When someone slices, this lets us avoid repeating constantly
269
+ slicing the keys out of the encoded positions."""
270
+
271
+ def __init__(self, base, doc_ids):
272
+ self.base = base
273
+ self.sliced = {}
274
+ self.doc_ids = doc_ids
275
+
276
+ def __getitem__(self, key):
277
+ """Slice with encoder, cache, and return."""
278
+ if key in self.sliced:
279
+ return self.sliced[key]
280
+
281
+ sliced = encoder.slice(self.base[key],
282
+ keys=self.doc_ids)
283
+ self.sliced[key] = sliced
284
+ return sliced
285
+
286
+ def __setitem__(self, key, value):
287
+ self.base[key] = value
288
+
289
+ def __iter__(self):
290
+ return iter(self.doc_ids)
291
+
292
+ def __len__(self):
293
+ return len(self.doc_ids)
294
+
295
+
262
296
  class PosnBitArray:
263
297
 
264
298
  def __init__(self, encoded_term_posns, max_doc_id: int):
@@ -274,9 +308,21 @@ class PosnBitArray:
274
308
  self.docfreq(term_id)
275
309
  self.termfreqs(term_id)
276
310
 
311
+ def filter(self, doc_ids) -> "PosnBitArray":
312
+ """Filter my doc ids to only those in doc_ids."""
313
+ enc_term_posns = self.encoded_term_posns
314
+ if isinstance(enc_term_posns, FilteredPosns):
315
+ self.encoded_term_posns = enc_term_posns.base
316
+ return PosnBitArray(FilteredPosns(self.encoded_term_posns, doc_ids), self.max_doc_id)
317
+
318
+ def _reset_filter(self):
319
+ if isinstance(self.encoded_term_posns, FilteredPosns):
320
+ self.encoded_term_posns = self.encoded_term_posns.base
321
+
277
322
  def clear_cache(self):
278
323
  self.docfreq_cache = {}
279
324
  self.termfreq_cache = {}
325
+ self._reset_filter()
280
326
 
281
327
  def copy(self):
282
328
  new = PosnBitArray(deepcopy(self.encoded_term_posns), self.max_doc_id)
@@ -335,18 +381,22 @@ class PosnBitArray:
335
381
  keys=np.asarray([doc_id], dtype=np.uint64))
336
382
  return term_posns
337
383
 
338
- def phrase_freqs(self, term_ids: List[int], phrase_freqs: np.ndarray,
339
- doc_ids: np.ndarray,
384
+ def empty_buffer(self):
385
+ return np.zeros(int(self.max_doc_id + 1), dtype=np.float64)
386
+
387
+ def phrase_freqs(self, term_ids: List[int],
340
388
  slop: int = 0,
389
+ doc_ids: Optional[np.ndarray] = None,
341
390
  min_posn: Optional[int] = None,
342
391
  max_posn: Optional[int] = None) -> np.ndarray:
392
+ phrase_freqs = self.empty_buffer()
343
393
  if len(term_ids) < 2:
344
394
  raise ValueError("Must have at least two terms")
345
- if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
395
+ if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
346
396
  enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
347
397
  else:
348
398
  keys = None
349
- if phrase_freqs.shape[0] != self.max_doc_id + 1:
399
+ if doc_ids is not None:
350
400
  keys = doc_ids.view(np.uint64)
351
401
 
352
402
  enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
@@ -355,7 +405,7 @@ class PosnBitArray:
355
405
  max_payload=max_posn) for term_id in term_ids]
356
406
 
357
407
  if slop == 0:
358
- return compute_phrase_freqs(enc_term_posns, phrase_freqs)
408
+ return compute_phrase_freqs(enc_term_posns, phrase_freqs, max_doc_id=np.uint64(self.max_doc_id))
359
409
  else:
360
410
  return span_search(enc_term_posns, phrase_freqs, slop)
361
411