searcharray 0.0.54__tar.gz → 0.0.55__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. {searcharray-0.0.54 → searcharray-0.0.55}/PKG-INFO +1 -1
  2. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/phrase/bigram_freqs.py +41 -30
  3. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/phrase/middle_out.py +62 -12
  4. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/postings.py +29 -81
  5. searcharray-0.0.55/searcharray/roaringish/__init__.py +7 -0
  6. searcharray-0.0.54/searcharray/roaringish/snp_ops.c → searcharray-0.0.55/searcharray/roaringish/intersect.c +4203 -10360
  7. searcharray-0.0.55/searcharray/roaringish/intersect.pyi +25 -0
  8. searcharray-0.0.55/searcharray/roaringish/intersect.pyx +395 -0
  9. searcharray-0.0.55/searcharray/roaringish/merge.c +31524 -0
  10. searcharray-0.0.55/searcharray/roaringish/merge.pyi +8 -0
  11. searcharray-0.0.55/searcharray/roaringish/merge.pyx +160 -0
  12. searcharray-0.0.55/searcharray/roaringish/popcount.c +31835 -0
  13. searcharray-0.0.55/searcharray/roaringish/popcount.pyi +20 -0
  14. searcharray-0.0.55/searcharray/roaringish/popcount.pyx +113 -0
  15. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/roaringish/roaringish.py +34 -3
  16. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/roaringish/roaringish_ops.c +300 -1076
  17. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/roaringish/roaringish_ops.pyx +1 -33
  18. searcharray-0.0.55/searcharray/roaringish/search.c +31761 -0
  19. searcharray-0.0.55/searcharray/roaringish/search.pyi +25 -0
  20. searcharray-0.0.55/searcharray/roaringish/search.pyx +141 -0
  21. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/roaringish/snp_ops.pxd +0 -10
  22. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/roaringish/spans.c +242 -370
  23. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/roaringish/spans.pyx +7 -1
  24. searcharray-0.0.55/searcharray/roaringish/unique.c +31772 -0
  25. searcharray-0.0.55/searcharray/roaringish/unique.pyi +6 -0
  26. searcharray-0.0.55/searcharray/roaringish/unique.pyx +147 -0
  27. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/similarity.py +15 -6
  28. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/solr.py +86 -42
  29. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray.egg-info/PKG-INFO +1 -1
  30. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray.egg-info/SOURCES.txt +15 -4
  31. {searcharray-0.0.54 → searcharray-0.0.55}/setup.py +3 -3
  32. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_minmax_posns.py +1 -5
  33. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_msmarco.py +88 -15
  34. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_phrase_matches.py +76 -19
  35. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_search.py +9 -6
  36. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_snp_ops.py +65 -3
  37. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_tmdb.py +1 -1
  38. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_utils.py +5 -2
  39. searcharray-0.0.54/searcharray/phrase/posn_diffs.py +0 -147
  40. searcharray-0.0.54/searcharray/phrase/scan_merge.py +0 -84
  41. searcharray-0.0.54/searcharray/roaringish/__init__.py +0 -4
  42. searcharray-0.0.54/searcharray/roaringish/snp_ops.pyx +0 -664
  43. {searcharray-0.0.54 → searcharray-0.0.55}/README.md +0 -0
  44. {searcharray-0.0.54 → searcharray-0.0.55}/pyproject.toml +0 -0
  45. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/__init__.py +0 -0
  46. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/indexing.py +0 -0
  47. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/phrase/__init__.py +0 -0
  48. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/phrase/spans.py +0 -0
  49. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/term_dict.py +0 -0
  50. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/utils/__init__.py +0 -0
  51. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/utils/bitcount.py +0 -0
  52. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/utils/mat_set.py +0 -0
  53. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/utils/row_viewable_matrix.py +0 -0
  54. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray/utils/sort.py +0 -0
  55. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray.egg-info/dependency_links.txt +0 -0
  56. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray.egg-info/entry_points.txt +0 -0
  57. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray.egg-info/requires.txt +0 -0
  58. {searcharray-0.0.54 → searcharray-0.0.55}/searcharray.egg-info/top_level.txt +0 -0
  59. {searcharray-0.0.54 → searcharray-0.0.55}/setup.cfg +0 -0
  60. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_bitcount64.py +0 -0
  61. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_extension_array.py +0 -0
  62. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_similarity.py +0 -0
  63. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_slop_matches.py +0 -0
  64. {searcharray-0.0.54 → searcharray-0.0.55}/test/test_solr.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.54
3
+ Version: 0.0.55
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -99,7 +99,10 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
99
99
  return phrase_freqs, (lhs_cont, rhs_cont)
100
100
 
101
101
 
102
- def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
102
+ def _inner_bigram_freqs(lhs: np.ndarray,
103
+ rhs: np.ndarray,
104
+ lhs_int: np.ndarray,
105
+ rhs_int: np.ndarray,
103
106
  phrase_freqs: np.ndarray,
104
107
  cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
105
108
  """Count bigram matches between two encoded arrays, within a 64 bit word with same MSBs.
@@ -108,6 +111,8 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
108
111
  -----------
109
112
  lhs: roaringish encoded posn array of left term
110
113
  rhs: roaringish encoded posn array of right term
114
+ lhs_int: np.ndarray, intersected lhs
115
+ rhs_int: np.ndarray, intersected rhs
111
116
  phrase_freqs: np.ndarray, preallocated phrase freqs for output
112
117
  cont_rhs: bool, whether to continue matching on the rhs or lhs
113
118
 
@@ -117,7 +122,7 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
117
122
  cont_next: the next (lhs or rhs) array to continue matching
118
123
 
119
124
  """
120
- lhs_int, rhs_int = encoder.intersect(lhs, rhs)
125
+ # lhs_int, rhs_int = encoder.intersect(lhs, rhs)
121
126
  lhs_doc_ids = encoder.keys(lhs_int)
122
127
  if len(lhs_int) != len(rhs_int):
123
128
  raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
@@ -149,6 +154,7 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
149
154
 
150
155
 
151
156
  def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
157
+ lhs_adj: np.ndarray, rhs_adj: np.ndarray,
152
158
  phrase_freqs: np.ndarray,
153
159
  cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
154
160
  """Count bigram matches between two encoded arrays where they occur in adjacent 64 bit words.
@@ -159,10 +165,9 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
159
165
  rhs_next: the next rhs array to continue matching
160
166
 
161
167
  """
162
- lhs_int, rhs_int = encoder.intersect_rshift(lhs, rhs, rshift=_neg1)
163
- lhs_doc_ids = encoder.keys(lhs_int)
168
+ lhs_doc_ids = encoder.keys(lhs_adj)
164
169
  # lhs lsb set and rhs lsb's most significant bit set
165
- matches = ((lhs_int & _upper_bit) != 0) & ((rhs_int & _1) != 0)
170
+ matches = ((lhs_adj & _upper_bit) != 0) & ((rhs_adj & _1) != 0)
166
171
  unique, counts = np.unique(lhs_doc_ids[matches], return_counts=True)
167
172
  phrase_freqs[unique] += counts
168
173
  # Set lsb to 0 where no match, lsb to 1 where match
@@ -170,40 +175,40 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
170
175
  lhs_next = None if cont == Continuation.RHS else np.asarray([], dtype=np.uint64)
171
176
  if np.any(matches):
172
177
  if cont in [Continuation.RHS, Continuation.BOTH]:
173
- rhs_next = rhs_int[matches]
178
+ rhs_next = rhs_adj[matches]
174
179
  assert rhs_next is not None
175
- rhs_next = encoder.header(rhs_int) | _1
180
+ rhs_next = encoder.header(rhs_adj) | _1
176
181
  elif cont in [Continuation.LHS, Continuation.BOTH]:
177
- rhs_next = rhs_int[matches]
182
+ rhs_next = rhs_adj[matches]
178
183
  assert rhs_next is not None
179
184
  rhs_next |= _1
180
- rhs_next = encoder.header(rhs_int) | _1
181
- lhs_next = lhs_int[matches]
182
- lhs_next = encoder.header(lhs_int) | _upper_bit
185
+ rhs_next = encoder.header(rhs_adj) | _1
186
+ lhs_next = lhs_adj[matches]
187
+ lhs_next = encoder.header(lhs_adj) | _upper_bit
183
188
  rhs_next = None
184
189
  return phrase_freqs, (lhs_next, rhs_next)
185
190
 
186
191
 
187
- def _set_adjbit_at_header(enc1: np.ndarray, just_lsb_enc: np.ndarray,
192
+ def _set_adjbit_at_header(next_inner: np.ndarray, next_adj: np.ndarray,
188
193
  cont: Continuation = Continuation.RHS) -> np.ndarray:
189
194
  """Merge two encoded arrays on their headers."""
190
- if len(enc1) == 0:
191
- return just_lsb_enc
192
- if len(just_lsb_enc) == 0:
193
- return enc1
195
+ if len(next_inner) == 0:
196
+ return next_adj
197
+ if len(next_adj) == 0:
198
+ return next_inner
194
199
 
195
- same_header_enc1, same_header_just_lsb = intersect(enc1, just_lsb_enc,
196
- mask=encoder.header_mask)
200
+ same_header_inner, same_header_adj = intersect(next_inner, next_adj,
201
+ mask=encoder.header_mask)
197
202
  # Set _1 on intersection
198
- ignore_mask = np.ones(len(just_lsb_enc), dtype=bool)
199
- ignore_mask[same_header_just_lsb] = False
200
- if len(same_header_enc1) > 0 and cont == Continuation.RHS:
201
- enc1[same_header_enc1] |= _1
202
- just_lsb_enc = just_lsb_enc[ignore_mask]
203
- if len(same_header_just_lsb) > 0 and cont == Continuation.LHS:
204
- enc1[same_header_enc1] |= _upper_bit
205
- just_lsb_enc = just_lsb_enc[ignore_mask]
206
- return merge(enc1, just_lsb_enc)
203
+ ignore_mask = np.ones(len(next_adj), dtype=bool)
204
+ ignore_mask[same_header_adj] = False
205
+ if len(same_header_inner) > 0 and cont == Continuation.RHS:
206
+ next_inner[same_header_inner] |= _1
207
+ next_adj = next_adj[ignore_mask]
208
+ if len(same_header_adj) > 0 and cont == Continuation.LHS:
209
+ next_inner[same_header_adj] |= _upper_bit
210
+ next_adj = next_adj[ignore_mask]
211
+ return merge(next_inner, next_adj)
207
212
 
208
213
 
209
214
  def bigram_freqs(lhs: np.ndarray,
@@ -242,9 +247,13 @@ def bigram_freqs(lhs: np.ndarray,
242
247
  # rhs_term: 1234 101 0101000100
243
248
  # ^^ ^^ <- inner matches, terms next to each other
244
249
  #
245
- phrase_freqs, (lhs_next_inner, rhs_next_inner)\
246
- = _inner_bigram_freqs(lhs, rhs, phrase_freqs, cont)
250
+ lhs_int, rhs_int, lhs_adj, rhs_adj = encoder.intersect_candidates(lhs, rhs)
247
251
 
252
+ phrase_freqs, (lhs_next_inner, rhs_next_inner)\
253
+ = _inner_bigram_freqs(lhs, rhs,
254
+ lhs_int, rhs_int,
255
+ phrase_freqs,
256
+ cont)
248
257
  # ***************************************************************************
249
258
  # "Adjacent" matches:
250
259
  # Of course we also need to check for matches that span two 64 bit words,
@@ -259,7 +268,9 @@ def bigram_freqs(lhs: np.ndarray,
259
268
  # 1 less than the posn_msb of rhs
260
269
  # so we detect these differently
261
270
  phrase_freqs, (lhs_next_adj, rhs_next_adj)\
262
- = _adjacent_bigram_freqs(lhs, rhs, phrase_freqs, cont)
271
+ = _adjacent_bigram_freqs(lhs, rhs,
272
+ lhs_adj, rhs_adj,
273
+ phrase_freqs, cont)
263
274
 
264
275
  # ***************************************************************************
265
276
  # rhs_next is where the bigram ends on the rhs side, we can use this
@@ -13,7 +13,7 @@ from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
13
13
  from searcharray.phrase.spans import span_search
14
14
  import numbers
15
15
  import logging
16
- from collections import defaultdict
16
+ from collections import defaultdict, abc
17
17
 
18
18
 
19
19
  logger = logging.getLogger(__name__)
@@ -72,6 +72,7 @@ def trim_phrase_search(encoded_posns: List[np.ndarray],
72
72
 
73
73
  def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
74
74
  phrase_freqs: np.ndarray,
75
+ max_doc_id: np.uint64 = _0,
75
76
  trim: bool = True) -> np.ndarray:
76
77
  """Compute phrase freqs from a set of encoded positions."""
77
78
  if len(encoded_posns) < 2:
@@ -86,7 +87,8 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
86
87
  for rhs in encoded_posns[1:]:
87
88
  # Only count the count of the last bigram (ignoring the ones where priors did not match)
88
89
  phrase_freqs[mask] = 0
89
- phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs, cont=Continuation.RHS)
90
+ phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
91
+ cont=Continuation.RHS)
90
92
  assert conts[1] is not None
91
93
  lhs = conts[1]
92
94
  mask &= (phrase_freqs > 0)
@@ -99,6 +101,7 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
99
101
 
100
102
  def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
101
103
  phrase_freqs: np.ndarray,
104
+ max_doc_id: np.uint64 = _0,
102
105
  trim: bool = True) -> np.ndarray:
103
106
  """Compute phrase freqs from a set of encoded positions."""
104
107
  if len(encoded_posns) < 2:
@@ -113,7 +116,8 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
113
116
  for lhs in encoded_posns[-2::-1]:
114
117
  # Only count the count of the last bigram (ignoring the ones where priors did not match)
115
118
  phrase_freqs[mask] = 0
116
- phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs, cont=Continuation.LHS)
119
+ phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
120
+ cont=Continuation.LHS)
117
121
  assert conts[0] is not None
118
122
  rhs = conts[0]
119
123
  mask &= (phrase_freqs > 0)
@@ -126,18 +130,19 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
126
130
 
127
131
  def compute_phrase_freqs(encoded_posns: List[np.ndarray],
128
132
  phrase_freqs: np.ndarray,
133
+ max_doc_id: np.uint64 = _0,
129
134
  trim: bool = False) -> np.ndarray:
130
135
  """Compute phrase freqs from a set of encoded positions."""
131
136
  shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
132
137
  if shortest_len_index <= 1:
133
- return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim)
138
+ return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim, max_doc_id=max_doc_id)
134
139
  elif shortest_len_index >= len(encoded_posns) - 2:
135
- return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim)
140
+ return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim, max_doc_id=max_doc_id)
136
141
  else:
137
142
  # We optimize this case by going middle-out
138
143
  # We can take the min of both directions phrase freqs
139
- lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim)
140
- rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim)
144
+ lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim, max_doc_id=max_doc_id)
145
+ rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim, max_doc_id=max_doc_id)
141
146
  phrase_freqs = np.minimum(lhs, rhs)
142
147
  return phrase_freqs
143
148
 
@@ -259,6 +264,35 @@ def index_range(rng, key):
259
264
  return np.asarray(list(rng))[key]
260
265
 
261
266
 
267
+ class FilteredPosns(abc.Mapping):
268
+ """When someone slices, this lets us avoid repeating constantly
269
+ slicing the keys out of the encoded positions."""
270
+
271
+ def __init__(self, base, doc_ids):
272
+ self.base = base
273
+ self.sliced = {}
274
+ self.doc_ids = doc_ids
275
+
276
+ def __getitem__(self, key):
277
+ """Slice with encoder, cache, and return."""
278
+ if key in self.sliced:
279
+ return self.sliced[key]
280
+
281
+ sliced = encoder.slice(self.base[key],
282
+ keys=self.doc_ids)
283
+ self.sliced[key] = sliced
284
+ return sliced
285
+
286
+ def __setitem__(self, key, value):
287
+ self.base[key] = value
288
+
289
+ def __iter__(self):
290
+ return iter(self.doc_ids)
291
+
292
+ def __len__(self):
293
+ return len(self.doc_ids)
294
+
295
+
262
296
  class PosnBitArray:
263
297
 
264
298
  def __init__(self, encoded_term_posns, max_doc_id: int):
@@ -274,9 +308,21 @@ class PosnBitArray:
274
308
  self.docfreq(term_id)
275
309
  self.termfreqs(term_id)
276
310
 
311
+ def filter(self, doc_ids) -> "PosnBitArray":
312
+ """Filter my doc ids to only those in doc_ids."""
313
+ enc_term_posns = self.encoded_term_posns
314
+ if isinstance(enc_term_posns, FilteredPosns):
315
+ self.encoded_term_posns = enc_term_posns.base
316
+ return PosnBitArray(FilteredPosns(self.encoded_term_posns, doc_ids), self.max_doc_id)
317
+
318
+ def _reset_filter(self):
319
+ if isinstance(self.encoded_term_posns, FilteredPosns):
320
+ self.encoded_term_posns = self.encoded_term_posns.base
321
+
277
322
  def clear_cache(self):
278
323
  self.docfreq_cache = {}
279
324
  self.termfreq_cache = {}
325
+ self._reset_filter()
280
326
 
281
327
  def copy(self):
282
328
  new = PosnBitArray(deepcopy(self.encoded_term_posns), self.max_doc_id)
@@ -335,18 +381,22 @@ class PosnBitArray:
335
381
  keys=np.asarray([doc_id], dtype=np.uint64))
336
382
  return term_posns
337
383
 
338
- def phrase_freqs(self, term_ids: List[int], phrase_freqs: np.ndarray,
339
- doc_ids: np.ndarray,
384
+ def empty_buffer(self):
385
+ return np.zeros(int(self.max_doc_id + 1), dtype=np.float64)
386
+
387
+ def phrase_freqs(self, term_ids: List[int],
340
388
  slop: int = 0,
389
+ doc_ids: Optional[np.ndarray] = None,
341
390
  min_posn: Optional[int] = None,
342
391
  max_posn: Optional[int] = None) -> np.ndarray:
392
+ phrase_freqs = self.empty_buffer()
343
393
  if len(term_ids) < 2:
344
394
  raise ValueError("Must have at least two terms")
345
- if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
395
+ if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
346
396
  enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
347
397
  else:
348
398
  keys = None
349
- if phrase_freqs.shape[0] != self.max_doc_id + 1:
399
+ if doc_ids is not None:
350
400
  keys = doc_ids.view(np.uint64)
351
401
 
352
402
  enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
@@ -355,7 +405,7 @@ class PosnBitArray:
355
405
  max_payload=max_posn) for term_id in term_ids]
356
406
 
357
407
  if slop == 0:
358
- return compute_phrase_freqs(enc_term_posns, phrase_freqs)
408
+ return compute_phrase_freqs(enc_term_posns, phrase_freqs, max_doc_id=np.uint64(self.max_doc_id))
359
409
  else:
360
410
  return span_search(enc_term_posns, phrase_freqs, slop)
361
411
 
@@ -12,8 +12,6 @@ from typing import List, Union, Optional, Iterable
12
12
 
13
13
 
14
14
  import numpy as np
15
- from searcharray.phrase.scan_merge import scan_merge_ins
16
- from searcharray.phrase.posn_diffs import compute_phrase_freqs
17
15
  from searcharray.phrase.middle_out import PosnBitArray
18
16
  from searcharray.similarity import Similarity, default_bm25
19
17
  from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
@@ -318,7 +316,7 @@ class SearchArray(ExtensionArray):
318
316
  if not self.avoid_copies:
319
317
  sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
320
318
  else:
321
- sliced_posns = self.posns
319
+ sliced_posns = self.posns.filter(sliced_tfs.rows)
322
320
 
323
321
  arr = SearchArray([], tokenizer=self.tokenizer)
324
322
  arr.term_mat = sliced_tfs
@@ -560,7 +558,7 @@ class SearchArray(ExtensionArray):
560
558
  """
561
559
 
562
560
  # ***********************************************************
563
- # Search functionality
561
+ # Search API
564
562
  # ***********************************************************
565
563
  def termfreqs(self, token: Union[List[str], str],
566
564
  slop: int = 0,
@@ -568,23 +566,25 @@ class SearchArray(ExtensionArray):
568
566
  max_posn: Optional[int] = None) -> np.ndarray:
569
567
  token = self._check_token_arg(token)
570
568
  if isinstance(token, list):
571
- return self.phrase_freq(token, slop=slop, min_posn=min_posn, max_posn=max_posn)
569
+ return self._phrase_freq(token, slop=slop, min_posn=min_posn, max_posn=max_posn)
572
570
 
573
571
  try:
574
572
  term_id = self.term_dict.get_term_id(token)
575
- slice_of_rows = None
573
+ slice_of_rows = self.term_mat.rows
576
574
  if self.term_mat.subset:
577
575
  matches = np.zeros(len(self), dtype=np.float32)
578
- slice_of_rows = self.term_mat.rows
579
576
  doc_ids, termfreqs = self.posns.termfreqs(term_id,
580
- doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
577
+ doc_ids=slice_of_rows,
578
+ min_posn=min_posn,
579
+ max_posn=max_posn)
581
580
  mask = np.isin(self.term_mat.rows, doc_ids)
582
581
  matches[mask] = termfreqs
583
582
  return matches
584
583
  else:
585
584
  doc_ids, termfreqs = self.posns.termfreqs(term_id,
586
- doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
587
- # matches[doc_ids] = termfreqs
585
+ doc_ids=None,
586
+ min_posn=min_posn,
587
+ max_posn=max_posn)
588
588
  return as_dense(doc_ids, termfreqs, len(self))
589
589
  except TermMissingError:
590
590
  return np.zeros(len(self), dtype=np.float32)
@@ -601,16 +601,8 @@ class SearchArray(ExtensionArray):
601
601
  def doclengths(self) -> np.ndarray:
602
602
  return self.doc_lens
603
603
 
604
- def match(self, token: Union[List[str], str], slop: int = 0) -> np.ndarray:
605
- """Return a boolean numpy array indicating which elements contain the given term."""
606
- token = self._check_token_arg(token)
607
- if isinstance(token, list):
608
- term_freq = self.phrase_freq(token, slop=slop)
609
- else:
610
- term_freq = self.termfreqs(token)
611
- return term_freq > 0
612
-
613
- def score(self, token: Union[str, List[str]], similarity: Similarity = default_bm25,
604
+ def score(self, token: Union[str, List[str]],
605
+ similarity: Similarity = default_bm25,
614
606
  min_posn: Optional[int] = None,
615
607
  max_posn: Optional[int] = None) -> np.ndarray:
616
608
  """Score each doc using a similarity function.
@@ -619,6 +611,8 @@ class SearchArray(ExtensionArray):
619
611
  ----------
620
612
  token : str or list of str of what to search (already tokenized)
621
613
  similarity : How to score the documents. Default is BM25.
614
+ min_posn : int - minimum position of the term in the document, in multiples of 18
615
+ max_posn : int - maximum position of the term in the document, in multiples of 18
622
616
  """
623
617
  # Get term freqs per token
624
618
  token = self._check_token_arg(token)
@@ -642,67 +636,21 @@ class SearchArray(ExtensionArray):
642
636
  posns = self.posns.positions(term_id, doc_ids=key)
643
637
  return posns
644
638
 
645
- def and_query(self, tokens: Union[List[str], List[List[str]]]) -> np.ndarray:
646
- """Return a mask on the postings array indicating which elements contain all terms."""
647
- masks = [self.match(term) for term in tokens]
648
- mask = np.ones(len(self), dtype=bool)
649
- for curr_mask in masks:
650
- mask = mask & curr_mask
651
- return mask
652
-
653
- def or_query(self, tokens: Union[List[str], List[List[str]]], min_should_match: int = 1) -> np.ndarray:
654
- """Return a mask on the postings array indicating which elements contain all terms."""
655
- masks = [self.match(term) for term in tokens]
656
- mask = np.sum(masks, axis=0) >= min_should_match
657
- return mask
658
-
659
- def phrase_freq(self, tokens: List[str],
660
- slop=0,
661
- min_posn: Optional[int] = None,
662
- max_posn: Optional[int] = None) -> np.ndarray:
663
- phrase_freqs = np.zeros(len(self))
639
+ def _phrase_freq(self, tokens: List[str],
640
+ slop=0,
641
+ min_posn: Optional[int] = None,
642
+ max_posn: Optional[int] = None) -> np.ndarray:
664
643
  try:
665
- doc_ids = self.term_mat.rows
644
+ # Decide how/if we need to filter doc ids
666
645
  term_ids = [self.term_dict.get_term_id(token) for token in tokens]
667
- return self.posns.phrase_freqs(term_ids, doc_ids=doc_ids,
668
- phrase_freqs=phrase_freqs,
669
- slop=slop,
670
- min_posn=min_posn,
671
- max_posn=max_posn)
672
- except TermMissingError:
673
- return phrase_freqs
674
-
675
- def phrase_freq_scan(self, tokens: List[str], mask=None, slop=0) -> np.ndarray:
676
- if mask is None:
677
- mask = self.and_query(tokens)
678
-
679
- if np.sum(mask) == 0:
680
- return mask
681
-
682
- # Gather positions
683
- posns = [self.positions(token, mask) for token in tokens]
684
- phrase_freqs = np.zeros(len(self))
685
-
686
- phrase_freqs[mask] = scan_merge_ins(posns, phrase_freqs[mask], slop=slop + 1)
687
- return phrase_freqs
688
-
689
- def phrase_freq_every_diff(self, tokens: List[str], slop=0) -> np.ndarray:
690
- phrase_freqs = -np.ones(len(self))
691
-
692
- mask = self.and_query(tokens)
693
- phrase_freqs[~mask] = 0
694
- if np.sum(mask) == 0:
646
+ phrase_freqs = self.posns.phrase_freqs(term_ids,
647
+ slop=slop,
648
+ min_posn=min_posn,
649
+ max_posn=max_posn)
650
+ if self.term_mat.subset:
651
+ return phrase_freqs[self.term_mat.rows]
695
652
  return phrase_freqs
696
-
697
- term_posns = [self.positions(term, mask) for term in tokens]
698
- for width in [10, 20, 30, 40]:
699
- phrase_freqs[mask] = compute_phrase_freqs(term_posns,
700
- phrase_freqs[mask],
701
- slop=slop + 1,
702
- width=width)
703
-
704
- remaining_mask = phrase_freqs == -1
705
- if np.any(remaining_mask):
706
- remainder_freqs = self.phrase_freq_scan(tokens, mask=remaining_mask, slop=slop)
707
- phrase_freqs[remaining_mask] = remainder_freqs[remaining_mask]
708
- return phrase_freqs
653
+ except TermMissingError:
654
+ if self.term_mat.subset:
655
+ return np.zeros(len(self), dtype=np.float32)
656
+ return self.posns.empty_buffer()
@@ -0,0 +1,7 @@
1
+ from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
2
+ from searcharray.roaringish.search import count_odds # noqa
3
+ from searcharray.roaringish.popcount import popcount_reduce_at, popcount64 # noqa
4
+ from searcharray.roaringish.intersect import intersect, adjacent # noqa
5
+ from searcharray.roaringish.unique import unique # noqa
6
+ from searcharray.roaringish.merge import merge # noqa
7
+ from searcharray.roaringish.spans import span_search # noqa