searcharray 0.0.53__tar.gz → 0.0.55__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. {searcharray-0.0.53 → searcharray-0.0.55}/PKG-INFO +1 -1
  2. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/phrase/bigram_freqs.py +43 -39
  3. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/phrase/middle_out.py +62 -12
  4. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/postings.py +63 -81
  5. searcharray-0.0.55/searcharray/roaringish/__init__.py +7 -0
  6. searcharray-0.0.53/searcharray/roaringish/snp_ops.c → searcharray-0.0.55/searcharray/roaringish/intersect.c +12344 -17504
  7. searcharray-0.0.55/searcharray/roaringish/intersect.pyi +25 -0
  8. searcharray-0.0.55/searcharray/roaringish/intersect.pyx +395 -0
  9. searcharray-0.0.55/searcharray/roaringish/merge.c +31524 -0
  10. searcharray-0.0.55/searcharray/roaringish/merge.pyi +8 -0
  11. searcharray-0.0.55/searcharray/roaringish/merge.pyx +160 -0
  12. searcharray-0.0.55/searcharray/roaringish/popcount.c +31835 -0
  13. searcharray-0.0.55/searcharray/roaringish/popcount.pyi +20 -0
  14. searcharray-0.0.55/searcharray/roaringish/popcount.pyx +113 -0
  15. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/roaringish.py +34 -3
  16. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/roaringish_ops.c +300 -1076
  17. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/roaringish_ops.pyx +1 -33
  18. searcharray-0.0.55/searcharray/roaringish/search.c +31761 -0
  19. searcharray-0.0.55/searcharray/roaringish/search.pyi +25 -0
  20. searcharray-0.0.55/searcharray/roaringish/search.pyx +141 -0
  21. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/snp_ops.pxd +0 -10
  22. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/spans.c +242 -370
  23. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/roaringish/spans.pyx +7 -1
  24. searcharray-0.0.55/searcharray/roaringish/unique.c +31772 -0
  25. searcharray-0.0.55/searcharray/roaringish/unique.pyi +6 -0
  26. searcharray-0.0.55/searcharray/roaringish/unique.pyx +147 -0
  27. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/similarity.py +15 -6
  28. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/solr.py +86 -42
  29. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/PKG-INFO +1 -1
  30. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/SOURCES.txt +15 -4
  31. {searcharray-0.0.53 → searcharray-0.0.55}/setup.py +3 -3
  32. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_minmax_posns.py +1 -5
  33. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_msmarco.py +176 -145
  34. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_phrase_matches.py +76 -19
  35. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_search.py +9 -6
  36. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_snp_ops.py +100 -7
  37. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_tmdb.py +1 -1
  38. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_utils.py +8 -4
  39. searcharray-0.0.53/searcharray/phrase/posn_diffs.py +0 -147
  40. searcharray-0.0.53/searcharray/phrase/scan_merge.py +0 -84
  41. searcharray-0.0.53/searcharray/roaringish/__init__.py +0 -4
  42. searcharray-0.0.53/searcharray/roaringish/snp_ops.pyx +0 -624
  43. {searcharray-0.0.53 → searcharray-0.0.55}/README.md +0 -0
  44. {searcharray-0.0.53 → searcharray-0.0.55}/pyproject.toml +0 -0
  45. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/__init__.py +0 -0
  46. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/indexing.py +0 -0
  47. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/phrase/__init__.py +0 -0
  48. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/phrase/spans.py +0 -0
  49. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/term_dict.py +0 -0
  50. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/utils/__init__.py +0 -0
  51. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/utils/bitcount.py +0 -0
  52. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/utils/mat_set.py +0 -0
  53. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/utils/row_viewable_matrix.py +0 -0
  54. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray/utils/sort.py +0 -0
  55. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/dependency_links.txt +0 -0
  56. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/entry_points.txt +0 -0
  57. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/requires.txt +0 -0
  58. {searcharray-0.0.53 → searcharray-0.0.55}/searcharray.egg-info/top_level.txt +0 -0
  59. {searcharray-0.0.53 → searcharray-0.0.55}/setup.cfg +0 -0
  60. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_bitcount64.py +0 -0
  61. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_extension_array.py +0 -0
  62. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_similarity.py +0 -0
  63. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_slop_matches.py +0 -0
  64. {searcharray-0.0.53 → searcharray-0.0.55}/test/test_solr.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.53
3
+ Version: 0.0.55
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -5,7 +5,7 @@ from searcharray.roaringish import RoaringishEncoder
5
5
  import logging
6
6
  from enum import Enum
7
7
 
8
- from searcharray.roaringish import intersect, popcount64, merge
8
+ from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at
9
9
 
10
10
 
11
11
  logger = logging.getLogger(__name__)
@@ -99,7 +99,10 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
99
99
  return phrase_freqs, (lhs_cont, rhs_cont)
100
100
 
101
101
 
102
- def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
102
+ def _inner_bigram_freqs(lhs: np.ndarray,
103
+ rhs: np.ndarray,
104
+ lhs_int: np.ndarray,
105
+ rhs_int: np.ndarray,
103
106
  phrase_freqs: np.ndarray,
104
107
  cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
105
108
  """Count bigram matches between two encoded arrays, within a 64 bit word with same MSBs.
@@ -108,6 +111,8 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
108
111
  -----------
109
112
  lhs: roaringish encoded posn array of left term
110
113
  rhs: roaringish encoded posn array of right term
114
+ lhs_int: np.ndarray, intersected lhs
115
+ rhs_int: np.ndarray, intersected rhs
111
116
  phrase_freqs: np.ndarray, preallocated phrase freqs for output
112
117
  cont_rhs: bool, whether to continue matching on the rhs or lhs
113
118
 
@@ -117,7 +122,7 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
117
122
  cont_next: the next (lhs or rhs) array to continue matching
118
123
 
119
124
  """
120
- lhs_int, rhs_int = encoder.intersect(lhs, rhs)
125
+ # lhs_int, rhs_int = encoder.intersect(lhs, rhs)
121
126
  lhs_doc_ids = encoder.keys(lhs_int)
122
127
  if len(lhs_int) != len(rhs_int):
123
128
  raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
@@ -144,18 +149,12 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
144
149
  lhs_next = overlap_bits.copy()
145
150
  lhs_next |= (lhs_int & (encoder.key_mask | encoder.payload_msb_mask))
146
151
 
147
- matches2 = overlap_bits > 0
148
- if np.any(matches2):
149
- transitions = np.argwhere(np.diff(lhs_doc_ids[matches2]) != 0).flatten() + 1
150
- transitions = np.insert(transitions, 0, 0)
151
- counted_bits = popcount64(overlap_bits[matches2])
152
- reduced = np.add.reduceat(counted_bits,
153
- transitions)
154
- phrase_freqs[np.unique(lhs_doc_ids[matches2])] += reduced
152
+ popcount_reduce_at(lhs_doc_ids, overlap_bits, phrase_freqs)
155
153
  return phrase_freqs, (lhs_next, rhs_next)
156
154
 
157
155
 
158
156
  def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
157
+ lhs_adj: np.ndarray, rhs_adj: np.ndarray,
159
158
  phrase_freqs: np.ndarray,
160
159
  cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
161
160
  """Count bigram matches between two encoded arrays where they occur in adjacent 64 bit words.
@@ -166,10 +165,9 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
166
165
  rhs_next: the next rhs array to continue matching
167
166
 
168
167
  """
169
- lhs_int, rhs_int = encoder.intersect_rshift(lhs, rhs, rshift=_neg1)
170
- lhs_doc_ids = encoder.keys(lhs_int)
168
+ lhs_doc_ids = encoder.keys(lhs_adj)
171
169
  # lhs lsb set and rhs lsb's most significant bit set
172
- matches = ((lhs_int & _upper_bit) != 0) & ((rhs_int & _1) != 0)
170
+ matches = ((lhs_adj & _upper_bit) != 0) & ((rhs_adj & _1) != 0)
173
171
  unique, counts = np.unique(lhs_doc_ids[matches], return_counts=True)
174
172
  phrase_freqs[unique] += counts
175
173
  # Set lsb to 0 where no match, lsb to 1 where match
@@ -177,40 +175,40 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
177
175
  lhs_next = None if cont == Continuation.RHS else np.asarray([], dtype=np.uint64)
178
176
  if np.any(matches):
179
177
  if cont in [Continuation.RHS, Continuation.BOTH]:
180
- rhs_next = rhs_int[matches]
178
+ rhs_next = rhs_adj[matches]
181
179
  assert rhs_next is not None
182
- rhs_next = encoder.header(rhs_int) | _1
180
+ rhs_next = encoder.header(rhs_adj) | _1
183
181
  elif cont in [Continuation.LHS, Continuation.BOTH]:
184
- rhs_next = rhs_int[matches]
182
+ rhs_next = rhs_adj[matches]
185
183
  assert rhs_next is not None
186
184
  rhs_next |= _1
187
- rhs_next = encoder.header(rhs_int) | _1
188
- lhs_next = lhs_int[matches]
189
- lhs_next = encoder.header(lhs_int) | _upper_bit
185
+ rhs_next = encoder.header(rhs_adj) | _1
186
+ lhs_next = lhs_adj[matches]
187
+ lhs_next = encoder.header(lhs_adj) | _upper_bit
190
188
  rhs_next = None
191
189
  return phrase_freqs, (lhs_next, rhs_next)
192
190
 
193
191
 
194
- def _set_adjbit_at_header(enc1: np.ndarray, just_lsb_enc: np.ndarray,
192
+ def _set_adjbit_at_header(next_inner: np.ndarray, next_adj: np.ndarray,
195
193
  cont: Continuation = Continuation.RHS) -> np.ndarray:
196
194
  """Merge two encoded arrays on their headers."""
197
- if len(enc1) == 0:
198
- return just_lsb_enc
199
- if len(just_lsb_enc) == 0:
200
- return enc1
195
+ if len(next_inner) == 0:
196
+ return next_adj
197
+ if len(next_adj) == 0:
198
+ return next_inner
201
199
 
202
- same_header_enc1, same_header_just_lsb = intersect(enc1, just_lsb_enc,
203
- mask=encoder.header_mask)
200
+ same_header_inner, same_header_adj = intersect(next_inner, next_adj,
201
+ mask=encoder.header_mask)
204
202
  # Set _1 on intersection
205
- ignore_mask = np.ones(len(just_lsb_enc), dtype=bool)
206
- ignore_mask[same_header_just_lsb] = False
207
- if len(same_header_enc1) > 0 and cont == Continuation.RHS:
208
- enc1[same_header_enc1] |= _1
209
- just_lsb_enc = just_lsb_enc[ignore_mask]
210
- if len(same_header_just_lsb) > 0 and cont == Continuation.LHS:
211
- enc1[same_header_enc1] |= _upper_bit
212
- just_lsb_enc = just_lsb_enc[ignore_mask]
213
- return merge(enc1, just_lsb_enc)
203
+ ignore_mask = np.ones(len(next_adj), dtype=bool)
204
+ ignore_mask[same_header_adj] = False
205
+ if len(same_header_inner) > 0 and cont == Continuation.RHS:
206
+ next_inner[same_header_inner] |= _1
207
+ next_adj = next_adj[ignore_mask]
208
+ if len(same_header_adj) > 0 and cont == Continuation.LHS:
209
+ next_inner[same_header_adj] |= _upper_bit
210
+ next_adj = next_adj[ignore_mask]
211
+ return merge(next_inner, next_adj)
214
212
 
215
213
 
216
214
  def bigram_freqs(lhs: np.ndarray,
@@ -249,9 +247,13 @@ def bigram_freqs(lhs: np.ndarray,
249
247
  # rhs_term: 1234 101 0101000100
250
248
  # ^^ ^^ <- inner matches, terms next to each other
251
249
  #
252
- phrase_freqs, (lhs_next_inner, rhs_next_inner)\
253
- = _inner_bigram_freqs(lhs, rhs, phrase_freqs, cont)
250
+ lhs_int, rhs_int, lhs_adj, rhs_adj = encoder.intersect_candidates(lhs, rhs)
254
251
 
252
+ phrase_freqs, (lhs_next_inner, rhs_next_inner)\
253
+ = _inner_bigram_freqs(lhs, rhs,
254
+ lhs_int, rhs_int,
255
+ phrase_freqs,
256
+ cont)
255
257
  # ***************************************************************************
256
258
  # "Adjacent" matches:
257
259
  # Of course we also need to check for matches that span two 64 bit words,
@@ -266,7 +268,9 @@ def bigram_freqs(lhs: np.ndarray,
266
268
  # 1 less than the posn_msb of rhs
267
269
  # so we detect these differently
268
270
  phrase_freqs, (lhs_next_adj, rhs_next_adj)\
269
- = _adjacent_bigram_freqs(lhs, rhs, phrase_freqs, cont)
271
+ = _adjacent_bigram_freqs(lhs, rhs,
272
+ lhs_adj, rhs_adj,
273
+ phrase_freqs, cont)
270
274
 
271
275
  # ***************************************************************************
272
276
  # rhs_next is where the bigram ends on the rhs side, we can use this
@@ -13,7 +13,7 @@ from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
13
13
  from searcharray.phrase.spans import span_search
14
14
  import numbers
15
15
  import logging
16
- from collections import defaultdict
16
+ from collections import defaultdict, abc
17
17
 
18
18
 
19
19
  logger = logging.getLogger(__name__)
@@ -72,6 +72,7 @@ def trim_phrase_search(encoded_posns: List[np.ndarray],
72
72
 
73
73
  def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
74
74
  phrase_freqs: np.ndarray,
75
+ max_doc_id: np.uint64 = _0,
75
76
  trim: bool = True) -> np.ndarray:
76
77
  """Compute phrase freqs from a set of encoded positions."""
77
78
  if len(encoded_posns) < 2:
@@ -86,7 +87,8 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
86
87
  for rhs in encoded_posns[1:]:
87
88
  # Only count the count of the last bigram (ignoring the ones where priors did not match)
88
89
  phrase_freqs[mask] = 0
89
- phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs, cont=Continuation.RHS)
90
+ phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
91
+ cont=Continuation.RHS)
90
92
  assert conts[1] is not None
91
93
  lhs = conts[1]
92
94
  mask &= (phrase_freqs > 0)
@@ -99,6 +101,7 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
99
101
 
100
102
  def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
101
103
  phrase_freqs: np.ndarray,
104
+ max_doc_id: np.uint64 = _0,
102
105
  trim: bool = True) -> np.ndarray:
103
106
  """Compute phrase freqs from a set of encoded positions."""
104
107
  if len(encoded_posns) < 2:
@@ -113,7 +116,8 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
113
116
  for lhs in encoded_posns[-2::-1]:
114
117
  # Only count the count of the last bigram (ignoring the ones where priors did not match)
115
118
  phrase_freqs[mask] = 0
116
- phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs, cont=Continuation.LHS)
119
+ phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
120
+ cont=Continuation.LHS)
117
121
  assert conts[0] is not None
118
122
  rhs = conts[0]
119
123
  mask &= (phrase_freqs > 0)
@@ -126,18 +130,19 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
126
130
 
127
131
  def compute_phrase_freqs(encoded_posns: List[np.ndarray],
128
132
  phrase_freqs: np.ndarray,
133
+ max_doc_id: np.uint64 = _0,
129
134
  trim: bool = False) -> np.ndarray:
130
135
  """Compute phrase freqs from a set of encoded positions."""
131
136
  shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
132
137
  if shortest_len_index <= 1:
133
- return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim)
138
+ return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim, max_doc_id=max_doc_id)
134
139
  elif shortest_len_index >= len(encoded_posns) - 2:
135
- return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim)
140
+ return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim, max_doc_id=max_doc_id)
136
141
  else:
137
142
  # We optimize this case by going middle-out
138
143
  # We can take the min of both directions phrase freqs
139
- lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim)
140
- rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim)
144
+ lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim, max_doc_id=max_doc_id)
145
+ rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim, max_doc_id=max_doc_id)
141
146
  phrase_freqs = np.minimum(lhs, rhs)
142
147
  return phrase_freqs
143
148
 
@@ -259,6 +264,35 @@ def index_range(rng, key):
259
264
  return np.asarray(list(rng))[key]
260
265
 
261
266
 
267
+ class FilteredPosns(abc.Mapping):
268
+ """When someone slices, this lets us avoid repeating constantly
269
+ slicing the keys out of the encoded positions."""
270
+
271
+ def __init__(self, base, doc_ids):
272
+ self.base = base
273
+ self.sliced = {}
274
+ self.doc_ids = doc_ids
275
+
276
+ def __getitem__(self, key):
277
+ """Slice with encoder, cache, and return."""
278
+ if key in self.sliced:
279
+ return self.sliced[key]
280
+
281
+ sliced = encoder.slice(self.base[key],
282
+ keys=self.doc_ids)
283
+ self.sliced[key] = sliced
284
+ return sliced
285
+
286
+ def __setitem__(self, key, value):
287
+ self.base[key] = value
288
+
289
+ def __iter__(self):
290
+ return iter(self.doc_ids)
291
+
292
+ def __len__(self):
293
+ return len(self.doc_ids)
294
+
295
+
262
296
  class PosnBitArray:
263
297
 
264
298
  def __init__(self, encoded_term_posns, max_doc_id: int):
@@ -274,9 +308,21 @@ class PosnBitArray:
274
308
  self.docfreq(term_id)
275
309
  self.termfreqs(term_id)
276
310
 
311
+ def filter(self, doc_ids) -> "PosnBitArray":
312
+ """Filter my doc ids to only those in doc_ids."""
313
+ enc_term_posns = self.encoded_term_posns
314
+ if isinstance(enc_term_posns, FilteredPosns):
315
+ self.encoded_term_posns = enc_term_posns.base
316
+ return PosnBitArray(FilteredPosns(self.encoded_term_posns, doc_ids), self.max_doc_id)
317
+
318
+ def _reset_filter(self):
319
+ if isinstance(self.encoded_term_posns, FilteredPosns):
320
+ self.encoded_term_posns = self.encoded_term_posns.base
321
+
277
322
  def clear_cache(self):
278
323
  self.docfreq_cache = {}
279
324
  self.termfreq_cache = {}
325
+ self._reset_filter()
280
326
 
281
327
  def copy(self):
282
328
  new = PosnBitArray(deepcopy(self.encoded_term_posns), self.max_doc_id)
@@ -335,18 +381,22 @@ class PosnBitArray:
335
381
  keys=np.asarray([doc_id], dtype=np.uint64))
336
382
  return term_posns
337
383
 
338
- def phrase_freqs(self, term_ids: List[int], phrase_freqs: np.ndarray,
339
- doc_ids: np.ndarray,
384
+ def empty_buffer(self):
385
+ return np.zeros(int(self.max_doc_id + 1), dtype=np.float64)
386
+
387
+ def phrase_freqs(self, term_ids: List[int],
340
388
  slop: int = 0,
389
+ doc_ids: Optional[np.ndarray] = None,
341
390
  min_posn: Optional[int] = None,
342
391
  max_posn: Optional[int] = None) -> np.ndarray:
392
+ phrase_freqs = self.empty_buffer()
343
393
  if len(term_ids) < 2:
344
394
  raise ValueError("Must have at least two terms")
345
- if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
395
+ if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
346
396
  enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
347
397
  else:
348
398
  keys = None
349
- if phrase_freqs.shape[0] != self.max_doc_id + 1:
399
+ if doc_ids is not None:
350
400
  keys = doc_ids.view(np.uint64)
351
401
 
352
402
  enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
@@ -355,7 +405,7 @@ class PosnBitArray:
355
405
  max_payload=max_posn) for term_id in term_ids]
356
406
 
357
407
  if slop == 0:
358
- return compute_phrase_freqs(enc_term_posns, phrase_freqs)
408
+ return compute_phrase_freqs(enc_term_posns, phrase_freqs, max_doc_id=np.uint64(self.max_doc_id))
359
409
  else:
360
410
  return span_search(enc_term_posns, phrase_freqs, slop)
361
411
 
@@ -12,8 +12,6 @@ from typing import List, Union, Optional, Iterable
12
12
 
13
13
 
14
14
  import numpy as np
15
- from searcharray.phrase.scan_merge import scan_merge_ins
16
- from searcharray.phrase.posn_diffs import compute_phrase_freqs
17
15
  from searcharray.phrase.middle_out import PosnBitArray
18
16
  from searcharray.similarity import Similarity, default_bm25
19
17
  from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
@@ -318,7 +316,7 @@ class SearchArray(ExtensionArray):
318
316
  if not self.avoid_copies:
319
317
  sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
320
318
  else:
321
- sliced_posns = self.posns
319
+ sliced_posns = self.posns.filter(sliced_tfs.rows)
322
320
 
323
321
  arr = SearchArray([], tokenizer=self.tokenizer)
324
322
  arr.term_mat = sliced_tfs
@@ -525,8 +523,42 @@ class SearchArray(ExtensionArray):
525
523
  else:
526
524
  raise TypeError("Expected a string or list of strings for phrases")
527
525
 
526
+ def memory_report(self):
527
+ """Return a string with memory usage information."""
528
+ term_mat_bytes = self.term_mat.nbytes
529
+ posns_bytes = self.posns.nbytes
530
+ term_dict_bytes = self.term_dict.nbytes
531
+ num_terms = len(self.term_dict)
532
+ top_10_terms_size = []
533
+
534
+ for i in range(10):
535
+ term_len = len(self.posns.encoded_term_posns[i])
536
+ term = self.term_dict.get_term(i)
537
+ top_10_terms_size.append((term, term_len))
538
+
539
+ return f"""
540
+ SearchArray Memory Report
541
+ -------------------------
542
+ Number of Terms: {num_terms}
543
+ -------------------------
544
+ Term Matrix: {term_mat_bytes}
545
+ Positions: {posns_bytes}
546
+ Term Dictionary: {term_dict_bytes}
547
+ --------------------------
548
+ Term 0: {top_10_terms_size[0]}
549
+ Term 1: {top_10_terms_size[1]}
550
+ Term 2: {top_10_terms_size[2]}
551
+ Term 3: {top_10_terms_size[3]}
552
+ Term 4: {top_10_terms_size[4]}
553
+ Term 5: {top_10_terms_size[5]}
554
+ Term 6: {top_10_terms_size[6]}
555
+ Term 7: {top_10_terms_size[7]}
556
+ Term 8: {top_10_terms_size[8]}
557
+ Term 9: {top_10_terms_size[9]}
558
+ """
559
+
528
560
  # ***********************************************************
529
- # Search functionality
561
+ # Search API
530
562
  # ***********************************************************
531
563
  def termfreqs(self, token: Union[List[str], str],
532
564
  slop: int = 0,
@@ -534,23 +566,25 @@ class SearchArray(ExtensionArray):
534
566
  max_posn: Optional[int] = None) -> np.ndarray:
535
567
  token = self._check_token_arg(token)
536
568
  if isinstance(token, list):
537
- return self.phrase_freq(token, slop=slop, min_posn=min_posn, max_posn=max_posn)
569
+ return self._phrase_freq(token, slop=slop, min_posn=min_posn, max_posn=max_posn)
538
570
 
539
571
  try:
540
572
  term_id = self.term_dict.get_term_id(token)
541
- slice_of_rows = None
573
+ slice_of_rows = self.term_mat.rows
542
574
  if self.term_mat.subset:
543
575
  matches = np.zeros(len(self), dtype=np.float32)
544
- slice_of_rows = self.term_mat.rows
545
576
  doc_ids, termfreqs = self.posns.termfreqs(term_id,
546
- doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
577
+ doc_ids=slice_of_rows,
578
+ min_posn=min_posn,
579
+ max_posn=max_posn)
547
580
  mask = np.isin(self.term_mat.rows, doc_ids)
548
581
  matches[mask] = termfreqs
549
582
  return matches
550
583
  else:
551
584
  doc_ids, termfreqs = self.posns.termfreqs(term_id,
552
- doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
553
- # matches[doc_ids] = termfreqs
585
+ doc_ids=None,
586
+ min_posn=min_posn,
587
+ max_posn=max_posn)
554
588
  return as_dense(doc_ids, termfreqs, len(self))
555
589
  except TermMissingError:
556
590
  return np.zeros(len(self), dtype=np.float32)
@@ -567,16 +601,8 @@ class SearchArray(ExtensionArray):
567
601
  def doclengths(self) -> np.ndarray:
568
602
  return self.doc_lens
569
603
 
570
- def match(self, token: Union[List[str], str], slop: int = 0) -> np.ndarray:
571
- """Return a boolean numpy array indicating which elements contain the given term."""
572
- token = self._check_token_arg(token)
573
- if isinstance(token, list):
574
- term_freq = self.phrase_freq(token, slop=slop)
575
- else:
576
- term_freq = self.termfreqs(token)
577
- return term_freq > 0
578
-
579
- def score(self, token: Union[str, List[str]], similarity: Similarity = default_bm25,
604
+ def score(self, token: Union[str, List[str]],
605
+ similarity: Similarity = default_bm25,
580
606
  min_posn: Optional[int] = None,
581
607
  max_posn: Optional[int] = None) -> np.ndarray:
582
608
  """Score each doc using a similarity function.
@@ -585,6 +611,8 @@ class SearchArray(ExtensionArray):
585
611
  ----------
586
612
  token : str or list of str of what to search (already tokenized)
587
613
  similarity : How to score the documents. Default is BM25.
614
+ min_posn : int - minimum position of the term in the document, in multiples of 18
615
+ max_posn : int - maximum position of the term in the document, in multiples of 18
588
616
  """
589
617
  # Get term freqs per token
590
618
  token = self._check_token_arg(token)
@@ -608,67 +636,21 @@ class SearchArray(ExtensionArray):
608
636
  posns = self.posns.positions(term_id, doc_ids=key)
609
637
  return posns
610
638
 
611
- def and_query(self, tokens: Union[List[str], List[List[str]]]) -> np.ndarray:
612
- """Return a mask on the postings array indicating which elements contain all terms."""
613
- masks = [self.match(term) for term in tokens]
614
- mask = np.ones(len(self), dtype=bool)
615
- for curr_mask in masks:
616
- mask = mask & curr_mask
617
- return mask
618
-
619
- def or_query(self, tokens: Union[List[str], List[List[str]]], min_should_match: int = 1) -> np.ndarray:
620
- """Return a mask on the postings array indicating which elements contain all terms."""
621
- masks = [self.match(term) for term in tokens]
622
- mask = np.sum(masks, axis=0) >= min_should_match
623
- return mask
624
-
625
- def phrase_freq(self, tokens: List[str],
626
- slop=0,
627
- min_posn: Optional[int] = None,
628
- max_posn: Optional[int] = None) -> np.ndarray:
629
- phrase_freqs = np.zeros(len(self))
639
+ def _phrase_freq(self, tokens: List[str],
640
+ slop=0,
641
+ min_posn: Optional[int] = None,
642
+ max_posn: Optional[int] = None) -> np.ndarray:
630
643
  try:
631
- doc_ids = self.term_mat.rows
644
+ # Decide how/if we need to filter doc ids
632
645
  term_ids = [self.term_dict.get_term_id(token) for token in tokens]
633
- return self.posns.phrase_freqs(term_ids, doc_ids=doc_ids,
634
- phrase_freqs=phrase_freqs,
635
- slop=slop,
636
- min_posn=min_posn,
637
- max_posn=max_posn)
638
- except TermMissingError:
639
- return phrase_freqs
640
-
641
- def phrase_freq_scan(self, tokens: List[str], mask=None, slop=0) -> np.ndarray:
642
- if mask is None:
643
- mask = self.and_query(tokens)
644
-
645
- if np.sum(mask) == 0:
646
- return mask
647
-
648
- # Gather positions
649
- posns = [self.positions(token, mask) for token in tokens]
650
- phrase_freqs = np.zeros(len(self))
651
-
652
- phrase_freqs[mask] = scan_merge_ins(posns, phrase_freqs[mask], slop=slop + 1)
653
- return phrase_freqs
654
-
655
- def phrase_freq_every_diff(self, tokens: List[str], slop=0) -> np.ndarray:
656
- phrase_freqs = -np.ones(len(self))
657
-
658
- mask = self.and_query(tokens)
659
- phrase_freqs[~mask] = 0
660
- if np.sum(mask) == 0:
646
+ phrase_freqs = self.posns.phrase_freqs(term_ids,
647
+ slop=slop,
648
+ min_posn=min_posn,
649
+ max_posn=max_posn)
650
+ if self.term_mat.subset:
651
+ return phrase_freqs[self.term_mat.rows]
661
652
  return phrase_freqs
662
-
663
- term_posns = [self.positions(term, mask) for term in tokens]
664
- for width in [10, 20, 30, 40]:
665
- phrase_freqs[mask] = compute_phrase_freqs(term_posns,
666
- phrase_freqs[mask],
667
- slop=slop + 1,
668
- width=width)
669
-
670
- remaining_mask = phrase_freqs == -1
671
- if np.any(remaining_mask):
672
- remainder_freqs = self.phrase_freq_scan(tokens, mask=remaining_mask, slop=slop)
673
- phrase_freqs[remaining_mask] = remainder_freqs[remaining_mask]
674
- return phrase_freqs
653
+ except TermMissingError:
654
+ if self.term_mat.subset:
655
+ return np.zeros(len(self), dtype=np.float32)
656
+ return self.posns.empty_buffer()
@@ -0,0 +1,7 @@
1
+ from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
2
+ from searcharray.roaringish.search import count_odds # noqa
3
+ from searcharray.roaringish.popcount import popcount_reduce_at, popcount64 # noqa
4
+ from searcharray.roaringish.intersect import intersect, adjacent # noqa
5
+ from searcharray.roaringish.unique import unique # noqa
6
+ from searcharray.roaringish.merge import merge # noqa
7
+ from searcharray.roaringish.spans import span_search # noqa