searcharray 0.0.57__tar.gz → 0.0.59__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {searcharray-0.0.57 → searcharray-0.0.59}/PKG-INFO +7 -6
  2. {searcharray-0.0.57 → searcharray-0.0.59}/README.md +6 -5
  3. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/indexing.py +1 -1
  4. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/phrase/bigram_freqs.py +9 -10
  5. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/phrase/middle_out.py +8 -8
  6. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/phrase/spans.py +71 -21
  7. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/postings.py +14 -1
  8. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/__init__.py +1 -1
  9. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/intersect.c +1724 -1115
  10. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/intersect.pyx +1 -1
  11. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/merge.c +949 -347
  12. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/popcount.c +2285 -987
  13. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/popcount.pyi +18 -0
  14. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/popcount.pyx +43 -0
  15. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/roaringish.py +10 -0
  16. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/roaringish_ops.c +865 -257
  17. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/search.c +970 -373
  18. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/spans.c +8825 -5546
  19. searcharray-0.0.59/searcharray/roaringish/spans.pyx +174 -0
  20. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/unique.c +947 -342
  21. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/solr.py +12 -6
  22. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/PKG-INFO +7 -6
  23. {searcharray-0.0.57 → searcharray-0.0.59}/setup.py +1 -1
  24. {searcharray-0.0.57 → searcharray-0.0.59}/test/test_msmarco.py +33 -1
  25. {searcharray-0.0.57 → searcharray-0.0.59}/test/test_phrase_matches.py +49 -1
  26. searcharray-0.0.59/test/test_slop_matches.py +88 -0
  27. {searcharray-0.0.57 → searcharray-0.0.59}/test/test_tmdb.py +27 -5
  28. {searcharray-0.0.57 → searcharray-0.0.59}/test/test_utils.py +1 -0
  29. searcharray-0.0.57/searcharray/roaringish/spans.pyx +0 -77
  30. searcharray-0.0.57/test/test_slop_matches.py +0 -52
  31. {searcharray-0.0.57 → searcharray-0.0.59}/pyproject.toml +0 -0
  32. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/__init__.py +0 -0
  33. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/phrase/__init__.py +0 -0
  34. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/intersect.pyi +0 -0
  35. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/merge.pyi +0 -0
  36. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/merge.pyx +0 -0
  37. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/roaringish_ops.pyx +0 -0
  38. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/search.pyi +0 -0
  39. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/search.pyx +0 -0
  40. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/snp_ops.pxd +0 -0
  41. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/unique.pyi +0 -0
  42. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/roaringish/unique.pyx +0 -0
  43. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/similarity.py +0 -0
  44. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/term_dict.py +0 -0
  45. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/utils/__init__.py +0 -0
  46. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/utils/bitcount.py +0 -0
  47. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/utils/mat_set.py +0 -0
  48. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/utils/row_viewable_matrix.py +0 -0
  49. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray/utils/sort.py +0 -0
  50. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/SOURCES.txt +0 -0
  51. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/dependency_links.txt +0 -0
  52. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/entry_points.txt +0 -0
  53. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/requires.txt +0 -0
  54. {searcharray-0.0.57 → searcharray-0.0.59}/searcharray.egg-info/top_level.txt +0 -0
  55. {searcharray-0.0.57 → searcharray-0.0.59}/setup.cfg +0 -0
  56. {searcharray-0.0.57 → searcharray-0.0.59}/test/test_bitcount64.py +0 -0
  57. {searcharray-0.0.57 → searcharray-0.0.59}/test/test_extension_array.py +0 -0
  58. {searcharray-0.0.57 → searcharray-0.0.59}/test/test_minmax_posns.py +0 -0
  59. {searcharray-0.0.57 → searcharray-0.0.59}/test/test_search.py +0 -0
  60. {searcharray-0.0.57 → searcharray-0.0.59}/test/test_similarity.py +0 -0
  61. {searcharray-0.0.57 → searcharray-0.0.59}/test/test_snp_ops.py +0 -0
  62. {searcharray-0.0.57 → searcharray-0.0.59}/test/test_solr.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.57
3
+ Version: 0.0.59
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -36,13 +36,14 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
36
36
  Think Lucene, but as a Pandas column.
37
37
 
38
38
  ```python
39
- In[2]: from searcharray import SearchArray
39
+ from searcharray import SearchArray
40
+ import pandas as pd
40
41
 
41
- In[3]: df['title_indexed'] = SearchArray.index(df['title'])
42
- np.sort(df['title_indexed'].array.score('Cat'))
42
+ df['title_indexed'] = SearchArray.index(df['title'])
43
+ np.sort(df['title_indexed'].array.score('Cat'))
43
44
 
44
- Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
45
- 15.84568033, 15.84568033])
45
+ > BM25 scores:
46
+ > array([ 0. , 0. , 0. , ..., 15.84568033, 15.84568033, 15.84568033])
46
47
  ```
47
48
 
48
49
  ## Docs | Guide
@@ -7,13 +7,14 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
7
7
  Think Lucene, but as a Pandas column.
8
8
 
9
9
  ```python
10
- In[2]: from searcharray import SearchArray
10
+ from searcharray import SearchArray
11
+ import pandas as pd
11
12
 
12
- In[3]: df['title_indexed'] = SearchArray.index(df['title'])
13
- np.sort(df['title_indexed'].array.score('Cat'))
13
+ df['title_indexed'] = SearchArray.index(df['title'])
14
+ np.sort(df['title_indexed'].array.score('Cat'))
14
15
 
15
- Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
16
- 15.84568033, 15.84568033])
16
+ > BM25 scores:
17
+ > array([ 0. , 0. , 0. , ..., 15.84568033, 15.84568033, 15.84568033])
17
18
  ```
18
19
 
19
20
  ## Docs | Guide
@@ -121,7 +121,7 @@ def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate
121
121
 
122
122
  # Encode posns to bit array
123
123
  logger.info("Encoding positions to bit array")
124
- posns = PosnBitArrayFromFlatBuilder(terms_w_posns)
124
+ posns = PosnBitArrayFromFlatBuilder(terms_w_posns, max_doc_id=(batch_beg + len(array) - 1))
125
125
  bit_posns = posns.build()
126
126
 
127
127
  if np.any(doc_lens > MAX_POSN):
@@ -5,7 +5,7 @@ from searcharray.roaringish import RoaringishEncoder
5
5
  import logging
6
6
  from enum import Enum
7
7
 
8
- from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at
8
+ from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at, key_sum_over
9
9
 
10
10
 
11
11
  logger = logging.getLogger(__name__)
@@ -82,7 +82,8 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
82
82
  adj_count = encoder.payload_lsb(overlap)
83
83
  adjacents = popcount64(adj_count).view(np.int64)
84
84
 
85
- phrase_freqs[lhs_doc_ids] += _adj_to_phrase_freq(overlap, adjacents)
85
+ adjusted = _adj_to_phrase_freq(overlap, adjacents).astype(np.uint64)
86
+ key_sum_over(lhs_doc_ids, adjusted, phrase_freqs)
86
87
  # Continue with ?? ?? foo foo
87
88
  # term_int without lsbs
88
89
  term_int_msbs = lhs_int & ~encoder.payload_lsb_mask
@@ -90,12 +91,13 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
90
91
  lhs_cont = None
91
92
  # rhs_cont = term_int_msbs | encoder.payload_lsb(rhs_shift)
92
93
  rhs_cont = encoder.payload_lsb(rhs_shift & rhs_int) | term_int_msbs
93
-
94
94
  lhs_cont = term_int_msbs | encoder.payload_lsb(lhs_int & (lhs_int >> _1))
95
+
95
96
  if cont not in [Continuation.RHS, Continuation.BOTH]:
96
97
  rhs_cont = None
97
98
  if cont not in [Continuation.LHS, Continuation.BOTH]:
98
99
  lhs_cont = None
100
+
99
101
  return phrase_freqs, (lhs_cont, rhs_cont)
100
102
 
101
103
 
@@ -177,14 +179,11 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
177
179
  if cont in [Continuation.RHS, Continuation.BOTH]:
178
180
  rhs_next = rhs_adj[matches]
179
181
  assert rhs_next is not None
180
- rhs_next = encoder.header(rhs_adj) | _1
181
- elif cont in [Continuation.LHS, Continuation.BOTH]:
182
- rhs_next = rhs_adj[matches]
183
- assert rhs_next is not None
184
- rhs_next |= _1
185
- rhs_next = encoder.header(rhs_adj) | _1
182
+ rhs_next = encoder.header(rhs_next) | _1
183
+ if cont in [Continuation.LHS, Continuation.BOTH]:
186
184
  lhs_next = lhs_adj[matches]
187
- lhs_next = encoder.header(lhs_adj) | _upper_bit
185
+ assert lhs_next is not None
186
+ lhs_next = encoder.header(lhs_next) | _upper_bit
188
187
  rhs_next = None
189
188
  return phrase_freqs, (lhs_next, rhs_next)
190
189
 
@@ -89,12 +89,10 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
89
89
  phrase_freqs[mask] = 0
90
90
  phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
91
91
  cont=Continuation.RHS)
92
+
92
93
  assert conts[1] is not None
93
94
  lhs = conts[1]
94
95
  mask &= (phrase_freqs > 0)
95
- # print("-- pf", phrase_freqs)
96
- # dec = encoder.decode(lhs)
97
- # print("-- co", dec)
98
96
  phrase_freqs[~mask] = 0
99
97
  return phrase_freqs
100
98
 
@@ -118,12 +116,10 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
118
116
  phrase_freqs[mask] = 0
119
117
  phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
120
118
  cont=Continuation.LHS)
119
+
121
120
  assert conts[0] is not None
122
121
  rhs = conts[0]
123
122
  mask &= (phrase_freqs > 0)
124
- # print("-- pf", phrase_freqs)
125
- # dec = encoder.decode(lhs)
126
- # print("-- co", dec)
127
123
  phrase_freqs[~mask] = 0
128
124
  return phrase_freqs
129
125
 
@@ -158,8 +154,9 @@ class PosnBitArrayFromFlatBuilder:
158
154
 
159
155
  """
160
156
 
161
- def __init__(self, flat_array: np.ndarray):
157
+ def __init__(self, flat_array: np.ndarray, max_doc_id: int) -> None:
162
158
  self.flat_array = flat_array
159
+ self.max_doc_id = max_doc_id
163
160
 
164
161
  def build(self):
165
162
  """Slice the flat array into a 2d array of doc ids and posns."""
@@ -179,7 +176,9 @@ class PosnBitArrayFromFlatBuilder:
179
176
  sliced = encoded[beg_idx:end_idx]
180
177
  encoded_term_posns[term_ids[into_terms]] = sliced
181
178
 
182
- return PosnBitArray(encoded_term_posns, self.flat_array[1].max())
179
+ largest_doc_id_with_term = np.max(self.flat_array[1])
180
+ assert self.max_doc_id >= largest_doc_id_with_term
181
+ return PosnBitArray(encoded_term_posns, self.max_doc_id)
183
182
 
184
183
 
185
184
  class PosnBitArrayBuilder:
@@ -390,6 +389,7 @@ class PosnBitArray:
390
389
  min_posn: Optional[int] = None,
391
390
  max_posn: Optional[int] = None) -> np.ndarray:
392
391
  phrase_freqs = self.empty_buffer()
392
+
393
393
  if len(term_ids) < 2:
394
394
  raise ValueError("Must have at least two terms")
395
395
  if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
@@ -25,7 +25,7 @@ _upper_bit = _1 << (encoder.payload_lsb_bits - _1)
25
25
  # MSB
26
26
  # term A |10| | 12| 13| | 18 * Intersect A+B -> 10,12
27
27
  # term B |10| | 12| | 14 15 17 | * RHS intersect -> 13 -> 14
28
- # term C | 11 14
28
+ # term C | 11 | 14
29
29
  # term D 10 | 15
30
30
  # term E | 11 12
31
31
  # ------
@@ -37,39 +37,87 @@ _upper_bit = _1 << (encoder.payload_lsb_bits - _1)
37
37
  #
38
38
  # ----
39
39
  #
40
+ # Intersecting
41
+ # term A |10| | 12| 13 18
42
+ # term B |10| | 12| 14 15 17
43
+ # Adjacent
44
+ # term A 10 12| 13| 18
45
+ # term B 10 12| | 14 15 17 |
46
+ # Giving 9.5, 10.5 11.5, 12.5, 13.5 , 17.5
47
+ # ------
48
+ # Next pair, intersecting
49
+ # term B 10 12 | 14 | 15 17 * RHS intersect -> 13 -> 14
50
+ # term C 11 | 14 |
51
+ # Adjacent
52
+ # term B 10 12 14 15 17
53
+ # term C |11 14
54
+ # Giving 10.5, 13.5, 14.5
55
+ # w/ above 10.5, 13.5
56
+ # ------
57
+ #
58
+ #
40
59
  # Finally: X X X X <- 4 words selected from each to find max span
41
60
  #
61
+ # Does it work to compute any intersection, and then attach any adjecent that exist
62
+ #
42
63
  #
64
+
65
+ def hdr(arr):
66
+ return encoder.header(arr) >> (_64 - encoder.header_bits)
67
+
68
+
43
69
  def _intersect_all(posns_encoded: List[np.ndarray]):
44
70
  """Intersect all encoded positions at roaringish MSBs."""
45
71
  if len(posns_encoded) < 2:
46
72
  raise ValueError("Need at least two positions to intersect")
73
+ last_lhs_headers = None
74
+ last_rhs_headers = None
75
+ curr = posns_encoded[0]
76
+ for term_idx, posns_next in enumerate(posns_encoded[1:]):
77
+ lhs_int_idx, rhs_int = intersect(curr, posns_next, mask=encoder.header_mask)
78
+ int_headers = encoder.header(curr[lhs_int_idx])
47
79
 
48
- lhs_int = posns_encoded[0]
49
- lhs_to_left = posns_encoded[0]
50
- lhs_to_right = posns_encoded[0]
80
+ # What is adjacent on LHS / RHS of interserction
81
+ # Next to left
82
+ # 0 -> 2
83
+ # <- 1 (curr to right)
84
+ # 1 -> (keep
85
+ # 0 <- 2
86
+ curr_to_right, next_to_left = adjacent(curr, posns_next, mask=encoder.header_mask)
87
+ lhs_headers = merge(int_headers, posns_next[next_to_left])
88
+ rhs_headers = merge(int_headers, curr[curr_to_right])
89
+ next_to_right, curr_to_left = adjacent(posns_next, curr, mask=encoder.header_mask)
90
+ lhs_headers = merge(lhs_headers, curr[curr_to_left])
91
+ rhs_headers = merge(rhs_headers, posns_next[next_to_right])
51
92
 
52
- for term_idx, posns_next in enumerate(posns_encoded[1:]):
53
- _, rhs_int = intersect(lhs_int, posns_next, mask=encoder.header_mask)
54
- _, rhs_ls = adjacent(lhs_to_left, posns_next, mask=encoder.header_mask)
55
- rhs_rs, _ = adjacent(lhs_to_right, posns_next, mask=encoder.header_mask)
93
+ if last_lhs_headers is not None:
94
+ lhs, _ = intersect(last_lhs_headers, lhs_headers, mask=encoder.header_mask)
95
+ rhs, _ = intersect(last_rhs_headers, rhs_headers, mask=encoder.header_mask)
96
+ last_lhs_headers = last_lhs_headers[lhs]
97
+ last_rhs_headers = last_rhs_headers[rhs]
98
+ else:
99
+ last_lhs_headers = lhs_headers
100
+ last_rhs_headers = rhs_headers
56
101
 
57
- # Update LHS to rhs_int + rhs_ls + rhs_rs indices
58
- lhs_int = posns_next[rhs_int]
59
- lhs_to_left = posns_next[rhs_ls]
60
- lhs_to_right = posns_next[rhs_rs]
102
+ # Update by intersecting with current working lhs / rhs headers
61
103
 
104
+ assert last_rhs_headers is not None
105
+ assert last_lhs_headers is not None
106
+ to_rhs = last_rhs_headers + (_1 << (_64 - encoder.header_bits))
107
+ to_lhs = last_lhs_headers - (_1 << (_64 - encoder.header_bits))
108
+ all_headers = merge(to_rhs, to_lhs, drop_duplicates=True)
109
+ all_headers = merge(last_lhs_headers, all_headers, drop_duplicates=True)
110
+ all_headers = merge(last_rhs_headers, all_headers, drop_duplicates=True)
62
111
  # Get active MSBs now
63
112
  # Merge all the rest to grab them
64
- int_header = encoder.header(lhs_int)
65
- to_left_header = encoder.header(lhs_to_left)
66
- to_right_header = encoder.header(lhs_to_right)
67
- merged = merge(int_header, to_left_header, drop_duplicates=True)
68
- merged = merge(merged, to_right_header, drop_duplicates=True)
69
113
 
70
- # Slice only matches at MSBs
114
+ # Slice only matches at header MSBs
115
+ new_posns_encoded = posns_encoded.copy()
71
116
  for i in range(len(posns_encoded)):
72
- posns_encoded[i] = encoder.slice(posns_encoded[i], merged)
117
+ new_posns_encoded[i] = encoder.slice(posns_encoded[i], header=all_headers)
118
+ lengths = np.cumsum([0] + [len(posns) for posns in new_posns_encoded], dtype=np.uint64)
119
+ concatted = np.concatenate(new_posns_encoded, dtype=np.uint64)
120
+ return concatted, lengths
73
121
 
74
122
 
75
123
  # Picking up from intersections:
@@ -122,11 +170,13 @@ def span_search(posns_encoded: List[np.ndarray],
122
170
  slop: int) -> np.ndarray:
123
171
  """Find span matches up to PAYLOAD_LSB bits span distance."""
124
172
  # Find inner span candidates
125
- _intersect_all(posns_encoded)
173
+ posns, lengths = _intersect_all(posns_encoded)
126
174
 
127
175
  # Populate phrase freqs with matches of slop
128
- r_span_search(posns_encoded, phrase_freqs, slop,
176
+ r_span_search(posns, lengths,
177
+ phrase_freqs, slop,
129
178
  encoder.key_mask,
130
179
  encoder.header_mask,
180
+ encoder.key_bits,
131
181
  encoder.payload_lsb_bits)
132
182
  return phrase_freqs
@@ -469,6 +469,15 @@ class SearchArray(ExtensionArray):
469
469
  logger.warning("Unique called on SearchArray. This is not supported.")
470
470
  return self[:]
471
471
 
472
+ def __iter__(self):
473
+ if len(self) > 10000:
474
+ warning_text = """Iterating over SearchArray is very slow and not reccomended.
475
+
476
+ If you're looping a dataframe, slice out the non-SearchArray columns first."""
477
+ logger.warning(warning_text)
478
+ warnings.warn(warning_text)
479
+ return super().__iter__()
480
+
472
481
  def take(self, indices, allow_fill=False, fill_value=None):
473
482
  # Want to take rows of term freqs
474
483
  row_indices = np.arange(len(self.term_mat.rows))
@@ -610,6 +619,7 @@ class SearchArray(ExtensionArray):
610
619
 
611
620
  def score(self, token: Union[str, List[str]],
612
621
  similarity: Similarity = default_bm25,
622
+ slop: int = 0,
613
623
  min_posn: Optional[int] = None,
614
624
  max_posn: Optional[int] = None) -> np.ndarray:
615
625
  """Score each doc using a similarity function.
@@ -629,7 +639,8 @@ class SearchArray(ExtensionArray):
629
639
  tokens_l = [token] if isinstance(token, str) else token
630
640
  all_dfs = np.asarray([self.docfreq(token) for token in tokens_l])
631
641
 
632
- tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn)
642
+ tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn,
643
+ slop=slop)
633
644
  token = self._check_token_arg(token)
634
645
  doc_lens = self.doclengths()
635
646
 
@@ -647,6 +658,8 @@ class SearchArray(ExtensionArray):
647
658
  slop=0,
648
659
  min_posn: Optional[int] = None,
649
660
  max_posn: Optional[int] = None) -> np.ndarray:
661
+ if slop > 0:
662
+ logger.warning("!! Slop is experimental and may be slow, crash, or inaccurate etc")
650
663
  try:
651
664
  # Decide how/if we need to filter doc ids
652
665
  term_ids = [self.term_dict.get_term_id(token) for token in tokens]
@@ -1,6 +1,6 @@
1
1
  from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
2
2
  from searcharray.roaringish.search import count_odds # noqa
3
- from searcharray.roaringish.popcount import popcount_reduce_at, popcount64 # noqa
3
+ from searcharray.roaringish.popcount import popcount_reduce_at, popcount64, key_sum_over # noqa
4
4
  from searcharray.roaringish.intersect import intersect, adjacent # noqa
5
5
  from searcharray.roaringish.unique import unique # noqa
6
6
  from searcharray.roaringish.merge import merge # noqa