searcharray 0.0.39__tar.gz → 0.0.40__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {searcharray-0.0.39 → searcharray-0.0.40}/PKG-INFO +2 -1
  2. {searcharray-0.0.39 → searcharray-0.0.40}/pyproject.toml +1 -1
  3. searcharray-0.0.40/searcharray/phrase/bigram_freqs.py +304 -0
  4. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/phrase/middle_out.py +78 -95
  5. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/postings.py +14 -7
  6. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/utils/roaringish.py +57 -26
  7. searcharray-0.0.40/searcharray/utils/snp_ops.c +35148 -0
  8. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/PKG-INFO +2 -1
  9. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/SOURCES.txt +4 -0
  10. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/requires.txt +1 -0
  11. {searcharray-0.0.39 → searcharray-0.0.40}/setup.py +6 -2
  12. searcharray-0.0.40/test/test_minmax_posns.py +56 -0
  13. {searcharray-0.0.39 → searcharray-0.0.40}/test/test_msmarco.py +11 -1
  14. {searcharray-0.0.39 → searcharray-0.0.40}/test/test_phrase_matches.py +14 -69
  15. searcharray-0.0.40/test/test_snp_ops.py +286 -0
  16. {searcharray-0.0.39 → searcharray-0.0.40}/README.md +0 -0
  17. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/__init__.py +0 -0
  18. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/indexing.py +0 -0
  19. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/phrase/__init__.py +0 -0
  20. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/phrase/posn_diffs.py +0 -0
  21. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/phrase/scan_merge.py +0 -0
  22. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/similarity.py +0 -0
  23. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/solr.py +0 -0
  24. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/term_dict.py +0 -0
  25. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/utils/__init__.py +0 -0
  26. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/utils/bitcount.py +0 -0
  27. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/utils/mat_set.py +0 -0
  28. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/utils/row_viewable_matrix.py +0 -0
  29. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/dependency_links.txt +0 -0
  30. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/entry_points.txt +0 -0
  31. {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/top_level.txt +0 -0
  32. {searcharray-0.0.39 → searcharray-0.0.40}/setup.cfg +0 -0
  33. {searcharray-0.0.39 → searcharray-0.0.40}/test/test_bitcount64.py +0 -0
  34. {searcharray-0.0.39 → searcharray-0.0.40}/test/test_extension_array.py +0 -0
  35. {searcharray-0.0.39 → searcharray-0.0.40}/test/test_search.py +0 -0
  36. {searcharray-0.0.39 → searcharray-0.0.40}/test/test_solr.py +0 -0
  37. {searcharray-0.0.39 → searcharray-0.0.40}/test/test_tmdb.py +0 -0
  38. {searcharray-0.0.39 → searcharray-0.0.40}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.39
3
+ Version: 0.0.40
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -22,6 +22,7 @@ Requires-Python: >=3.7, <4
22
22
  Description-Content-Type: text/markdown
23
23
  Requires-Dist: pandas>=2.0.0
24
24
  Requires-Dist: sortednp
25
+ Requires-Dist: Cython
25
26
  Provides-Extra: dev
26
27
  Requires-Dist: check-manifest; extra == "dev"
27
28
  Provides-Extra: test
@@ -1,5 +1,5 @@
1
1
  [build-system]
2
2
  # These are the assumed default build requirements from pip:
3
3
  # https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support
4
- requires = ["setuptools>=43.0.0", "wheel"]
4
+ requires = ["setuptools>=43.0.0", "wheel", "numpy", "cython"]
5
5
  build-backend = "setuptools.build_meta"
@@ -0,0 +1,304 @@
1
+ """Given two roaringish encoded arrays, count bigram matches between them."""
2
+ import numpy as np
3
+ from typing import Tuple, Optional
4
+ from searcharray.utils.roaringish import RoaringishEncoder
5
+ import logging
6
+ from enum import Enum
7
+
8
+ from searcharray.utils.bitcount import bit_count64
9
+ import sortednp as snp
10
+
11
+
12
+ logger = logging.getLogger(__name__)
13
+
14
+ # When running in pytest
15
+ import sys # noqa
16
+ handler = logging.StreamHandler(sys.stdout)
17
+ handler.setLevel(logging.ERROR)
18
+ formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
19
+ handler.setFormatter(formatter)
20
+ logger.addHandler(handler)
21
+ logger.setLevel(logging.ERROR)
22
+
23
+
24
+ encoder = RoaringishEncoder()
25
+
26
+ # To not constantly type coerce
27
+ _64 = np.uint64(64)
28
+ _2 = np.uint64(2)
29
+ _1 = np.uint64(1)
30
+ _0 = np.uint64(0)
31
+ _neg1 = np.int64(-1)
32
+ _upper_bit = _1 << (encoder.payload_lsb_bits - _1)
33
+
34
+ MAX_POSN = encoder.max_payload
35
+
36
+
37
+ class Continuation(Enum):
38
+ """Enum for continuation of bigram search."""
39
+ LHS = 0
40
+ RHS = 1
41
+ BOTH = 2
42
+
43
+
44
+ def _any_payload_empty(enc: np.ndarray):
45
+ """Check if the payload has any empty of enc."""
46
+ return np.any((enc & encoder.payload_lsb_mask) == 0)
47
+
48
+
49
+ def _adj_to_phrase_freq(overlap: np.ndarray, adjacents: np.ndarray) -> np.ndarray:
50
+ """Adjust phrase freqs for adjacent matches."""
51
+ # ?? 1 1 1 0 1
52
+ # we need to treat the 2nd consecutive 1 as 'not a match'
53
+ # and also update 'term' to not include it
54
+ # Search for foo foo
55
+ # foo foo -> phrase freq 1
56
+ # foo foo foo -> 2 adjs, phrase freqs = 1
57
+ # foo foo foo foo -> 3 adjs, phrase freqs = 2
58
+ # [foo foo] [foo foo] foo -> 4 adjs, phrase freqs = 2
59
+ # [foo foo] [foo foo] mal [foo foo] -> 4 adjs, phrase freqs = 3
60
+ consecutive_ones = encoder.payload_lsb(overlap & (overlap << _1))
61
+ consecutive_ones = bit_count64(consecutive_ones)
62
+ adjacents -= -np.floor_divide(consecutive_ones, -2).astype(np.int64)
63
+ return adjacents
64
+
65
+
66
+ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
67
+ lhs_doc_ids: np.ndarray,
68
+ phrase_freqs: np.ndarray,
69
+ cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
70
+ """Count bigram matches when lhs / rhs are same term.
71
+
72
+ Despite being the same term, its possible lhs_int != rhs_int due to lhs corresponding
73
+ to a past match of identical term."""
74
+ # Shift RHS to left by 1
75
+ # 3 2 1 0 3 2 1 0
76
+ # ?? foo foo foo -> foo foo foo ??
77
+ rhs_shift = rhs_int << _1
78
+ # Now we have unshifted and shifted to count adjacents:
79
+ # foo foo foo ?? & ?? foo foo foo
80
+ # -> ?? foo foo ?? or two times term is adjacent
81
+ # Count these bits...
82
+ overlap = lhs_int & rhs_shift
83
+ adj_count = encoder.payload_lsb(overlap)
84
+ adjacents = bit_count64(adj_count).astype(np.int64)
85
+
86
+ phrase_freqs[lhs_doc_ids] += _adj_to_phrase_freq(overlap, adjacents)
87
+ # Continue with ?? ?? foo foo
88
+ # term_int without lsbs
89
+ term_int_msbs = lhs_int & ~encoder.payload_lsb_mask
90
+ rhs_cont = None
91
+ lhs_cont = None
92
+ # rhs_cont = term_int_msbs | encoder.payload_lsb(rhs_shift)
93
+ rhs_cont = encoder.payload_lsb(rhs_shift & rhs_int) | term_int_msbs
94
+
95
+ lhs_cont = term_int_msbs | encoder.payload_lsb(lhs_int & (lhs_int >> _1))
96
+ if cont not in [Continuation.RHS, Continuation.BOTH]:
97
+ rhs_cont = None
98
+ if cont not in [Continuation.LHS, Continuation.BOTH]:
99
+ lhs_cont = None
100
+ return phrase_freqs, (lhs_cont, rhs_cont)
101
+
102
+
103
+ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
104
+ phrase_freqs: np.ndarray,
105
+ cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
106
+ """Count bigram matches between two encoded arrays, within a 64 bit word with same MSBs.
107
+
108
+ Parameters:
109
+ -----------
110
+ lhs: roaringish encoded posn array of left term
111
+ rhs: roaringish encoded posn array of right term
112
+ phrase_freqs: np.ndarray, preallocated phrase freqs for output
113
+ cont_rhs: bool, whether to continue matching on the rhs or lhs
114
+
115
+ Returns:
116
+ --------
117
+ count: number of matches per doc
118
+ cont_next: the next (lhs or rhs) array to continue matching
119
+
120
+ """
121
+ lhs_int, rhs_int = encoder.intersect(lhs, rhs)
122
+ lhs_doc_ids = encoder.keys(lhs_int)
123
+ if len(lhs_int) != len(rhs_int):
124
+ raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
125
+ if len(lhs_int) == 0 and cont == Continuation.RHS:
126
+ return phrase_freqs, (None, rhs_int)
127
+ if len(lhs_int) == 0 and cont == Continuation.LHS:
128
+ return phrase_freqs, (lhs_int, None)
129
+ if len(lhs_int) == 0 and cont == Continuation.BOTH:
130
+ return phrase_freqs, (lhs_int, rhs_int)
131
+
132
+ # For perf we don't check all values, but if overlapping posns allowed in future, maybe we should
133
+ same_term = (len(lhs_int) == len(rhs_int) and np.all(lhs_int == rhs_int))
134
+ if same_term:
135
+ return _inner_bigram_same_term(lhs_int, rhs_int, lhs_doc_ids, phrase_freqs, cont)
136
+
137
+ overlap_bits = (lhs_int & encoder.payload_lsb_mask) & ((rhs_int & encoder.payload_lsb_mask) >> _1)
138
+ rhs_next = None
139
+ lhs_next = None
140
+ if cont in [Continuation.RHS, Continuation.BOTH]:
141
+ rhs_next = (overlap_bits << _1) & encoder.payload_lsb_mask
142
+ rhs_next |= (rhs_int & (encoder.key_mask | encoder.payload_msb_mask))
143
+ if cont in [Continuation.LHS, Continuation.BOTH]:
144
+ lhs_next = overlap_bits.copy()
145
+ lhs_next |= (lhs_int & (encoder.key_mask | encoder.payload_msb_mask))
146
+
147
+ matches2 = overlap_bits > 0
148
+ if np.any(matches2):
149
+ transitions = np.argwhere(np.diff(lhs_doc_ids[matches2]) != 0).flatten() + 1
150
+ transitions = np.insert(transitions, 0, 0)
151
+ counted_bits = bit_count64(overlap_bits[matches2])
152
+ reduced = np.add.reduceat(counted_bits,
153
+ transitions)
154
+ phrase_freqs[np.unique(lhs_doc_ids[matches2])] += reduced
155
+ return phrase_freqs, (lhs_next, rhs_next)
156
+
157
+
158
+ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
159
+ phrase_freqs: np.ndarray,
160
+ cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
161
+ """Count bigram matches between two encoded arrays where they occur in adjacent 64 bit words.
162
+
163
+ Returns:
164
+ --------
165
+ count: number of matches per doc
166
+ rhs_next: the next rhs array to continue matching
167
+
168
+ """
169
+ lhs_int, rhs_int = encoder.intersect_rshift(lhs, rhs, rshift=_neg1)
170
+ lhs_doc_ids = encoder.keys(lhs_int)
171
+ # lhs lsb set and rhs lsb's most significant bit set
172
+ matches = ((lhs_int & _upper_bit) != 0) & ((rhs_int & _1) != 0)
173
+ unique, counts = np.unique(lhs_doc_ids[matches], return_counts=True)
174
+ phrase_freqs[unique] += counts
175
+ # Set lsb to 0 where no match, lsb to 1 where match
176
+ rhs_next = None if cont == Continuation.LHS else np.asarray([], dtype=np.uint64)
177
+ lhs_next = None if cont == Continuation.RHS else np.asarray([], dtype=np.uint64)
178
+ if np.any(matches):
179
+ if cont in [Continuation.RHS, Continuation.BOTH]:
180
+ rhs_next = rhs_int[matches]
181
+ assert rhs_next is not None
182
+ rhs_next = encoder.header(rhs_int) | _1
183
+ elif cont in [Continuation.LHS, Continuation.BOTH]:
184
+ rhs_next = rhs_int[matches]
185
+ assert rhs_next is not None
186
+ rhs_next |= _1
187
+ rhs_next = encoder.header(rhs_int) | _1
188
+ lhs_next = lhs_int[matches]
189
+ lhs_next = encoder.header(lhs_int) | _upper_bit
190
+ rhs_next = None
191
+ return phrase_freqs, (lhs_next, rhs_next)
192
+
193
+
194
+ def _set_adjbit_at_header(enc1: np.ndarray, just_lsb_enc: np.ndarray,
195
+ cont: Continuation = Continuation.RHS) -> np.ndarray:
196
+ """Merge two encoded arrays on their headers."""
197
+ if len(enc1) == 0:
198
+ return just_lsb_enc
199
+ if len(just_lsb_enc) == 0:
200
+ return enc1
201
+
202
+ enc1_header = encoder.header(enc1)
203
+ just_lsb_enc_header = encoder.header(just_lsb_enc)
204
+
205
+ _, (same_header_enc1, same_header_just_lsb) = snp.intersect(enc1_header, just_lsb_enc_header, indices=True)
206
+ # Set _1 on intersection
207
+ ignore_mask = np.ones(len(just_lsb_enc), dtype=bool)
208
+ ignore_mask[same_header_just_lsb] = False
209
+ if len(same_header_enc1) > 0 and cont == Continuation.RHS:
210
+ enc1[same_header_enc1] |= _1
211
+ just_lsb_enc = just_lsb_enc[ignore_mask]
212
+ if len(same_header_just_lsb) > 0 and cont == Continuation.LHS:
213
+ enc1[same_header_enc1] |= _upper_bit
214
+ just_lsb_enc = just_lsb_enc[ignore_mask]
215
+ return np.sort(np.concatenate([enc1, just_lsb_enc]))
216
+
217
+
218
+ def bigram_freqs(lhs: np.ndarray,
219
+ rhs: np.ndarray,
220
+ phrase_freqs: np.ndarray,
221
+ cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
222
+ """Count bigram matches between two roaringish encoded posn arrays.
223
+ Also return connection on right hand of the bigram
224
+
225
+ Parameters:
226
+ -----------
227
+ lhs: roaringish encoded posn array of left term
228
+ rhs: roaringish encoded posn array of right term
229
+ phrase_freqs: np.ndarray, preallocated phrase freqs for output
230
+ cont_rhs: bool, whether to continue matching on the rhs or lhs
231
+
232
+ Returns:
233
+ --------
234
+ phrase_freqs: number of matches per doc (updated input)
235
+ rhs_next: encoded array of end of each bigram
236
+ (use as lhs on subsequent calls to continue the phrase)
237
+
238
+ """
239
+ # Example zooming into just one 64 bit word (this of course is repeated over
240
+ # the entire array, for all docs and posn msbs)
241
+ #
242
+ # ***************************************************************************
243
+ # "Inner" matches:
244
+ # As roaringish arrays put posns on a 64 bit word, we first search for
245
+ # simpler matches within 64 bit words from same docs with same posn header.
246
+ # We do this by intersecting on the header (doc_id and posn_msb), then checking
247
+ # the following pattern in the posn bits:
248
+ #
249
+ # doc_id posns_msb posns_bits
250
+ # lhs_term: 1234 101 0010001010
251
+ # rhs_term: 1234 101 0101000100
252
+ # ^^ ^^ <- inner matches, terms next to each other
253
+ #
254
+ phrase_freqs, (lhs_next_inner, rhs_next_inner)\
255
+ = _inner_bigram_freqs(lhs, rhs, phrase_freqs, cont)
256
+
257
+ # ***************************************************************************
258
+ # "Adjacent" matches:
259
+ # Of course we also need to check for matches that span two 64 bit words,
260
+ # thats what adjacent matches do. First finding where header meets header + 1,
261
+ # then checking the bit pattern shown below:
262
+ #
263
+ # doc_id posns_msb posns_bits
264
+ # lhs_term: 1234 101 1010000000
265
+ # rhs_term: 1234 111 0001000101
266
+ # ^ ^ <- adjacent matches, terms next to each other
267
+ # but note the posn_msb of lhs is
268
+ # 1 less than the posn_msb of rhs
269
+ # so we detect these differently
270
+ phrase_freqs, (lhs_next_adj, rhs_next_adj)\
271
+ = _adjacent_bigram_freqs(lhs, rhs, phrase_freqs, cont)
272
+
273
+ # ***************************************************************************
274
+ # rhs_next is where the bigram ends on the rhs side, we can use this
275
+ # to continue the phrase matching
276
+ #
277
+ # Combining the examples above:
278
+ # rhs_next_inner: 1234 101 0001000100
279
+ # ^ ^ <- where we had a match (inner)
280
+ #
281
+ # Merged with
282
+ # rhs_next_adj: 1234 111 0000000001
283
+ # ^ <- where we had a match (adjacent)
284
+ #
285
+ # So we merge these... in this case, we have a two-value array
286
+ #
287
+ # rhs_next: 1234,101,0001000100|1234,111,0000000001
288
+ #
289
+ #
290
+ # Now rhs_next can be the LHS of the next call to bigram_freqs
291
+ # to continue the phrase
292
+ #
293
+ rhs_next = None
294
+ lhs_next = None
295
+ if cont in [Continuation.RHS, Continuation.BOTH]:
296
+ assert rhs_next_inner is not None
297
+ assert rhs_next_adj is not None
298
+ rhs_next = _set_adjbit_at_header(rhs_next_inner, rhs_next_adj, Continuation.RHS)
299
+ if cont in [Continuation.LHS, Continuation.BOTH]:
300
+ assert lhs_next_inner is not None
301
+ assert lhs_next_adj is not None
302
+ lhs_next = _set_adjbit_at_header(lhs_next_inner, lhs_next_adj, Continuation.LHS)
303
+
304
+ return phrase_freqs, (lhs_next, rhs_next)
@@ -10,6 +10,7 @@ import sortednp as snp
10
10
  from copy import deepcopy
11
11
  from typing import List, Tuple, Dict, Union, cast, Optional
12
12
  from searcharray.utils.roaringish import RoaringishEncoder, convert_keys, sorted_unique
13
+ from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
13
14
  import numbers
14
15
  import logging
15
16
  from collections import defaultdict
@@ -41,89 +42,6 @@ _neg1 = np.int64(-1)
41
42
  MAX_POSN = encoder.max_payload
42
43
 
43
44
 
44
- def inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
45
- phrase_freqs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
46
- """Count bigram matches between two encoded arrays, within a 64 bit word with same MSBs.
47
-
48
- Returns:
49
- --------
50
- count: number of matches per doc
51
- rhs_next: the next rhs array to continue matching
52
-
53
- """
54
- lhs_int, rhs_int = encoder.intersect(lhs, rhs)
55
- lhs_doc_ids = encoder.keys(lhs_int)
56
- if len(lhs_int) != len(rhs_int):
57
- raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
58
- if len(lhs_int) == 0:
59
- return phrase_freqs, rhs_int
60
- same_term = (len(lhs_int) == len(rhs_int) and lhs_int[0] == rhs_int[0])
61
- if same_term:
62
- # Find adjacent matches
63
- rhs_shift = rhs_int << _1
64
- overlap = lhs_int & rhs_shift
65
- overlap = encoder.payload_lsb(overlap)
66
- adjacents = bit_count64(overlap).astype(np.int64)
67
- adjacents -= -np.floor_divide(adjacents, -2) # ceiling divide
68
- phrase_freqs[lhs_doc_ids] += adjacents
69
- return phrase_freqs, rhs_int
70
-
71
- overlap_bits = (lhs_int & encoder.payload_lsb_mask) & ((rhs_int & encoder.payload_lsb_mask) >> _1)
72
- rhs_next2 = (overlap_bits << _1) & encoder.payload_lsb_mask
73
- rhs_next2 |= (rhs_int & (encoder.key_mask | encoder.payload_msb_mask))
74
- phrase_freqs2 = phrase_freqs.copy()
75
- matches2 = overlap_bits > 0
76
- if np.any(matches2):
77
- transitions = np.argwhere(np.diff(lhs_doc_ids[matches2]) != 0).flatten() + 1
78
- transitions = np.insert(transitions, 0, 0)
79
- counted_bits = bit_count64(overlap_bits[matches2])
80
- reduced = np.add.reduceat(counted_bits,
81
- transitions)
82
- phrase_freqs2[np.unique(lhs_doc_ids[matches2])] += reduced
83
- return phrase_freqs2, rhs_next2
84
-
85
-
86
- def adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
87
- phrase_freqs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
88
- """Count bigram matches between two encoded arrays where they occur in adjacent 64 bit words.
89
-
90
- Returns:
91
- --------
92
- count: number of matches per doc
93
- rhs_next: the next rhs array to continue matching
94
-
95
- """
96
- lhs_int, rhs_int = encoder.intersect_rshift(lhs, rhs, rshift=_neg1)
97
- lhs_doc_ids = encoder.keys(lhs_int)
98
- # lhs lsb set and rhs lsb's most significant bit set
99
- upper_bit = _1 << (encoder.payload_lsb_bits - _1)
100
- matches = ((lhs_int & upper_bit) != 0) & ((rhs_int & _1) != 0)
101
- unique, counts = np.unique(lhs_doc_ids[matches], return_counts=True)
102
- phrase_freqs[unique] += counts
103
- rhs_next = rhs_int
104
- rhs_next[~matches] |= ~encoder.payload_lsb_mask
105
- rhs_next[matches] |= (encoder.payload_lsb_mask & _1)
106
- return phrase_freqs, rhs_next
107
-
108
-
109
- def bigram_freqs(lhs: np.ndarray, rhs: np.ndarray, phrase_freqs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
110
- """Count bigram matches between two encoded arrays.
111
-
112
- Returns:
113
- --------
114
- count: number of matches per doc
115
- rhs_next: the next rhs array to continue matching
116
-
117
- """
118
- # Combine lhs and rhs matches from two strategies
119
- phrase_freqs, rhs_next_inner = inner_bigram_freqs(lhs, rhs, phrase_freqs)
120
- phrase_freqs, rhs_next_adj = adjacent_bigram_freqs(lhs, rhs, phrase_freqs)
121
- rhs_next = np.sort(np.concatenate([rhs_next_inner, rhs_next_adj]))
122
-
123
- # Combine
124
- return phrase_freqs, rhs_next
125
-
126
-
127
45
  def trim_phrase_search(encoded_posns: List[np.ndarray],
128
46
  phrase_freqs: np.ndarray) -> List[np.ndarray]:
129
47
  """Trim long phrases by searching the rarest terms first."""
@@ -147,33 +65,86 @@ def trim_phrase_search(encoded_posns: List[np.ndarray],
147
65
  for enc_posn_idx in range(len(encoded_posns)):
148
66
  if enc_posn_idx == shortest_idx:
149
67
  continue
150
- if len(encoded_posns[enc_posn_idx]) > (10 * min_len):
68
+ if len(encoded_posns[enc_posn_idx]) > (20 * min_len):
151
69
  encoded_posns[enc_posn_idx] = encoder.slice(encoded_posns[enc_posn_idx],
152
70
  shortest_keys)
153
71
 
154
72
  return encoded_posns
155
73
 
156
74
 
157
- def compute_phrase_freqs(encoded_posns: List[np.ndarray],
158
- phrase_freqs: np.ndarray) -> np.ndarray:
75
+ def _compute_phrase_freqs_rhs(encoded_posns: List[np.ndarray],
76
+ phrase_freqs: np.ndarray,
77
+ trim: bool = True) -> np.ndarray:
78
+ """Compute phrase freqs from a set of encoded positions."""
159
79
  if len(encoded_posns) < 2:
160
80
  raise ValueError("phrase must have at least two terms")
161
81
 
162
82
  # Trim long phrases by searching the rarest terms first
163
- if len(encoded_posns) > 3:
83
+ if trim and len(encoded_posns) > 3:
164
84
  encoded_posns = trim_phrase_search(encoded_posns, phrase_freqs)
165
-
166
85
  mask = np.ones(len(phrase_freqs), dtype=bool)
86
+
167
87
  lhs = encoded_posns[0]
168
88
  for rhs in encoded_posns[1:]:
169
89
  # Only count the count of the last bigram (ignoring the ones where priors did not match)
170
90
  phrase_freqs[mask] = 0
171
- phrase_freqs, lhs = bigram_freqs(lhs, rhs, phrase_freqs)
91
+ phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs, cont=Continuation.RHS)
92
+ assert conts[1] is not None
93
+ lhs = conts[1]
172
94
  mask &= (phrase_freqs > 0)
95
+ # print("-- pf", phrase_freqs)
96
+ # dec = encoder.decode(lhs)
97
+ # print("-- co", dec)
173
98
  phrase_freqs[~mask] = 0
174
99
  return phrase_freqs
175
100
 
176
101
 
102
+ def _compute_phrase_freqs_lhs(encoded_posns: List[np.ndarray],
103
+ phrase_freqs: np.ndarray,
104
+ trim: bool = True) -> np.ndarray:
105
+ """Compute phrase freqs from a set of encoded positions."""
106
+ if len(encoded_posns) < 2:
107
+ raise ValueError("phrase must have at least two terms")
108
+
109
+ # Trim long phrases by searching the rarest terms first
110
+ if trim and len(encoded_posns) > 3:
111
+ encoded_posns = trim_phrase_search(encoded_posns, phrase_freqs)
112
+ mask = np.ones(len(phrase_freqs), dtype=bool)
113
+
114
+ rhs = encoded_posns[-1]
115
+ for lhs in encoded_posns[-2::-1]:
116
+ # Only count the count of the last bigram (ignoring the ones where priors did not match)
117
+ phrase_freqs[mask] = 0
118
+ phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs, cont=Continuation.LHS)
119
+ assert conts[0] is not None
120
+ rhs = conts[0]
121
+ mask &= (phrase_freqs > 0)
122
+ # print("-- pf", phrase_freqs)
123
+ # dec = encoder.decode(lhs)
124
+ # print("-- co", dec)
125
+ phrase_freqs[~mask] = 0
126
+ return phrase_freqs
127
+
128
+
129
+ def compute_phrase_freqs(encoded_posns: List[np.ndarray],
130
+ phrase_freqs: np.ndarray,
131
+ trim: bool = False) -> np.ndarray:
132
+ """Compute phrase freqs from a set of encoded positions."""
133
+ shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
134
+ # Slice LHS / RHS at shortest_len_index
135
+ if shortest_len_index <= 1:
136
+ return _compute_phrase_freqs_lhs(encoded_posns, phrase_freqs, trim=trim)
137
+ elif shortest_len_index >= len(encoded_posns) - 2:
138
+ return _compute_phrase_freqs_rhs(encoded_posns, phrase_freqs, trim=trim)
139
+ else:
140
+ # We optimize this case by going middle-out
141
+ # We can take the min of both directions phrase freqs
142
+ lhs = _compute_phrase_freqs_lhs(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim)
143
+ rhs = _compute_phrase_freqs_rhs(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim)
144
+ phrase_freqs = np.minimum(lhs, rhs)
145
+ return phrase_freqs
146
+
147
+
177
148
  class PosnBitArrayFromFlatBuilder:
178
149
  """ Build from sorted array shape num terms x 3.
179
150
 
@@ -365,14 +336,18 @@ class PosnBitArray:
365
336
  return term_posns
366
337
 
367
338
  def phrase_freqs(self, term_ids: List[int], phrase_freqs: np.ndarray,
368
- doc_ids: np.ndarray) -> np.ndarray:
339
+ doc_ids: np.ndarray,
340
+ min_posn: Optional[int] = None,
341
+ max_posn: Optional[int] = None) -> np.ndarray:
369
342
  if len(term_ids) < 2:
370
343
  raise ValueError("Must have at least two terms")
371
- if phrase_freqs.shape[0] == self.max_doc_id + 1:
344
+ if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
372
345
  enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
373
346
  else:
374
347
  enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
375
- keys=doc_ids.view(np.uint64)) for term_id in term_ids]
348
+ keys=doc_ids.view(np.uint64),
349
+ min_payload=min_posn,
350
+ max_payload=max_posn) for term_id in term_ids]
376
351
  return compute_phrase_freqs(enc_term_posns, phrase_freqs)
377
352
 
378
353
  def positions(self, term_id: int, doc_ids) -> Union[List[np.ndarray], np.ndarray]:
@@ -408,15 +383,23 @@ class PosnBitArray:
408
383
  decs = [dec[1] for dec in decoded]
409
384
  return decs
410
385
 
411
- def termfreqs(self, term_id: int, doc_ids: Optional[np.ndarray] = None) -> Tuple[np.ndarray, np.ndarray]:
386
+ def termfreqs(self, term_id: int,
387
+ doc_ids: Optional[np.ndarray] = None,
388
+ min_posn: Optional[int] = None,
389
+ max_posn: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray]:
412
390
  """Count term freqs using unique positions."""
413
- if doc_ids is None:
391
+ if doc_ids is None and max_posn is None and min_posn is None:
414
392
  return self._termfreqs_with_cache(term_id)
393
+ if doc_ids is None:
394
+ doc_ids = sorted_unique(encoder.keys(self.encoded_term_posns[term_id]))
415
395
 
416
396
  encoded = self.encoded_term_posns[term_id]
417
397
  term_posns = encoded
418
398
  term_posns = encoder.slice(encoded,
419
- keys=doc_ids.astype(np.uint64))
399
+ keys=doc_ids.astype(np.uint64),
400
+ min_payload=min_posn,
401
+ max_payload=max_posn)
402
+
420
403
  return self._computed_term_freqs(term_posns)
421
404
 
422
405
  def _computed_term_freqs(self, term_posns) -> Tuple[np.ndarray, np.ndarray]:
@@ -519,10 +519,12 @@ class SearchArray(ExtensionArray):
519
519
  # ***********************************************************
520
520
  # Search functionality
521
521
  # ***********************************************************
522
- def termfreqs(self, token: Union[List[str], str]) -> np.ndarray:
522
+ def termfreqs(self, token: Union[List[str], str],
523
+ min_posn: Optional[int] = None,
524
+ max_posn: Optional[int] = None) -> np.ndarray:
523
525
  token = self._check_token_arg(token)
524
526
  if isinstance(token, list):
525
- return self.phrase_freq(token)
527
+ return self.phrase_freq(token, min_posn=min_posn, max_posn=max_posn)
526
528
 
527
529
  try:
528
530
  term_id = self.term_dict.get_term_id(token)
@@ -531,13 +533,13 @@ class SearchArray(ExtensionArray):
531
533
  if self.term_mat.subset:
532
534
  slice_of_rows = self.term_mat.rows
533
535
  doc_ids, termfreqs = self.posns.termfreqs(term_id,
534
- doc_ids=slice_of_rows)
536
+ doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
535
537
  mask = np.isin(self.term_mat.rows, doc_ids)
536
538
  matches[mask] = termfreqs
537
539
  return matches
538
540
  else:
539
541
  doc_ids, termfreqs = self.posns.termfreqs(term_id,
540
- doc_ids=slice_of_rows)
542
+ doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
541
543
  matches[doc_ids] = termfreqs
542
544
  return matches
543
545
  except TermMissingError:
@@ -609,14 +611,19 @@ class SearchArray(ExtensionArray):
609
611
  mask = np.sum(masks, axis=0) >= min_should_match
610
612
  return mask
611
613
 
612
- def phrase_freq(self, tokens: List[str], slop=1) -> np.ndarray:
613
- if slop == 1 and len(tokens) == len(set(tokens)):
614
+ def phrase_freq(self, tokens: List[str],
615
+ slop=1,
616
+ min_posn: Optional[int] = None,
617
+ max_posn: Optional[int] = None) -> np.ndarray:
618
+ if slop == 1:
614
619
  phrase_freqs = np.zeros(len(self))
615
620
  try:
616
621
  doc_ids = self.term_mat.rows
617
622
  term_ids = [self.term_dict.get_term_id(token) for token in tokens]
618
623
  return self.posns.phrase_freqs(term_ids, doc_ids=doc_ids,
619
- phrase_freqs=phrase_freqs)
624
+ phrase_freqs=phrase_freqs,
625
+ min_posn=min_posn,
626
+ max_posn=max_posn)
620
627
  except TermMissingError:
621
628
  return phrase_freqs
622
629
  else: