searcharray 0.0.39__tar.gz → 0.0.40__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.39 → searcharray-0.0.40}/PKG-INFO +2 -1
- {searcharray-0.0.39 → searcharray-0.0.40}/pyproject.toml +1 -1
- searcharray-0.0.40/searcharray/phrase/bigram_freqs.py +304 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/phrase/middle_out.py +78 -95
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/postings.py +14 -7
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/utils/roaringish.py +57 -26
- searcharray-0.0.40/searcharray/utils/snp_ops.c +35148 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/PKG-INFO +2 -1
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/SOURCES.txt +4 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/requires.txt +1 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/setup.py +6 -2
- searcharray-0.0.40/test/test_minmax_posns.py +56 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/test/test_msmarco.py +11 -1
- {searcharray-0.0.39 → searcharray-0.0.40}/test/test_phrase_matches.py +14 -69
- searcharray-0.0.40/test/test_snp_ops.py +286 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/README.md +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/__init__.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/indexing.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/similarity.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/solr.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/setup.cfg +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/test/test_extension_array.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/test/test_search.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/test/test_solr.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/test/test_tmdb.py +0 -0
- {searcharray-0.0.39 → searcharray-0.0.40}/test/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: searcharray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.40
|
|
4
4
|
Summary: Searchable pandas text extension arrays for prototyping search
|
|
5
5
|
Home-page: https://github.com/softwaredoug/searcharray
|
|
6
6
|
Author: Doug Turnbull
|
|
@@ -22,6 +22,7 @@ Requires-Python: >=3.7, <4
|
|
|
22
22
|
Description-Content-Type: text/markdown
|
|
23
23
|
Requires-Dist: pandas>=2.0.0
|
|
24
24
|
Requires-Dist: sortednp
|
|
25
|
+
Requires-Dist: Cython
|
|
25
26
|
Provides-Extra: dev
|
|
26
27
|
Requires-Dist: check-manifest; extra == "dev"
|
|
27
28
|
Provides-Extra: test
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
[build-system]
|
|
2
2
|
# These are the assumed default build requirements from pip:
|
|
3
3
|
# https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support
|
|
4
|
-
requires = ["setuptools>=43.0.0", "wheel"]
|
|
4
|
+
requires = ["setuptools>=43.0.0", "wheel", "numpy", "cython"]
|
|
5
5
|
build-backend = "setuptools.build_meta"
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
"""Given two roaringish encoded arrays, count bigram matches between them."""
|
|
2
|
+
import numpy as np
|
|
3
|
+
from typing import Tuple, Optional
|
|
4
|
+
from searcharray.utils.roaringish import RoaringishEncoder
|
|
5
|
+
import logging
|
|
6
|
+
from enum import Enum
|
|
7
|
+
|
|
8
|
+
from searcharray.utils.bitcount import bit_count64
|
|
9
|
+
import sortednp as snp
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
# When running in pytest
|
|
15
|
+
import sys # noqa
|
|
16
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
17
|
+
handler.setLevel(logging.ERROR)
|
|
18
|
+
formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
|
|
19
|
+
handler.setFormatter(formatter)
|
|
20
|
+
logger.addHandler(handler)
|
|
21
|
+
logger.setLevel(logging.ERROR)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
encoder = RoaringishEncoder()
|
|
25
|
+
|
|
26
|
+
# To not constantly type coerce
|
|
27
|
+
_64 = np.uint64(64)
|
|
28
|
+
_2 = np.uint64(2)
|
|
29
|
+
_1 = np.uint64(1)
|
|
30
|
+
_0 = np.uint64(0)
|
|
31
|
+
_neg1 = np.int64(-1)
|
|
32
|
+
_upper_bit = _1 << (encoder.payload_lsb_bits - _1)
|
|
33
|
+
|
|
34
|
+
MAX_POSN = encoder.max_payload
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Continuation(Enum):
|
|
38
|
+
"""Enum for continuation of bigram search."""
|
|
39
|
+
LHS = 0
|
|
40
|
+
RHS = 1
|
|
41
|
+
BOTH = 2
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _any_payload_empty(enc: np.ndarray):
|
|
45
|
+
"""Check if the payload has any empty of enc."""
|
|
46
|
+
return np.any((enc & encoder.payload_lsb_mask) == 0)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _adj_to_phrase_freq(overlap: np.ndarray, adjacents: np.ndarray) -> np.ndarray:
|
|
50
|
+
"""Adjust phrase freqs for adjacent matches."""
|
|
51
|
+
# ?? 1 1 1 0 1
|
|
52
|
+
# we need to treat the 2nd consecutive 1 as 'not a match'
|
|
53
|
+
# and also update 'term' to not include it
|
|
54
|
+
# Search for foo foo
|
|
55
|
+
# foo foo -> phrase freq 1
|
|
56
|
+
# foo foo foo -> 2 adjs, phrase freqs = 1
|
|
57
|
+
# foo foo foo foo -> 3 adjs, phrase freqs = 2
|
|
58
|
+
# [foo foo] [foo foo] foo -> 4 adjs, phrase freqs = 2
|
|
59
|
+
# [foo foo] [foo foo] mal [foo foo] -> 4 adjs, phrase freqs = 3
|
|
60
|
+
consecutive_ones = encoder.payload_lsb(overlap & (overlap << _1))
|
|
61
|
+
consecutive_ones = bit_count64(consecutive_ones)
|
|
62
|
+
adjacents -= -np.floor_divide(consecutive_ones, -2).astype(np.int64)
|
|
63
|
+
return adjacents
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
|
|
67
|
+
lhs_doc_ids: np.ndarray,
|
|
68
|
+
phrase_freqs: np.ndarray,
|
|
69
|
+
cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
70
|
+
"""Count bigram matches when lhs / rhs are same term.
|
|
71
|
+
|
|
72
|
+
Despite being the same term, its possible lhs_int != rhs_int due to lhs corresponding
|
|
73
|
+
to a past match of identical term."""
|
|
74
|
+
# Shift RHS to left by 1
|
|
75
|
+
# 3 2 1 0 3 2 1 0
|
|
76
|
+
# ?? foo foo foo -> foo foo foo ??
|
|
77
|
+
rhs_shift = rhs_int << _1
|
|
78
|
+
# Now we have unshifted and shifted to count adjacents:
|
|
79
|
+
# foo foo foo ?? & ?? foo foo foo
|
|
80
|
+
# -> ?? foo foo ?? or two times term is adjacent
|
|
81
|
+
# Count these bits...
|
|
82
|
+
overlap = lhs_int & rhs_shift
|
|
83
|
+
adj_count = encoder.payload_lsb(overlap)
|
|
84
|
+
adjacents = bit_count64(adj_count).astype(np.int64)
|
|
85
|
+
|
|
86
|
+
phrase_freqs[lhs_doc_ids] += _adj_to_phrase_freq(overlap, adjacents)
|
|
87
|
+
# Continue with ?? ?? foo foo
|
|
88
|
+
# term_int without lsbs
|
|
89
|
+
term_int_msbs = lhs_int & ~encoder.payload_lsb_mask
|
|
90
|
+
rhs_cont = None
|
|
91
|
+
lhs_cont = None
|
|
92
|
+
# rhs_cont = term_int_msbs | encoder.payload_lsb(rhs_shift)
|
|
93
|
+
rhs_cont = encoder.payload_lsb(rhs_shift & rhs_int) | term_int_msbs
|
|
94
|
+
|
|
95
|
+
lhs_cont = term_int_msbs | encoder.payload_lsb(lhs_int & (lhs_int >> _1))
|
|
96
|
+
if cont not in [Continuation.RHS, Continuation.BOTH]:
|
|
97
|
+
rhs_cont = None
|
|
98
|
+
if cont not in [Continuation.LHS, Continuation.BOTH]:
|
|
99
|
+
lhs_cont = None
|
|
100
|
+
return phrase_freqs, (lhs_cont, rhs_cont)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
104
|
+
phrase_freqs: np.ndarray,
|
|
105
|
+
cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
106
|
+
"""Count bigram matches between two encoded arrays, within a 64 bit word with same MSBs.
|
|
107
|
+
|
|
108
|
+
Parameters:
|
|
109
|
+
-----------
|
|
110
|
+
lhs: roaringish encoded posn array of left term
|
|
111
|
+
rhs: roaringish encoded posn array of right term
|
|
112
|
+
phrase_freqs: np.ndarray, preallocated phrase freqs for output
|
|
113
|
+
cont_rhs: bool, whether to continue matching on the rhs or lhs
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
--------
|
|
117
|
+
count: number of matches per doc
|
|
118
|
+
cont_next: the next (lhs or rhs) array to continue matching
|
|
119
|
+
|
|
120
|
+
"""
|
|
121
|
+
lhs_int, rhs_int = encoder.intersect(lhs, rhs)
|
|
122
|
+
lhs_doc_ids = encoder.keys(lhs_int)
|
|
123
|
+
if len(lhs_int) != len(rhs_int):
|
|
124
|
+
raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
|
|
125
|
+
if len(lhs_int) == 0 and cont == Continuation.RHS:
|
|
126
|
+
return phrase_freqs, (None, rhs_int)
|
|
127
|
+
if len(lhs_int) == 0 and cont == Continuation.LHS:
|
|
128
|
+
return phrase_freqs, (lhs_int, None)
|
|
129
|
+
if len(lhs_int) == 0 and cont == Continuation.BOTH:
|
|
130
|
+
return phrase_freqs, (lhs_int, rhs_int)
|
|
131
|
+
|
|
132
|
+
# For perf we don't check all values, but if overlapping posns allowed in future, maybe we should
|
|
133
|
+
same_term = (len(lhs_int) == len(rhs_int) and np.all(lhs_int == rhs_int))
|
|
134
|
+
if same_term:
|
|
135
|
+
return _inner_bigram_same_term(lhs_int, rhs_int, lhs_doc_ids, phrase_freqs, cont)
|
|
136
|
+
|
|
137
|
+
overlap_bits = (lhs_int & encoder.payload_lsb_mask) & ((rhs_int & encoder.payload_lsb_mask) >> _1)
|
|
138
|
+
rhs_next = None
|
|
139
|
+
lhs_next = None
|
|
140
|
+
if cont in [Continuation.RHS, Continuation.BOTH]:
|
|
141
|
+
rhs_next = (overlap_bits << _1) & encoder.payload_lsb_mask
|
|
142
|
+
rhs_next |= (rhs_int & (encoder.key_mask | encoder.payload_msb_mask))
|
|
143
|
+
if cont in [Continuation.LHS, Continuation.BOTH]:
|
|
144
|
+
lhs_next = overlap_bits.copy()
|
|
145
|
+
lhs_next |= (lhs_int & (encoder.key_mask | encoder.payload_msb_mask))
|
|
146
|
+
|
|
147
|
+
matches2 = overlap_bits > 0
|
|
148
|
+
if np.any(matches2):
|
|
149
|
+
transitions = np.argwhere(np.diff(lhs_doc_ids[matches2]) != 0).flatten() + 1
|
|
150
|
+
transitions = np.insert(transitions, 0, 0)
|
|
151
|
+
counted_bits = bit_count64(overlap_bits[matches2])
|
|
152
|
+
reduced = np.add.reduceat(counted_bits,
|
|
153
|
+
transitions)
|
|
154
|
+
phrase_freqs[np.unique(lhs_doc_ids[matches2])] += reduced
|
|
155
|
+
return phrase_freqs, (lhs_next, rhs_next)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
159
|
+
phrase_freqs: np.ndarray,
|
|
160
|
+
cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
161
|
+
"""Count bigram matches between two encoded arrays where they occur in adjacent 64 bit words.
|
|
162
|
+
|
|
163
|
+
Returns:
|
|
164
|
+
--------
|
|
165
|
+
count: number of matches per doc
|
|
166
|
+
rhs_next: the next rhs array to continue matching
|
|
167
|
+
|
|
168
|
+
"""
|
|
169
|
+
lhs_int, rhs_int = encoder.intersect_rshift(lhs, rhs, rshift=_neg1)
|
|
170
|
+
lhs_doc_ids = encoder.keys(lhs_int)
|
|
171
|
+
# lhs lsb set and rhs lsb's most significant bit set
|
|
172
|
+
matches = ((lhs_int & _upper_bit) != 0) & ((rhs_int & _1) != 0)
|
|
173
|
+
unique, counts = np.unique(lhs_doc_ids[matches], return_counts=True)
|
|
174
|
+
phrase_freqs[unique] += counts
|
|
175
|
+
# Set lsb to 0 where no match, lsb to 1 where match
|
|
176
|
+
rhs_next = None if cont == Continuation.LHS else np.asarray([], dtype=np.uint64)
|
|
177
|
+
lhs_next = None if cont == Continuation.RHS else np.asarray([], dtype=np.uint64)
|
|
178
|
+
if np.any(matches):
|
|
179
|
+
if cont in [Continuation.RHS, Continuation.BOTH]:
|
|
180
|
+
rhs_next = rhs_int[matches]
|
|
181
|
+
assert rhs_next is not None
|
|
182
|
+
rhs_next = encoder.header(rhs_int) | _1
|
|
183
|
+
elif cont in [Continuation.LHS, Continuation.BOTH]:
|
|
184
|
+
rhs_next = rhs_int[matches]
|
|
185
|
+
assert rhs_next is not None
|
|
186
|
+
rhs_next |= _1
|
|
187
|
+
rhs_next = encoder.header(rhs_int) | _1
|
|
188
|
+
lhs_next = lhs_int[matches]
|
|
189
|
+
lhs_next = encoder.header(lhs_int) | _upper_bit
|
|
190
|
+
rhs_next = None
|
|
191
|
+
return phrase_freqs, (lhs_next, rhs_next)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _set_adjbit_at_header(enc1: np.ndarray, just_lsb_enc: np.ndarray,
|
|
195
|
+
cont: Continuation = Continuation.RHS) -> np.ndarray:
|
|
196
|
+
"""Merge two encoded arrays on their headers."""
|
|
197
|
+
if len(enc1) == 0:
|
|
198
|
+
return just_lsb_enc
|
|
199
|
+
if len(just_lsb_enc) == 0:
|
|
200
|
+
return enc1
|
|
201
|
+
|
|
202
|
+
enc1_header = encoder.header(enc1)
|
|
203
|
+
just_lsb_enc_header = encoder.header(just_lsb_enc)
|
|
204
|
+
|
|
205
|
+
_, (same_header_enc1, same_header_just_lsb) = snp.intersect(enc1_header, just_lsb_enc_header, indices=True)
|
|
206
|
+
# Set _1 on intersection
|
|
207
|
+
ignore_mask = np.ones(len(just_lsb_enc), dtype=bool)
|
|
208
|
+
ignore_mask[same_header_just_lsb] = False
|
|
209
|
+
if len(same_header_enc1) > 0 and cont == Continuation.RHS:
|
|
210
|
+
enc1[same_header_enc1] |= _1
|
|
211
|
+
just_lsb_enc = just_lsb_enc[ignore_mask]
|
|
212
|
+
if len(same_header_just_lsb) > 0 and cont == Continuation.LHS:
|
|
213
|
+
enc1[same_header_enc1] |= _upper_bit
|
|
214
|
+
just_lsb_enc = just_lsb_enc[ignore_mask]
|
|
215
|
+
return np.sort(np.concatenate([enc1, just_lsb_enc]))
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def bigram_freqs(lhs: np.ndarray,
|
|
219
|
+
rhs: np.ndarray,
|
|
220
|
+
phrase_freqs: np.ndarray,
|
|
221
|
+
cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
222
|
+
"""Count bigram matches between two roaringish encoded posn arrays.
|
|
223
|
+
Also return connection on right hand of the bigram
|
|
224
|
+
|
|
225
|
+
Parameters:
|
|
226
|
+
-----------
|
|
227
|
+
lhs: roaringish encoded posn array of left term
|
|
228
|
+
rhs: roaringish encoded posn array of right term
|
|
229
|
+
phrase_freqs: np.ndarray, preallocated phrase freqs for output
|
|
230
|
+
cont_rhs: bool, whether to continue matching on the rhs or lhs
|
|
231
|
+
|
|
232
|
+
Returns:
|
|
233
|
+
--------
|
|
234
|
+
phrase_freqs: number of matches per doc (updated input)
|
|
235
|
+
rhs_next: encoded array of end of each bigram
|
|
236
|
+
(use as lhs on subsequent calls to continue the phrase)
|
|
237
|
+
|
|
238
|
+
"""
|
|
239
|
+
# Example zooming into just one 64 bit word (this of course is repeated over
|
|
240
|
+
# the entire array, for all docs and posn msbs)
|
|
241
|
+
#
|
|
242
|
+
# ***************************************************************************
|
|
243
|
+
# "Inner" matches:
|
|
244
|
+
# As roaringish arrays put posns on a 64 bit word, we first search for
|
|
245
|
+
# simpler matches within 64 bit words from same docs with same posn header.
|
|
246
|
+
# We do this by intersecting on the header (doc_id and posn_msb), then checking
|
|
247
|
+
# the following pattern in the posn bits:
|
|
248
|
+
#
|
|
249
|
+
# doc_id posns_msb posns_bits
|
|
250
|
+
# lhs_term: 1234 101 0010001010
|
|
251
|
+
# rhs_term: 1234 101 0101000100
|
|
252
|
+
# ^^ ^^ <- inner matches, terms next to each other
|
|
253
|
+
#
|
|
254
|
+
phrase_freqs, (lhs_next_inner, rhs_next_inner)\
|
|
255
|
+
= _inner_bigram_freqs(lhs, rhs, phrase_freqs, cont)
|
|
256
|
+
|
|
257
|
+
# ***************************************************************************
|
|
258
|
+
# "Adjacent" matches:
|
|
259
|
+
# Of course we also need to check for matches that span two 64 bit words,
|
|
260
|
+
# thats what adjacent matches do. First finding where header meets header + 1,
|
|
261
|
+
# then checking the bit pattern shown below:
|
|
262
|
+
#
|
|
263
|
+
# doc_id posns_msb posns_bits
|
|
264
|
+
# lhs_term: 1234 101 1010000000
|
|
265
|
+
# rhs_term: 1234 111 0001000101
|
|
266
|
+
# ^ ^ <- adjacent matches, terms next to each other
|
|
267
|
+
# but note the posn_msb of lhs is
|
|
268
|
+
# 1 less than the posn_msb of rhs
|
|
269
|
+
# so we detect these differently
|
|
270
|
+
phrase_freqs, (lhs_next_adj, rhs_next_adj)\
|
|
271
|
+
= _adjacent_bigram_freqs(lhs, rhs, phrase_freqs, cont)
|
|
272
|
+
|
|
273
|
+
# ***************************************************************************
|
|
274
|
+
# rhs_next is where the bigram ends on the rhs side, we can use this
|
|
275
|
+
# to continue the phrase matching
|
|
276
|
+
#
|
|
277
|
+
# Combining the examples above:
|
|
278
|
+
# rhs_next_inner: 1234 101 0001000100
|
|
279
|
+
# ^ ^ <- where we had a match (inner)
|
|
280
|
+
#
|
|
281
|
+
# Merged with
|
|
282
|
+
# rhs_next_adj: 1234 111 0000000001
|
|
283
|
+
# ^ <- where we had a match (adjacent)
|
|
284
|
+
#
|
|
285
|
+
# So we merge these... in this case, we have a two-value array
|
|
286
|
+
#
|
|
287
|
+
# rhs_next: 1234,101,0001000100|1234,111,0000000001
|
|
288
|
+
#
|
|
289
|
+
#
|
|
290
|
+
# Now rhs_next can be the LHS of the next call to bigram_freqs
|
|
291
|
+
# to continue the phrase
|
|
292
|
+
#
|
|
293
|
+
rhs_next = None
|
|
294
|
+
lhs_next = None
|
|
295
|
+
if cont in [Continuation.RHS, Continuation.BOTH]:
|
|
296
|
+
assert rhs_next_inner is not None
|
|
297
|
+
assert rhs_next_adj is not None
|
|
298
|
+
rhs_next = _set_adjbit_at_header(rhs_next_inner, rhs_next_adj, Continuation.RHS)
|
|
299
|
+
if cont in [Continuation.LHS, Continuation.BOTH]:
|
|
300
|
+
assert lhs_next_inner is not None
|
|
301
|
+
assert lhs_next_adj is not None
|
|
302
|
+
lhs_next = _set_adjbit_at_header(lhs_next_inner, lhs_next_adj, Continuation.LHS)
|
|
303
|
+
|
|
304
|
+
return phrase_freqs, (lhs_next, rhs_next)
|
|
@@ -10,6 +10,7 @@ import sortednp as snp
|
|
|
10
10
|
from copy import deepcopy
|
|
11
11
|
from typing import List, Tuple, Dict, Union, cast, Optional
|
|
12
12
|
from searcharray.utils.roaringish import RoaringishEncoder, convert_keys, sorted_unique
|
|
13
|
+
from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
|
|
13
14
|
import numbers
|
|
14
15
|
import logging
|
|
15
16
|
from collections import defaultdict
|
|
@@ -41,89 +42,6 @@ _neg1 = np.int64(-1)
|
|
|
41
42
|
MAX_POSN = encoder.max_payload
|
|
42
43
|
|
|
43
44
|
|
|
44
|
-
def inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
45
|
-
phrase_freqs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
|
46
|
-
"""Count bigram matches between two encoded arrays, within a 64 bit word with same MSBs.
|
|
47
|
-
|
|
48
|
-
Returns:
|
|
49
|
-
--------
|
|
50
|
-
count: number of matches per doc
|
|
51
|
-
rhs_next: the next rhs array to continue matching
|
|
52
|
-
|
|
53
|
-
"""
|
|
54
|
-
lhs_int, rhs_int = encoder.intersect(lhs, rhs)
|
|
55
|
-
lhs_doc_ids = encoder.keys(lhs_int)
|
|
56
|
-
if len(lhs_int) != len(rhs_int):
|
|
57
|
-
raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
|
|
58
|
-
if len(lhs_int) == 0:
|
|
59
|
-
return phrase_freqs, rhs_int
|
|
60
|
-
same_term = (len(lhs_int) == len(rhs_int) and lhs_int[0] == rhs_int[0])
|
|
61
|
-
if same_term:
|
|
62
|
-
# Find adjacent matches
|
|
63
|
-
rhs_shift = rhs_int << _1
|
|
64
|
-
overlap = lhs_int & rhs_shift
|
|
65
|
-
overlap = encoder.payload_lsb(overlap)
|
|
66
|
-
adjacents = bit_count64(overlap).astype(np.int64)
|
|
67
|
-
adjacents -= -np.floor_divide(adjacents, -2) # ceiling divide
|
|
68
|
-
phrase_freqs[lhs_doc_ids] += adjacents
|
|
69
|
-
return phrase_freqs, rhs_int
|
|
70
|
-
|
|
71
|
-
overlap_bits = (lhs_int & encoder.payload_lsb_mask) & ((rhs_int & encoder.payload_lsb_mask) >> _1)
|
|
72
|
-
rhs_next2 = (overlap_bits << _1) & encoder.payload_lsb_mask
|
|
73
|
-
rhs_next2 |= (rhs_int & (encoder.key_mask | encoder.payload_msb_mask))
|
|
74
|
-
phrase_freqs2 = phrase_freqs.copy()
|
|
75
|
-
matches2 = overlap_bits > 0
|
|
76
|
-
if np.any(matches2):
|
|
77
|
-
transitions = np.argwhere(np.diff(lhs_doc_ids[matches2]) != 0).flatten() + 1
|
|
78
|
-
transitions = np.insert(transitions, 0, 0)
|
|
79
|
-
counted_bits = bit_count64(overlap_bits[matches2])
|
|
80
|
-
reduced = np.add.reduceat(counted_bits,
|
|
81
|
-
transitions)
|
|
82
|
-
phrase_freqs2[np.unique(lhs_doc_ids[matches2])] += reduced
|
|
83
|
-
return phrase_freqs2, rhs_next2
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
def adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
87
|
-
phrase_freqs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
|
88
|
-
"""Count bigram matches between two encoded arrays where they occur in adjacent 64 bit words.
|
|
89
|
-
|
|
90
|
-
Returns:
|
|
91
|
-
--------
|
|
92
|
-
count: number of matches per doc
|
|
93
|
-
rhs_next: the next rhs array to continue matching
|
|
94
|
-
|
|
95
|
-
"""
|
|
96
|
-
lhs_int, rhs_int = encoder.intersect_rshift(lhs, rhs, rshift=_neg1)
|
|
97
|
-
lhs_doc_ids = encoder.keys(lhs_int)
|
|
98
|
-
# lhs lsb set and rhs lsb's most significant bit set
|
|
99
|
-
upper_bit = _1 << (encoder.payload_lsb_bits - _1)
|
|
100
|
-
matches = ((lhs_int & upper_bit) != 0) & ((rhs_int & _1) != 0)
|
|
101
|
-
unique, counts = np.unique(lhs_doc_ids[matches], return_counts=True)
|
|
102
|
-
phrase_freqs[unique] += counts
|
|
103
|
-
rhs_next = rhs_int
|
|
104
|
-
rhs_next[~matches] |= ~encoder.payload_lsb_mask
|
|
105
|
-
rhs_next[matches] |= (encoder.payload_lsb_mask & _1)
|
|
106
|
-
return phrase_freqs, rhs_next
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
def bigram_freqs(lhs: np.ndarray, rhs: np.ndarray, phrase_freqs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
|
110
|
-
"""Count bigram matches between two encoded arrays.
|
|
111
|
-
|
|
112
|
-
Returns:
|
|
113
|
-
--------
|
|
114
|
-
count: number of matches per doc
|
|
115
|
-
rhs_next: the next rhs array to continue matching
|
|
116
|
-
|
|
117
|
-
"""
|
|
118
|
-
# Combine lhs and rhs matches from two strategies
|
|
119
|
-
phrase_freqs, rhs_next_inner = inner_bigram_freqs(lhs, rhs, phrase_freqs)
|
|
120
|
-
phrase_freqs, rhs_next_adj = adjacent_bigram_freqs(lhs, rhs, phrase_freqs)
|
|
121
|
-
rhs_next = np.sort(np.concatenate([rhs_next_inner, rhs_next_adj]))
|
|
122
|
-
|
|
123
|
-
# Combine
|
|
124
|
-
return phrase_freqs, rhs_next
|
|
125
|
-
|
|
126
|
-
|
|
127
45
|
def trim_phrase_search(encoded_posns: List[np.ndarray],
|
|
128
46
|
phrase_freqs: np.ndarray) -> List[np.ndarray]:
|
|
129
47
|
"""Trim long phrases by searching the rarest terms first."""
|
|
@@ -147,33 +65,86 @@ def trim_phrase_search(encoded_posns: List[np.ndarray],
|
|
|
147
65
|
for enc_posn_idx in range(len(encoded_posns)):
|
|
148
66
|
if enc_posn_idx == shortest_idx:
|
|
149
67
|
continue
|
|
150
|
-
if len(encoded_posns[enc_posn_idx]) > (
|
|
68
|
+
if len(encoded_posns[enc_posn_idx]) > (20 * min_len):
|
|
151
69
|
encoded_posns[enc_posn_idx] = encoder.slice(encoded_posns[enc_posn_idx],
|
|
152
70
|
shortest_keys)
|
|
153
71
|
|
|
154
72
|
return encoded_posns
|
|
155
73
|
|
|
156
74
|
|
|
157
|
-
def
|
|
158
|
-
|
|
75
|
+
def _compute_phrase_freqs_rhs(encoded_posns: List[np.ndarray],
|
|
76
|
+
phrase_freqs: np.ndarray,
|
|
77
|
+
trim: bool = True) -> np.ndarray:
|
|
78
|
+
"""Compute phrase freqs from a set of encoded positions."""
|
|
159
79
|
if len(encoded_posns) < 2:
|
|
160
80
|
raise ValueError("phrase must have at least two terms")
|
|
161
81
|
|
|
162
82
|
# Trim long phrases by searching the rarest terms first
|
|
163
|
-
if len(encoded_posns) > 3:
|
|
83
|
+
if trim and len(encoded_posns) > 3:
|
|
164
84
|
encoded_posns = trim_phrase_search(encoded_posns, phrase_freqs)
|
|
165
|
-
|
|
166
85
|
mask = np.ones(len(phrase_freqs), dtype=bool)
|
|
86
|
+
|
|
167
87
|
lhs = encoded_posns[0]
|
|
168
88
|
for rhs in encoded_posns[1:]:
|
|
169
89
|
# Only count the count of the last bigram (ignoring the ones where priors did not match)
|
|
170
90
|
phrase_freqs[mask] = 0
|
|
171
|
-
phrase_freqs,
|
|
91
|
+
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs, cont=Continuation.RHS)
|
|
92
|
+
assert conts[1] is not None
|
|
93
|
+
lhs = conts[1]
|
|
172
94
|
mask &= (phrase_freqs > 0)
|
|
95
|
+
# print("-- pf", phrase_freqs)
|
|
96
|
+
# dec = encoder.decode(lhs)
|
|
97
|
+
# print("-- co", dec)
|
|
173
98
|
phrase_freqs[~mask] = 0
|
|
174
99
|
return phrase_freqs
|
|
175
100
|
|
|
176
101
|
|
|
102
|
+
def _compute_phrase_freqs_lhs(encoded_posns: List[np.ndarray],
|
|
103
|
+
phrase_freqs: np.ndarray,
|
|
104
|
+
trim: bool = True) -> np.ndarray:
|
|
105
|
+
"""Compute phrase freqs from a set of encoded positions."""
|
|
106
|
+
if len(encoded_posns) < 2:
|
|
107
|
+
raise ValueError("phrase must have at least two terms")
|
|
108
|
+
|
|
109
|
+
# Trim long phrases by searching the rarest terms first
|
|
110
|
+
if trim and len(encoded_posns) > 3:
|
|
111
|
+
encoded_posns = trim_phrase_search(encoded_posns, phrase_freqs)
|
|
112
|
+
mask = np.ones(len(phrase_freqs), dtype=bool)
|
|
113
|
+
|
|
114
|
+
rhs = encoded_posns[-1]
|
|
115
|
+
for lhs in encoded_posns[-2::-1]:
|
|
116
|
+
# Only count the count of the last bigram (ignoring the ones where priors did not match)
|
|
117
|
+
phrase_freqs[mask] = 0
|
|
118
|
+
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs, cont=Continuation.LHS)
|
|
119
|
+
assert conts[0] is not None
|
|
120
|
+
rhs = conts[0]
|
|
121
|
+
mask &= (phrase_freqs > 0)
|
|
122
|
+
# print("-- pf", phrase_freqs)
|
|
123
|
+
# dec = encoder.decode(lhs)
|
|
124
|
+
# print("-- co", dec)
|
|
125
|
+
phrase_freqs[~mask] = 0
|
|
126
|
+
return phrase_freqs
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def compute_phrase_freqs(encoded_posns: List[np.ndarray],
|
|
130
|
+
phrase_freqs: np.ndarray,
|
|
131
|
+
trim: bool = False) -> np.ndarray:
|
|
132
|
+
"""Compute phrase freqs from a set of encoded positions."""
|
|
133
|
+
shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
|
|
134
|
+
# Slice LHS / RHS at shortest_len_index
|
|
135
|
+
if shortest_len_index <= 1:
|
|
136
|
+
return _compute_phrase_freqs_lhs(encoded_posns, phrase_freqs, trim=trim)
|
|
137
|
+
elif shortest_len_index >= len(encoded_posns) - 2:
|
|
138
|
+
return _compute_phrase_freqs_rhs(encoded_posns, phrase_freqs, trim=trim)
|
|
139
|
+
else:
|
|
140
|
+
# We optimize this case by going middle-out
|
|
141
|
+
# We can take the min of both directions phrase freqs
|
|
142
|
+
lhs = _compute_phrase_freqs_lhs(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim)
|
|
143
|
+
rhs = _compute_phrase_freqs_rhs(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim)
|
|
144
|
+
phrase_freqs = np.minimum(lhs, rhs)
|
|
145
|
+
return phrase_freqs
|
|
146
|
+
|
|
147
|
+
|
|
177
148
|
class PosnBitArrayFromFlatBuilder:
|
|
178
149
|
""" Build from sorted array shape num terms x 3.
|
|
179
150
|
|
|
@@ -365,14 +336,18 @@ class PosnBitArray:
|
|
|
365
336
|
return term_posns
|
|
366
337
|
|
|
367
338
|
def phrase_freqs(self, term_ids: List[int], phrase_freqs: np.ndarray,
|
|
368
|
-
doc_ids: np.ndarray
|
|
339
|
+
doc_ids: np.ndarray,
|
|
340
|
+
min_posn: Optional[int] = None,
|
|
341
|
+
max_posn: Optional[int] = None) -> np.ndarray:
|
|
369
342
|
if len(term_ids) < 2:
|
|
370
343
|
raise ValueError("Must have at least two terms")
|
|
371
|
-
if phrase_freqs.shape[0] == self.max_doc_id + 1:
|
|
344
|
+
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
|
|
372
345
|
enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
|
|
373
346
|
else:
|
|
374
347
|
enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
|
|
375
|
-
keys=doc_ids.view(np.uint64)
|
|
348
|
+
keys=doc_ids.view(np.uint64),
|
|
349
|
+
min_payload=min_posn,
|
|
350
|
+
max_payload=max_posn) for term_id in term_ids]
|
|
376
351
|
return compute_phrase_freqs(enc_term_posns, phrase_freqs)
|
|
377
352
|
|
|
378
353
|
def positions(self, term_id: int, doc_ids) -> Union[List[np.ndarray], np.ndarray]:
|
|
@@ -408,15 +383,23 @@ class PosnBitArray:
|
|
|
408
383
|
decs = [dec[1] for dec in decoded]
|
|
409
384
|
return decs
|
|
410
385
|
|
|
411
|
-
def termfreqs(self, term_id: int,
|
|
386
|
+
def termfreqs(self, term_id: int,
|
|
387
|
+
doc_ids: Optional[np.ndarray] = None,
|
|
388
|
+
min_posn: Optional[int] = None,
|
|
389
|
+
max_posn: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray]:
|
|
412
390
|
"""Count term freqs using unique positions."""
|
|
413
|
-
if doc_ids is None:
|
|
391
|
+
if doc_ids is None and max_posn is None and min_posn is None:
|
|
414
392
|
return self._termfreqs_with_cache(term_id)
|
|
393
|
+
if doc_ids is None:
|
|
394
|
+
doc_ids = sorted_unique(encoder.keys(self.encoded_term_posns[term_id]))
|
|
415
395
|
|
|
416
396
|
encoded = self.encoded_term_posns[term_id]
|
|
417
397
|
term_posns = encoded
|
|
418
398
|
term_posns = encoder.slice(encoded,
|
|
419
|
-
keys=doc_ids.astype(np.uint64)
|
|
399
|
+
keys=doc_ids.astype(np.uint64),
|
|
400
|
+
min_payload=min_posn,
|
|
401
|
+
max_payload=max_posn)
|
|
402
|
+
|
|
420
403
|
return self._computed_term_freqs(term_posns)
|
|
421
404
|
|
|
422
405
|
def _computed_term_freqs(self, term_posns) -> Tuple[np.ndarray, np.ndarray]:
|
|
@@ -519,10 +519,12 @@ class SearchArray(ExtensionArray):
|
|
|
519
519
|
# ***********************************************************
|
|
520
520
|
# Search functionality
|
|
521
521
|
# ***********************************************************
|
|
522
|
-
def termfreqs(self, token: Union[List[str], str]
|
|
522
|
+
def termfreqs(self, token: Union[List[str], str],
|
|
523
|
+
min_posn: Optional[int] = None,
|
|
524
|
+
max_posn: Optional[int] = None) -> np.ndarray:
|
|
523
525
|
token = self._check_token_arg(token)
|
|
524
526
|
if isinstance(token, list):
|
|
525
|
-
return self.phrase_freq(token)
|
|
527
|
+
return self.phrase_freq(token, min_posn=min_posn, max_posn=max_posn)
|
|
526
528
|
|
|
527
529
|
try:
|
|
528
530
|
term_id = self.term_dict.get_term_id(token)
|
|
@@ -531,13 +533,13 @@ class SearchArray(ExtensionArray):
|
|
|
531
533
|
if self.term_mat.subset:
|
|
532
534
|
slice_of_rows = self.term_mat.rows
|
|
533
535
|
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
534
|
-
doc_ids=slice_of_rows)
|
|
536
|
+
doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
|
|
535
537
|
mask = np.isin(self.term_mat.rows, doc_ids)
|
|
536
538
|
matches[mask] = termfreqs
|
|
537
539
|
return matches
|
|
538
540
|
else:
|
|
539
541
|
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
540
|
-
doc_ids=slice_of_rows)
|
|
542
|
+
doc_ids=slice_of_rows, min_posn=min_posn, max_posn=max_posn)
|
|
541
543
|
matches[doc_ids] = termfreqs
|
|
542
544
|
return matches
|
|
543
545
|
except TermMissingError:
|
|
@@ -609,14 +611,19 @@ class SearchArray(ExtensionArray):
|
|
|
609
611
|
mask = np.sum(masks, axis=0) >= min_should_match
|
|
610
612
|
return mask
|
|
611
613
|
|
|
612
|
-
def phrase_freq(self, tokens: List[str],
|
|
613
|
-
|
|
614
|
+
def phrase_freq(self, tokens: List[str],
|
|
615
|
+
slop=1,
|
|
616
|
+
min_posn: Optional[int] = None,
|
|
617
|
+
max_posn: Optional[int] = None) -> np.ndarray:
|
|
618
|
+
if slop == 1:
|
|
614
619
|
phrase_freqs = np.zeros(len(self))
|
|
615
620
|
try:
|
|
616
621
|
doc_ids = self.term_mat.rows
|
|
617
622
|
term_ids = [self.term_dict.get_term_id(token) for token in tokens]
|
|
618
623
|
return self.posns.phrase_freqs(term_ids, doc_ids=doc_ids,
|
|
619
|
-
phrase_freqs=phrase_freqs
|
|
624
|
+
phrase_freqs=phrase_freqs,
|
|
625
|
+
min_posn=min_posn,
|
|
626
|
+
max_posn=max_posn)
|
|
620
627
|
except TermMissingError:
|
|
621
628
|
return phrase_freqs
|
|
622
629
|
else:
|