searcharray 0.0.15__tar.gz → 0.0.17__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.15 → searcharray-0.0.17}/PKG-INFO +1 -2
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/phrase/middle_out.py +30 -122
- searcharray-0.0.17/searcharray/phrase/scan_merge.py +75 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/postings.py +69 -204
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/term_dict.py +14 -0
- searcharray-0.0.17/searcharray/utils/bitcount.py +30 -0
- searcharray-0.0.17/searcharray/utils/mat_set.py +143 -0
- searcharray-0.0.17/searcharray/utils/roaringish.py +139 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/utils/row_viewable_matrix.py +21 -9
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/PKG-INFO +1 -2
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/SOURCES.txt +3 -1
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/requires.txt +0 -1
- {searcharray-0.0.15 → searcharray-0.0.17}/setup.py +2 -2
- {searcharray-0.0.15 → searcharray-0.0.17}/test/test_extension_array.py +10 -1
- {searcharray-0.0.15 → searcharray-0.0.17}/test/test_msmarco.py +19 -5
- {searcharray-0.0.15 → searcharray-0.0.17}/test/test_phrase_matches.py +3 -23
- {searcharray-0.0.15 → searcharray-0.0.17}/test/test_tmdb.py +29 -2
- {searcharray-0.0.15 → searcharray-0.0.17}/test/test_utils.py +11 -1
- searcharray-0.0.15/searcharray/phrase/scan_merge.py +0 -230
- searcharray-0.0.15/searcharray/phrase/wide_spans.py +0 -183
- {searcharray-0.0.15 → searcharray-0.0.17}/README.md +0 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/pyproject.toml +0 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/__init__.py +0 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/phrase/padded_arrays.py +0 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/stemmers.py +0 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.15 → searcharray-0.0.17}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: searcharray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.17
|
|
4
4
|
Summary: Searchable pandas text extension arrays for prototyping search
|
|
5
5
|
Home-page: https://github.com/softwaredoug/searcharray
|
|
6
6
|
Author: Doug Turnbull
|
|
@@ -21,7 +21,6 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
21
21
|
Requires-Python: >=3.7, <4
|
|
22
22
|
Description-Content-Type: text/markdown
|
|
23
23
|
Requires-Dist: pandas>=2.0.0
|
|
24
|
-
Requires-Dist: scipy>=1.0.0
|
|
25
24
|
Requires-Dist: sortednp
|
|
26
25
|
Provides-Extra: dev
|
|
27
26
|
Requires-Dist: check-manifest; extra == "dev"
|
|
@@ -8,10 +8,12 @@ https://colab.research.google.com/drive/10tIEkdlCE_1J_CcgEcV0jkLfBc-0H4am?authus
|
|
|
8
8
|
import numpy as np
|
|
9
9
|
import sortednp as snp
|
|
10
10
|
from copy import deepcopy
|
|
11
|
-
from typing import List
|
|
11
|
+
from typing import List
|
|
12
|
+
from searcharray.utils.roaringish import RoaringishEncoder, convert_keys
|
|
12
13
|
import numbers
|
|
13
14
|
import logging
|
|
14
|
-
|
|
15
|
+
|
|
16
|
+
from searcharray.utils.bitcount import bit_count64
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
logger = logging.getLogger(__name__)
|
|
@@ -26,129 +28,21 @@ logger.addHandler(handler)
|
|
|
26
28
|
logger.setLevel(logging.ERROR)
|
|
27
29
|
|
|
28
30
|
|
|
29
|
-
|
|
30
|
-
DOC_ID_BITS = 28
|
|
31
|
-
POSN_MSB_MASK = 0x0000000FFFFC0000
|
|
32
|
-
POSN_MSB_BITS = 18
|
|
33
|
-
POSN_LSB_MASK = 0x000000000003FFFF
|
|
34
|
-
POSN_LSB_BITS = 18
|
|
35
|
-
|
|
36
|
-
assert DOC_ID_BITS + POSN_MSB_BITS + POSN_LSB_BITS == 64
|
|
37
|
-
assert DOC_ID_MASK | POSN_MSB_MASK | POSN_LSB_MASK == 0xFFFFFFFFFFFFFFFF
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def validate_posns(posns: np.ndarray):
|
|
41
|
-
if not np.all(posns < 2**POSN_LSB_BITS):
|
|
42
|
-
raise ValueError(f"Positions must be less than {2**POSN_LSB_BITS}")
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
def encode_posns(posns: np.ndarray, doc_ids: Optional[np.ndarray] = None):
|
|
46
|
-
"""Pack a sorted array of positions into compact bit array.
|
|
47
|
-
|
|
48
|
-
each returned array represents a single term, with doc_id as 32 MSBs
|
|
49
|
-
|
|
50
|
-
| 32 MSBs | 16 LSBs | 16 LSBs |
|
|
51
|
-
doc_id bits msbs posns
|
|
52
|
-
|
|
53
|
-
for later easy intersection of 32+16 msbs, then checking for adjacent
|
|
54
|
-
positions
|
|
55
|
-
|
|
56
|
-
"""
|
|
57
|
-
validate_posns(posns)
|
|
58
|
-
cols = posns // POSN_LSB_BITS # Header of bit to use
|
|
59
|
-
cols = cols.astype(np.uint64) << POSN_MSB_BITS
|
|
60
|
-
if doc_ids is not None:
|
|
61
|
-
cols |= doc_ids.astype(np.uint64) << (64 - DOC_ID_BITS)
|
|
62
|
-
values = posns % POSN_LSB_BITS # Value to encode
|
|
63
|
-
|
|
64
|
-
change_indices = np.nonzero(np.diff(cols))[0] + 1
|
|
65
|
-
change_indices = np.insert(change_indices, 0, 0)
|
|
66
|
-
|
|
67
|
-
encoded = cols | (1 << values)
|
|
68
|
-
if len(encoded) == 0:
|
|
69
|
-
return encoded
|
|
70
|
-
return np.bitwise_or.reduceat(encoded, change_indices)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def decode_posns(encoded, get_doc_ids=True):
|
|
74
|
-
start = perf_counter()
|
|
75
|
-
doc_ids = (encoded & DOC_ID_MASK) >> (64 - DOC_ID_BITS)
|
|
76
|
-
msbs = (encoded & POSN_MSB_MASK) >> POSN_MSB_BITS
|
|
77
|
-
to_concat = []
|
|
78
|
-
for bit in range(POSN_LSB_BITS):
|
|
79
|
-
mask = 1 << bit
|
|
80
|
-
lsbs = encoded & mask
|
|
81
|
-
set_lsbs = (lsbs != 0)
|
|
82
|
-
this_doc_ids = doc_ids[set_lsbs]
|
|
83
|
-
posns = bit + (msbs[set_lsbs] * POSN_LSB_BITS)
|
|
84
|
-
doc_with_posn = np.dstack([this_doc_ids, posns])[0]
|
|
85
|
-
to_concat.append(doc_with_posn)
|
|
86
|
-
|
|
87
|
-
logger.debug(f"loop took {perf_counter() - start:.2f} seconds")
|
|
88
|
-
stacked = np.vstack(to_concat)
|
|
89
|
-
# Sort by doc_id, then posn
|
|
90
|
-
sorted_posns = stacked[np.lexsort((stacked[:, 1], stacked[:, 0]))]
|
|
91
|
-
doc_ids, idx = np.unique(sorted_posns[:, 0], return_index=True)
|
|
92
|
-
grouped = np.split(sorted_posns[:, 1], idx[1:])
|
|
93
|
-
if get_doc_ids:
|
|
94
|
-
as_list = list(zip(doc_ids, grouped))
|
|
95
|
-
else:
|
|
96
|
-
as_list = grouped
|
|
97
|
-
|
|
98
|
-
logger.debug(f"groupby took {perf_counter() - start:.2f} seconds | got {len(as_list)}")
|
|
99
|
-
return as_list
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
def intersect_msbs(lhs: np.ndarray, rhs: np.ndarray):
|
|
103
|
-
"""Return the MSBs that are common to both lhs and rhs."""
|
|
104
|
-
# common = np.intersect1d(lhs_msbs, rhs_msbs)
|
|
105
|
-
_, (lhs_idx, rhs_idx) = snp.intersect(lhs >> POSN_LSB_BITS, rhs >> POSN_LSB_BITS, indices=True)
|
|
106
|
-
# With large arrays np.isin becomes a bottleneck
|
|
107
|
-
return lhs[lhs_idx], rhs[rhs_idx]
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
def convert_doc_ids(doc_ids):
|
|
111
|
-
if isinstance(doc_ids, numbers.Number):
|
|
112
|
-
return np.asarray([doc_ids], dtype=np.uint64)
|
|
113
|
-
elif isinstance(doc_ids, list):
|
|
114
|
-
return np.asarray(doc_ids, dtype=np.uint64)
|
|
115
|
-
elif isinstance(doc_ids, np.ndarray):
|
|
116
|
-
return doc_ids.astype(np.uint64)
|
|
117
|
-
elif isinstance(doc_ids, range) and len(doc_ids) > 0:
|
|
118
|
-
# return np.asarray(doc_ids, dtype=np.uint64) # UNFORTUNATE COPY
|
|
119
|
-
return np.arange(doc_ids[0], doc_ids[-1] + 1, dtype=np.uint64) + doc_ids[0]
|
|
120
|
-
elif isinstance(doc_ids, range):
|
|
121
|
-
return np.asarray([], dtype=np.uint64)
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
def get_docs(encoded: np.ndarray, doc_ids: np.ndarray):
|
|
125
|
-
"""Get list of encoded that have positions in doc_ids."""
|
|
126
|
-
doc_ids = convert_doc_ids(doc_ids)
|
|
127
|
-
assert len(doc_ids.shape) == 1
|
|
128
|
-
assert len(encoded.shape) == 1
|
|
129
|
-
encoded_doc_ids = encoded.astype(np.uint64) >> (64 - DOC_ID_BITS)
|
|
130
|
-
empty = doc_ids << (64 - DOC_ID_BITS)
|
|
131
|
-
_, (idx_docs, idx_enc) = snp.intersect(doc_ids, encoded_doc_ids, indices=True,
|
|
132
|
-
duplicates=snp.KEEP_MAX_N)
|
|
133
|
-
|
|
134
|
-
found = encoded[idx_enc]
|
|
135
|
-
|
|
136
|
-
merged = snp.merge(found, empty, duplicates=snp.DROP)
|
|
137
|
-
return merged
|
|
31
|
+
encoder = RoaringishEncoder()
|
|
138
32
|
|
|
139
33
|
|
|
140
34
|
def inner_bigram_match(lhs, rhs):
|
|
141
35
|
"""Count bigram matches between two encoded arrays."""
|
|
142
|
-
lhs, rhs =
|
|
36
|
+
lhs, rhs = encoder.intersect(lhs, rhs)
|
|
143
37
|
if len(lhs) != len(rhs):
|
|
144
38
|
raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
|
|
145
|
-
rhs_next = (rhs &
|
|
39
|
+
rhs_next = (rhs & encoder.key_mask)
|
|
146
40
|
rhs_next_mask = np.zeros(len(rhs), dtype=bool)
|
|
147
41
|
counts = []
|
|
148
42
|
# With popcount soon to be in numpy, this could potentially
|
|
149
43
|
# be simply a left shift of the RHS LSB poppcount, and and a popcount
|
|
150
44
|
# to count the overlaps
|
|
151
|
-
for bit in range(1,
|
|
45
|
+
for bit in range(1, encoder.payload_lsb_bits - 1):
|
|
152
46
|
lhs_mask = 1 << bit
|
|
153
47
|
rhs_mask = 1 << (bit + 1)
|
|
154
48
|
lhs_set = (lhs & lhs_mask) != 0
|
|
@@ -194,9 +88,9 @@ class PosnBitArrayBuilder:
|
|
|
194
88
|
doc_ids = self.term_posn_doc_ids[term_id]
|
|
195
89
|
if isinstance(doc_ids, list):
|
|
196
90
|
doc_ids = np.asarray(doc_ids, dtype=np.uint32)
|
|
197
|
-
encoded =
|
|
91
|
+
encoded = encoder.encode(keys=doc_ids, payload=posns)
|
|
198
92
|
if check:
|
|
199
|
-
decode_again =
|
|
93
|
+
decode_again = encoder.decode(encoded)
|
|
200
94
|
docs_to_posns = dict(decode_again)
|
|
201
95
|
doc_ids_again = []
|
|
202
96
|
posns_again = []
|
|
@@ -264,10 +158,11 @@ class PosnBitArray:
|
|
|
264
158
|
def slice(self, key):
|
|
265
159
|
sliced_term_posns = {}
|
|
266
160
|
doc_ids = index_range(self.doc_ids, key)
|
|
161
|
+
np_doc_ids = convert_keys(doc_ids)
|
|
267
162
|
for term_id, posns in self.encoded_term_posns.items():
|
|
268
163
|
encoded = self.encoded_term_posns[term_id]
|
|
269
164
|
assert len(encoded.shape) == 1
|
|
270
|
-
sliced_term_posns[term_id] =
|
|
165
|
+
sliced_term_posns[term_id] = encoder.slice(encoded, keys=np_doc_ids)
|
|
271
166
|
|
|
272
167
|
return PosnBitArray(sliced_term_posns, doc_ids=doc_ids)
|
|
273
168
|
|
|
@@ -289,8 +184,8 @@ class PosnBitArray:
|
|
|
289
184
|
self.encoded_term_posns[term_id] = snp.merge(posns_self, posns_other)
|
|
290
185
|
|
|
291
186
|
def doc_encoded_posns(self, term_id: int, doc_id: int) -> List:
|
|
292
|
-
term_posns =
|
|
293
|
-
|
|
187
|
+
term_posns = encoder.slice(self.encoded_term_posns[term_id],
|
|
188
|
+
keys=np.asarray([doc_id], dtype=np.uint64))
|
|
294
189
|
return term_posns
|
|
295
190
|
|
|
296
191
|
def positions(self, term_id: int, key) -> List:
|
|
@@ -300,15 +195,16 @@ class PosnBitArray:
|
|
|
300
195
|
doc_ids = np.asarray([doc_ids])
|
|
301
196
|
|
|
302
197
|
try:
|
|
303
|
-
|
|
304
|
-
|
|
198
|
+
np_doc_ids = convert_keys(doc_ids)
|
|
199
|
+
term_posns = encoder.slice(self.encoded_term_posns[term_id],
|
|
200
|
+
keys=np_doc_ids)
|
|
305
201
|
except KeyError:
|
|
306
202
|
r_val = [np.array([], dtype=np.uint32) for doc_id in doc_ids]
|
|
307
203
|
if len(r_val) == 1 and isinstance(key, numbers.Number):
|
|
308
204
|
r_val = r_val[0]
|
|
309
205
|
return r_val
|
|
310
206
|
|
|
311
|
-
decoded =
|
|
207
|
+
decoded = encoder.decode(encoded=term_posns, get_keys=True)
|
|
312
208
|
|
|
313
209
|
if len(decoded) == 0:
|
|
314
210
|
return np.array([], dtype=np.uint32)
|
|
@@ -328,6 +224,18 @@ class PosnBitArray:
|
|
|
328
224
|
decs = decs[0]
|
|
329
225
|
return decs
|
|
330
226
|
|
|
227
|
+
def termfreqs(self, term_id: int) -> np.ndarray:
|
|
228
|
+
"""Count term freqs using unique positions."""
|
|
229
|
+
encoded = self.encoded_term_posns[term_id]
|
|
230
|
+
doc_ids = encoder.keys(encoded)
|
|
231
|
+
change_indices = np.nonzero(np.diff(doc_ids))[0]
|
|
232
|
+
change_indices = np.concatenate(([0], change_indices + 1))
|
|
233
|
+
posns = encoded & encoder.payload_lsb_mask
|
|
234
|
+
bit_counts = bit_count64(posns)
|
|
235
|
+
|
|
236
|
+
term_freqs = np.add.reduceat(bit_counts, change_indices)
|
|
237
|
+
return np.unique(doc_ids), term_freqs
|
|
238
|
+
|
|
331
239
|
def insert(self, key, term_ids_to_posns, is_encoded=False):
|
|
332
240
|
new_posns = PosnBitArrayBuilder()
|
|
333
241
|
if is_encoded:
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from typing import List
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def _self_adjs(prior_posns, next_posns):
|
|
6
|
+
"""Given two arrays of positions, return the self adjacencies.
|
|
7
|
+
|
|
8
|
+
Prior is a subset of next, find all the locations in next in prior,
|
|
9
|
+
then get all the adjacent positions, returning a numpy array.
|
|
10
|
+
"""
|
|
11
|
+
in1d = np.in1d(next_posns, prior_posns)
|
|
12
|
+
if len(in1d) == 0:
|
|
13
|
+
return np.array([])
|
|
14
|
+
|
|
15
|
+
# Where there's an intersection
|
|
16
|
+
start_indices = np.argwhere(in1d).flatten()
|
|
17
|
+
# The adjacent to the intersections
|
|
18
|
+
adj_indices = np.argwhere(in1d).flatten() + 1
|
|
19
|
+
adj_indices = adj_indices[adj_indices < len(next_posns)]
|
|
20
|
+
|
|
21
|
+
arr = np.union1d(next_posns[start_indices], next_posns[adj_indices])
|
|
22
|
+
arr.sort()
|
|
23
|
+
return arr
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def scan_merge_ins(term_posns: List[np.ndarray],
|
|
27
|
+
phrase_freqs: np.ndarray, slop=1):
|
|
28
|
+
"""Merge bigram, by bigram, using np.searchsorted to find if insert posns match slop.
|
|
29
|
+
|
|
30
|
+
See https://colab.research.google.com/drive/1EeqHYuCiqyptd-awS67Re78pqVdTfH4A
|
|
31
|
+
"""
|
|
32
|
+
# Any identical terms, default to shitty algo for now
|
|
33
|
+
# if len(tokens) != len(set(tokens)):
|
|
34
|
+
# return self.phrase_freq_shitty(tokens, slop=slop)
|
|
35
|
+
|
|
36
|
+
# Iterate each phrase with its next term
|
|
37
|
+
prior_posns = term_posns[0]
|
|
38
|
+
for term_cnt, curr_posns in enumerate(term_posns[1:]):
|
|
39
|
+
assert len(prior_posns) == len(curr_posns)
|
|
40
|
+
bigram_freqs = np.zeros(len(curr_posns))
|
|
41
|
+
cont_posns = []
|
|
42
|
+
for idx in range(len(curr_posns)):
|
|
43
|
+
# Find insert position of every next term in prior term's positions
|
|
44
|
+
# Intuition:
|
|
45
|
+
# https://colab.research.google.com/drive/1EeqHYuCiqyptd-awS67Re78pqVdTfH4A
|
|
46
|
+
if len(prior_posns[idx]) == 0:
|
|
47
|
+
bigram_freqs[idx] = 0
|
|
48
|
+
cont_posns.append([])
|
|
49
|
+
continue
|
|
50
|
+
priors_in_self = _self_adjs(prior_posns[idx], curr_posns[idx])
|
|
51
|
+
takeaway = 0
|
|
52
|
+
satisfies_slop = None
|
|
53
|
+
cont_indices = None
|
|
54
|
+
# Different term
|
|
55
|
+
if len(priors_in_self) == 0:
|
|
56
|
+
ins_posns = np.searchsorted(prior_posns[idx], curr_posns[idx], side='right')
|
|
57
|
+
prior_adjacents = prior_posns[idx][ins_posns - 1]
|
|
58
|
+
adjacents = curr_posns[idx] - prior_adjacents
|
|
59
|
+
satisfies_slop = (adjacents <= slop) & ~(ins_posns == 0)
|
|
60
|
+
cont_indices = np.argwhere(satisfies_slop).flatten()
|
|
61
|
+
# Overlapping term
|
|
62
|
+
else:
|
|
63
|
+
adjacents = np.diff(priors_in_self)
|
|
64
|
+
satisfies_slop = adjacents <= slop
|
|
65
|
+
consecutive_slops = satisfies_slop[1:] & satisfies_slop[:-1]
|
|
66
|
+
sum_consecutive = np.sum(consecutive_slops)
|
|
67
|
+
takeaway = -np.floor_divide(sum_consecutive, -2) # ceiling divide
|
|
68
|
+
cont_indices = np.argwhere(satisfies_slop).flatten() + 1
|
|
69
|
+
|
|
70
|
+
bigram_freqs[idx] = np.sum(satisfies_slop) - takeaway
|
|
71
|
+
cont_posn = curr_posns[idx][cont_indices]
|
|
72
|
+
cont_posns.append(cont_posn)
|
|
73
|
+
phrase_freqs = bigram_freqs
|
|
74
|
+
prior_posns = cont_posns
|
|
75
|
+
return phrase_freqs
|