searcharray 0.0.13a0__tar.gz → 0.0.14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.13a0 → searcharray-0.0.14}/PKG-INFO +1 -1
- searcharray-0.0.14/searcharray/phrase/__init__.py +0 -0
- searcharray-0.0.14/searcharray/phrase/middle_out.py +326 -0
- searcharray-0.0.14/searcharray/phrase/padded_arrays.py +221 -0
- searcharray-0.0.14/searcharray/phrase/posn_diffs.py +147 -0
- searcharray-0.0.14/searcharray/phrase/scan_merge.py +230 -0
- searcharray-0.0.14/searcharray/phrase/wide_spans.py +183 -0
- searcharray-0.0.14/searcharray/utils/__init__.py +0 -0
- searcharray-0.0.14/searcharray/utils/row_viewable_matrix.py +77 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/SOURCES.txt +8 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/setup.py +3 -3
- {searcharray-0.0.13a0 → searcharray-0.0.14}/README.md +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/pyproject.toml +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray/__init__.py +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray/postings.py +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray/stemmers.py +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/setup.cfg +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/test/test_extension_array.py +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/test/test_msmarco.py +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/test/test_tmdb.py +0 -0
- {searcharray-0.0.13a0 → searcharray-0.0.14}/test/test_utils.py +0 -0
|
File without changes
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"""Encode positions in bits along with some neighboring information for wrapping.
|
|
2
|
+
|
|
3
|
+
See this notebook for motivation:
|
|
4
|
+
|
|
5
|
+
https://colab.research.google.com/drive/10tIEkdlCE_1J_CcgEcV0jkLfBc-0H4am?authuser=1#scrollTo=XWzy-n9dF3PG
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
import numpy as np
|
|
9
|
+
import sortednp as snp
|
|
10
|
+
from copy import deepcopy
|
|
11
|
+
from typing import List, Optional
|
|
12
|
+
import numbers
|
|
13
|
+
import logging
|
|
14
|
+
from time import perf_counter
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
# When running in pytest
|
|
20
|
+
import sys # noqa
|
|
21
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
22
|
+
handler.setLevel(logging.ERROR)
|
|
23
|
+
formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
|
|
24
|
+
handler.setFormatter(formatter)
|
|
25
|
+
logger.addHandler(handler)
|
|
26
|
+
logger.setLevel(logging.ERROR)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
DOC_ID_MASK = 0xFFFFFFF000000000
|
|
30
|
+
DOC_ID_BITS = 28
|
|
31
|
+
POSN_MSB_MASK = 0x0000000FFFFC0000
|
|
32
|
+
POSN_MSB_BITS = 18
|
|
33
|
+
POSN_LSB_MASK = 0x000000000003FFFF
|
|
34
|
+
POSN_LSB_BITS = 18
|
|
35
|
+
|
|
36
|
+
assert DOC_ID_BITS + POSN_MSB_BITS + POSN_LSB_BITS == 64
|
|
37
|
+
assert DOC_ID_MASK | POSN_MSB_MASK | POSN_LSB_MASK == 0xFFFFFFFFFFFFFFFF
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def validate_posns(posns: np.ndarray):
|
|
41
|
+
if not np.all(posns < 2**POSN_LSB_BITS):
|
|
42
|
+
raise ValueError(f"Positions must be less than {2**POSN_LSB_BITS}")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def encode_posns(posns: np.ndarray, doc_ids: Optional[np.ndarray] = None):
|
|
46
|
+
"""Pack a sorted array of positions into compact bit array.
|
|
47
|
+
|
|
48
|
+
each returned array represents a single term, with doc_id as 32 MSBs
|
|
49
|
+
|
|
50
|
+
| 32 MSBs | 16 LSBs | 16 LSBs |
|
|
51
|
+
doc_id bits msbs posns
|
|
52
|
+
|
|
53
|
+
for later easy intersection of 32+16 msbs, then checking for adjacent
|
|
54
|
+
positions
|
|
55
|
+
|
|
56
|
+
"""
|
|
57
|
+
validate_posns(posns)
|
|
58
|
+
cols = posns // POSN_LSB_BITS # Header of bit to use
|
|
59
|
+
cols = cols.astype(np.uint64) << POSN_MSB_BITS
|
|
60
|
+
if doc_ids is not None:
|
|
61
|
+
cols |= doc_ids.astype(np.uint64) << (64 - DOC_ID_BITS)
|
|
62
|
+
values = posns % POSN_LSB_BITS # Value to encode
|
|
63
|
+
|
|
64
|
+
change_indices = np.nonzero(np.diff(cols))[0] + 1
|
|
65
|
+
change_indices = np.insert(change_indices, 0, 0)
|
|
66
|
+
|
|
67
|
+
encoded = cols | (1 << values)
|
|
68
|
+
if len(encoded) == 0:
|
|
69
|
+
return encoded
|
|
70
|
+
return np.bitwise_or.reduceat(encoded, change_indices)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def decode_posns(encoded):
|
|
74
|
+
start = perf_counter()
|
|
75
|
+
doc_ids = (encoded & DOC_ID_MASK) >> (64 - DOC_ID_BITS)
|
|
76
|
+
msbs = (encoded & POSN_MSB_MASK) >> POSN_MSB_BITS
|
|
77
|
+
to_concat = []
|
|
78
|
+
for bit in range(POSN_LSB_BITS):
|
|
79
|
+
mask = 1 << bit
|
|
80
|
+
lsbs = encoded & mask
|
|
81
|
+
set_lsbs = (lsbs != 0)
|
|
82
|
+
this_doc_ids = doc_ids[set_lsbs]
|
|
83
|
+
posns = bit + (msbs[set_lsbs] * POSN_LSB_BITS)
|
|
84
|
+
doc_with_posn = np.dstack([this_doc_ids, posns])[0]
|
|
85
|
+
to_concat.append(doc_with_posn)
|
|
86
|
+
|
|
87
|
+
logger.debug(f"loop took {perf_counter() - start:.2f} seconds")
|
|
88
|
+
stacked = np.vstack(to_concat)
|
|
89
|
+
# Sort by doc_id, then posn
|
|
90
|
+
sorted_posns = stacked[np.lexsort((stacked[:, 1], stacked[:, 0]))]
|
|
91
|
+
doc_ids, idx = np.unique(sorted_posns[:, 0], return_index=True)
|
|
92
|
+
grouped = np.split(sorted_posns[:, 1], idx[1:])
|
|
93
|
+
as_list = list(zip(doc_ids, grouped))
|
|
94
|
+
|
|
95
|
+
logger.debug(f"groupby took {perf_counter() - start:.2f} seconds | got {len(as_list)}")
|
|
96
|
+
return as_list
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def intersect_msbs(lhs: np.ndarray, rhs: np.ndarray):
|
|
100
|
+
"""Return the MSBs that are common to both lhs and rhs."""
|
|
101
|
+
# common = np.intersect1d(lhs_msbs, rhs_msbs)
|
|
102
|
+
_, (lhs_idx, rhs_idx) = snp.intersect(lhs >> POSN_LSB_BITS, rhs >> POSN_LSB_BITS, indices=True)
|
|
103
|
+
# With large arrays np.isin becomes a bottleneck
|
|
104
|
+
return lhs[lhs_idx], rhs[rhs_idx]
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def convert_doc_ids(doc_ids):
|
|
108
|
+
if isinstance(doc_ids, numbers.Number):
|
|
109
|
+
return np.asarray([doc_ids], dtype=np.uint64)
|
|
110
|
+
elif isinstance(doc_ids, list):
|
|
111
|
+
return np.asarray(doc_ids, dtype=np.uint64)
|
|
112
|
+
elif isinstance(doc_ids, np.ndarray):
|
|
113
|
+
return doc_ids.astype(np.uint64)
|
|
114
|
+
elif isinstance(doc_ids, range):
|
|
115
|
+
return np.asarray(doc_ids, dtype=np.uint64) # UNFORTUNATE COPY
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def get_docs(encoded: np.ndarray, doc_ids: np.ndarray):
|
|
119
|
+
"""Get list of encoded that have positions in doc_ids."""
|
|
120
|
+
doc_ids = convert_doc_ids(doc_ids)
|
|
121
|
+
assert len(doc_ids.shape) == 1
|
|
122
|
+
assert len(encoded.shape) == 1
|
|
123
|
+
encoded_doc_ids = encoded.astype(np.uint64) >> (64 - DOC_ID_BITS)
|
|
124
|
+
empty = doc_ids << (64 - DOC_ID_BITS)
|
|
125
|
+
_, (idx_docs, idx_enc) = snp.intersect(doc_ids, encoded_doc_ids, indices=True,
|
|
126
|
+
duplicates=snp.KEEP_MAX_N)
|
|
127
|
+
|
|
128
|
+
found = encoded[idx_enc]
|
|
129
|
+
empties = empty[np.isin(doc_ids, found, invert=True)]
|
|
130
|
+
|
|
131
|
+
merged = snp.merge(found, empties, duplicates=snp.DROP)
|
|
132
|
+
return merged
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def inner_bigram_match(lhs, rhs):
|
|
136
|
+
"""Count bigram matches between two encoded arrays."""
|
|
137
|
+
lhs, rhs = intersect_msbs(lhs, rhs)
|
|
138
|
+
if len(lhs) != len(rhs):
|
|
139
|
+
raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
|
|
140
|
+
rhs_next = (rhs & DOC_ID_MASK)
|
|
141
|
+
rhs_next_mask = np.zeros(len(rhs), dtype=bool)
|
|
142
|
+
counts = []
|
|
143
|
+
# With popcount soon to be in numpy, this could potentially
|
|
144
|
+
# be simply a left shift of the RHS LSB poppcount, and and a popcount
|
|
145
|
+
# to count the overlaps
|
|
146
|
+
for bit in range(1, POSN_LSB_BITS - 1):
|
|
147
|
+
lhs_mask = 1 << bit
|
|
148
|
+
rhs_mask = 1 << (bit + 1)
|
|
149
|
+
lhs_set = (lhs & lhs_mask) != 0
|
|
150
|
+
rhs_set = (rhs & rhs_mask) != 0
|
|
151
|
+
|
|
152
|
+
matches = lhs_set & rhs_set
|
|
153
|
+
rhs_next[matches] |= rhs_mask
|
|
154
|
+
rhs_next_mask |= matches
|
|
155
|
+
counts.append(np.count_nonzero(matches))
|
|
156
|
+
return np.sum(counts), rhs_next[rhs_next_mask]
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class PosnBitArrayBuilder:
|
|
160
|
+
|
|
161
|
+
def __init__(self):
|
|
162
|
+
self.term_posns = {}
|
|
163
|
+
self.term_posn_doc_ids = {}
|
|
164
|
+
self.max_doc_id = 0
|
|
165
|
+
|
|
166
|
+
def add_posns(self, doc_id: int, term_id: int, posns):
|
|
167
|
+
if len(posns.shape) != 1:
|
|
168
|
+
raise ValueError("posns must be a 1D array")
|
|
169
|
+
if term_id not in self.term_posns:
|
|
170
|
+
self.term_posns[term_id] = []
|
|
171
|
+
self.term_posn_doc_ids[term_id] = []
|
|
172
|
+
doc_ids = [doc_id] * len(posns)
|
|
173
|
+
self.term_posns[term_id].extend(posns.tolist())
|
|
174
|
+
self.term_posn_doc_ids[term_id].extend(doc_ids)
|
|
175
|
+
self.max_doc_id = max(self.max_doc_id, doc_id)
|
|
176
|
+
|
|
177
|
+
def ensure_capacity(self, doc_id):
|
|
178
|
+
self.max_doc_id = max(self.max_doc_id, doc_id)
|
|
179
|
+
|
|
180
|
+
def build(self, check=False):
|
|
181
|
+
encoded_term_posns = {}
|
|
182
|
+
for term_id, posns in self.term_posns.items():
|
|
183
|
+
if len(posns) == 0:
|
|
184
|
+
posns = np.asarray([], dtype=np.uint32).flatten()
|
|
185
|
+
elif isinstance(posns, list):
|
|
186
|
+
posns_arr = np.asarray(posns, dtype=np.uint32)
|
|
187
|
+
assert len(posns_arr.shape) == 1
|
|
188
|
+
posns = posns_arr
|
|
189
|
+
doc_ids = self.term_posn_doc_ids[term_id]
|
|
190
|
+
if isinstance(doc_ids, list):
|
|
191
|
+
doc_ids = np.asarray(doc_ids, dtype=np.uint32)
|
|
192
|
+
encoded = encode_posns(doc_ids=doc_ids, posns=posns)
|
|
193
|
+
decode_again = decode_posns(encoded)
|
|
194
|
+
if check:
|
|
195
|
+
docs_to_posns = dict(decode_again)
|
|
196
|
+
doc_ids_again = []
|
|
197
|
+
posns_again = []
|
|
198
|
+
for doc_id, posns_dec in docs_to_posns.items():
|
|
199
|
+
for posn in posns_dec:
|
|
200
|
+
doc_ids_again.append(doc_id)
|
|
201
|
+
posns_again.append(posn)
|
|
202
|
+
assert np.array_equal(doc_ids_again, doc_ids)
|
|
203
|
+
assert np.array_equal(posns, posns_again)
|
|
204
|
+
encoded_term_posns[term_id] = encoded
|
|
205
|
+
|
|
206
|
+
return PosnBitArray(encoded_term_posns, range(0, self.max_doc_id + 1))
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def index_range(rng, key):
|
|
210
|
+
if key is None:
|
|
211
|
+
return rng
|
|
212
|
+
if isinstance(rng, np.ndarray):
|
|
213
|
+
return rng[key]
|
|
214
|
+
|
|
215
|
+
if isinstance(key, slice):
|
|
216
|
+
return rng[key]
|
|
217
|
+
elif isinstance(key, numbers.Number):
|
|
218
|
+
return rng[key]
|
|
219
|
+
elif isinstance(key, np.ndarray):
|
|
220
|
+
try:
|
|
221
|
+
# UNFORTUNATE COPY
|
|
222
|
+
r_val = np.asarray(list(rng))[key]
|
|
223
|
+
return r_val
|
|
224
|
+
except IndexError as e:
|
|
225
|
+
raise e
|
|
226
|
+
# Last resort
|
|
227
|
+
# UNFORTUNATE COPY
|
|
228
|
+
# Here probably elipses or a tuple of various things
|
|
229
|
+
return np.asarray(list(rng))[key]
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
class PosnBitArray:
|
|
233
|
+
|
|
234
|
+
def __init__(self, encoded_term_posns, doc_ids):
|
|
235
|
+
self.encoded_term_posns = encoded_term_posns
|
|
236
|
+
self.doc_ids = doc_ids
|
|
237
|
+
|
|
238
|
+
def copy(self):
|
|
239
|
+
new = PosnBitArray(deepcopy(self.encoded_term_posns),
|
|
240
|
+
self.doc_ids)
|
|
241
|
+
return new
|
|
242
|
+
|
|
243
|
+
def slice(self, key):
|
|
244
|
+
sliced_term_posns = {}
|
|
245
|
+
doc_ids = index_range(self.doc_ids, key)
|
|
246
|
+
for term_id, posns in self.encoded_term_posns.items():
|
|
247
|
+
encoded = self.encoded_term_posns[term_id]
|
|
248
|
+
assert len(encoded.shape) == 1
|
|
249
|
+
sliced_term_posns[term_id] = get_docs(encoded, doc_ids=doc_ids)
|
|
250
|
+
|
|
251
|
+
return PosnBitArray(sliced_term_posns, doc_ids=doc_ids)
|
|
252
|
+
|
|
253
|
+
def __getitem__(self, key):
|
|
254
|
+
return self.slice(key)
|
|
255
|
+
|
|
256
|
+
def merge(self, other):
|
|
257
|
+
for term_id, posns in self.encoded_term_posns.items():
|
|
258
|
+
try:
|
|
259
|
+
posns_other = other.encoded_term_posns[term_id]
|
|
260
|
+
self.encoded_term_posns[term_id] = snp.merge(posns, posns_other)
|
|
261
|
+
except KeyError:
|
|
262
|
+
pass
|
|
263
|
+
|
|
264
|
+
def positions(self, term_id: int, key) -> List:
|
|
265
|
+
# Check if key is in doc ids?
|
|
266
|
+
start = perf_counter()
|
|
267
|
+
doc_ids = index_range(self.doc_ids, key)
|
|
268
|
+
logger.debug(f"index_range took {perf_counter() - start} seconds")
|
|
269
|
+
if isinstance(doc_ids, numbers.Number):
|
|
270
|
+
doc_ids = np.asarray([doc_ids])
|
|
271
|
+
try:
|
|
272
|
+
term_posns = get_docs(self.encoded_term_posns[term_id],
|
|
273
|
+
doc_ids=doc_ids)
|
|
274
|
+
logger.debug(f"get_docs took {perf_counter() - start} seconds")
|
|
275
|
+
except KeyError:
|
|
276
|
+
r_val = [np.array([], dtype=np.uint32) for doc_id in doc_ids]
|
|
277
|
+
if len(r_val) == 1 and isinstance(key, numbers.Number):
|
|
278
|
+
r_val = r_val[0]
|
|
279
|
+
logger.debug(f"positions exit(1) took {perf_counter() - start} seconds")
|
|
280
|
+
return r_val
|
|
281
|
+
|
|
282
|
+
decoded = decode_posns(term_posns)
|
|
283
|
+
logger.debug(f"decode took {perf_counter() - start} seconds")
|
|
284
|
+
|
|
285
|
+
if len(decoded) == 0:
|
|
286
|
+
return np.array([], dtype=np.uint32)
|
|
287
|
+
logger.debug(f"positions exit(2) took {perf_counter() - start} seconds")
|
|
288
|
+
if len(decoded) != len(doc_ids):
|
|
289
|
+
# Fill non matches
|
|
290
|
+
as_dict = dict(decoded)
|
|
291
|
+
decs = []
|
|
292
|
+
for doc_id in doc_ids:
|
|
293
|
+
if doc_id in as_dict:
|
|
294
|
+
decs.append(as_dict[doc_id])
|
|
295
|
+
else:
|
|
296
|
+
decs.append(np.array([], dtype=np.uint32))
|
|
297
|
+
logger.debug(f"positions exit(3) took {perf_counter() - start} seconds")
|
|
298
|
+
return decs
|
|
299
|
+
else:
|
|
300
|
+
decs = [dec[1] for dec in decoded]
|
|
301
|
+
if len(decs) == 1 and isinstance(key, numbers.Number):
|
|
302
|
+
decs = decs[0]
|
|
303
|
+
logger.debug(f"positions exit(4) took {perf_counter() - start} seconds")
|
|
304
|
+
return decs
|
|
305
|
+
|
|
306
|
+
def insert(self, key, term_ids_to_posns):
|
|
307
|
+
new_posns = PosnBitArrayBuilder()
|
|
308
|
+
max_doc_id = 0
|
|
309
|
+
for doc_id, new_posns_row in enumerate(term_ids_to_posns):
|
|
310
|
+
for term_id, positions in new_posns_row:
|
|
311
|
+
new_posns.add_posns(doc_id, term_id, positions)
|
|
312
|
+
max_doc_id = max(doc_id, max_doc_id)
|
|
313
|
+
new_posns.max_doc_id = max_doc_id
|
|
314
|
+
ins_arr = new_posns.build()
|
|
315
|
+
self.merge(ins_arr)
|
|
316
|
+
|
|
317
|
+
@property
|
|
318
|
+
def nbytes(self):
|
|
319
|
+
arr_bytes = 0
|
|
320
|
+
for doc_id, posn in self.encoded_term_posns.items():
|
|
321
|
+
arr_bytes += posn.nbytes
|
|
322
|
+
return arr_bytes
|
|
323
|
+
|
|
324
|
+
def phrase_freqs(self, terms: List[int]):
|
|
325
|
+
"""Return the phrase frequencies for a list of terms."""
|
|
326
|
+
raise NotImplementedError("Not yet implemented")
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Query-doc term positions implemented as a padded array."""
|
|
2
|
+
from collections import defaultdict
|
|
3
|
+
from searcharray.utils.row_viewable_matrix import RowViewableMatrix
|
|
4
|
+
from scipy.sparse import dok_matrix
|
|
5
|
+
from time import perf_counter
|
|
6
|
+
import logging
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
# When running in pytest
|
|
12
|
+
import sys # noqa
|
|
13
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
14
|
+
handler.setLevel(logging.INFO)
|
|
15
|
+
formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
|
|
16
|
+
handler.setFormatter(formatter)
|
|
17
|
+
logger.addHandler(handler)
|
|
18
|
+
logger.setLevel(logging.INFO)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _pad_to_len(arr, fill, every=10):
|
|
22
|
+
"""Pad an array to a multiple of every."""
|
|
23
|
+
if len(arr) % every != 0:
|
|
24
|
+
arr = np.pad(arr, (0, every - len(arr) % every),
|
|
25
|
+
'constant', constant_values=fill)
|
|
26
|
+
return arr
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def mat_index_from_width(width, mats):
|
|
30
|
+
"""Return the index of the matrix to use for a given width."""
|
|
31
|
+
if width < len(mats) - 1:
|
|
32
|
+
return width // 10
|
|
33
|
+
else:
|
|
34
|
+
return len(mats) - 1
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class PaddedPosnArraysBuilder():
|
|
38
|
+
def __init__(self, pad=-100):
|
|
39
|
+
# Matrix per 10 positions
|
|
40
|
+
self.posns_mats = [defaultdict(int), defaultdict(int),
|
|
41
|
+
defaultdict(int), defaultdict(int),
|
|
42
|
+
defaultdict(int)]
|
|
43
|
+
|
|
44
|
+
self.posns_lookup = [np.array([]), ] # 0th is invalid
|
|
45
|
+
self.pad = pad
|
|
46
|
+
|
|
47
|
+
def add_posns(self, doc_id, term_id, positions):
|
|
48
|
+
idx = len(self.posns_lookup)
|
|
49
|
+
positions = np.array(positions)
|
|
50
|
+
|
|
51
|
+
# PAD TO LENGTH % 10
|
|
52
|
+
positions = _pad_to_len(positions, self.pad)
|
|
53
|
+
|
|
54
|
+
self.posns_lookup.append(positions)
|
|
55
|
+
mat_to_use = mat_index_from_width(len(positions), self.posns_mats)
|
|
56
|
+
self.posns_mats[mat_to_use][doc_id, term_id] += idx
|
|
57
|
+
|
|
58
|
+
def _build_mat(self, mat_dict, num_postings, num_terms):
|
|
59
|
+
posns_dok = dok_matrix((num_postings, num_terms), dtype=np.uint32)
|
|
60
|
+
dict.update(posns_dok, mat_dict)
|
|
61
|
+
posns_csr = posns_dok.tocsr()
|
|
62
|
+
posns_mat = RowViewableMatrix(posns_csr)
|
|
63
|
+
return posns_mat
|
|
64
|
+
|
|
65
|
+
def build(self, num_postings, num_terms):
|
|
66
|
+
mats = [self._build_mat(mat_dict, num_postings, num_terms)
|
|
67
|
+
for mat_dict in self.posns_mats]
|
|
68
|
+
|
|
69
|
+
padded_posn_arrays = PaddedPosnArrays()
|
|
70
|
+
padded_posn_arrays.posns_mats = mats
|
|
71
|
+
padded_posn_arrays.posns_lookup = self.posns_lookup
|
|
72
|
+
return padded_posn_arrays
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class PaddedPosnArrays():
|
|
76
|
+
|
|
77
|
+
def __init__(self, pad=-100):
|
|
78
|
+
self.posns_mats = None
|
|
79
|
+
self.posns_lookup = [np.array([])] # 0th is invalid
|
|
80
|
+
self.pad = pad
|
|
81
|
+
|
|
82
|
+
def ensure_capacity(self, num_postings, num_terms):
|
|
83
|
+
for mat in self.posns_mats:
|
|
84
|
+
mat.resize((num_postings, num_terms))
|
|
85
|
+
max_lookup = max(mat.mat.max() for mat in self.posns_mats)
|
|
86
|
+
if max_lookup > len(self.posns_lookup):
|
|
87
|
+
self.posns_lookup = np.resize(self.posns_lookup, max_lookup + 1)
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def nbytes(self):
|
|
91
|
+
posns_lookup_bytes = sum(x.nbytes for x in self.posns_lookup)
|
|
92
|
+
posns_mats_bytes = sum(x.nbytes for x in self.posns_mats)
|
|
93
|
+
return posns_mats_bytes + posns_lookup_bytes
|
|
94
|
+
|
|
95
|
+
def insert(self, key, term_ids_to_posns):
|
|
96
|
+
for update_doc_idx, new_posns_row in enumerate(term_ids_to_posns):
|
|
97
|
+
for term_id, positions in new_posns_row:
|
|
98
|
+
mat_idx = mat_index_from_width(len(positions), self.posns_mats)
|
|
99
|
+
update_docs = self.posns_mats[mat_idx][key]
|
|
100
|
+
lookup_location = update_docs[update_doc_idx][0, term_id]
|
|
101
|
+
self.posns_lookup[lookup_location] = positions
|
|
102
|
+
|
|
103
|
+
def positions(self, term_id, key=None, padded=False, width_hint=None):
|
|
104
|
+
|
|
105
|
+
if width_hint is not None:
|
|
106
|
+
mat_to_use = mat_index_from_width(width_hint, self.posns_mats)
|
|
107
|
+
posns_to_lookup = self.posns_mats[mat_to_use].copy_col_at(term_id)
|
|
108
|
+
else:
|
|
109
|
+
for mat in self.posns_mats:
|
|
110
|
+
if key is not None:
|
|
111
|
+
posns_to_lookup = mat.copy_col_at(term_id)[key]
|
|
112
|
+
else:
|
|
113
|
+
posns_to_lookup = self.posns_mat.copy_col_at(term_id)
|
|
114
|
+
if posns_to_lookup.nnz > 0:
|
|
115
|
+
break
|
|
116
|
+
|
|
117
|
+
# This could be faster if posns_lookup was more row slicable
|
|
118
|
+
posns_to_lookup = posns_to_lookup.toarray().flatten()
|
|
119
|
+
if padded:
|
|
120
|
+
posns = [self.posns_lookup[lookup] for lookup in posns_to_lookup]
|
|
121
|
+
else:
|
|
122
|
+
posns = [self.posns_lookup[lookup][self.posns_lookup[lookup] != self.pad] for lookup in posns_to_lookup]
|
|
123
|
+
return posns
|
|
124
|
+
|
|
125
|
+
def copy(self):
|
|
126
|
+
new = PaddedPosnArrays()
|
|
127
|
+
new.posns_mats = [mat.copy() for mat in self.posns_mats]
|
|
128
|
+
new.posns_lookup = self.posns_lookup.copy()
|
|
129
|
+
return new
|
|
130
|
+
|
|
131
|
+
def slice(self, key):
|
|
132
|
+
new = PaddedPosnArrays()
|
|
133
|
+
new.posns_mats = [mat.slice(key) for mat in self.posns_mats]
|
|
134
|
+
new.posns_lookup = self.posns_lookup
|
|
135
|
+
return new
|
|
136
|
+
|
|
137
|
+
def __getitem__(self, key):
|
|
138
|
+
return self.posns_mat[key]
|
|
139
|
+
|
|
140
|
+
def _compute_phrase_freqs(term_posns, slop=1):
|
|
141
|
+
start = perf_counter()
|
|
142
|
+
|
|
143
|
+
prior_term = term_posns[0]
|
|
144
|
+
logger.info(f"Preamble took {perf_counter() - start:.4f} seconds")
|
|
145
|
+
for term in term_posns[1:]:
|
|
146
|
+
is_same_term = (term.shape == prior_term.shape) and np.all(term[0] == prior_term[0])
|
|
147
|
+
|
|
148
|
+
# Compute positional differences
|
|
149
|
+
#
|
|
150
|
+
# Each row of posn_diffs is a term posn diff matrix
|
|
151
|
+
# Where columns are prior_term posns, rows are term posns
|
|
152
|
+
# This shows every possible term diff
|
|
153
|
+
#
|
|
154
|
+
# Example:
|
|
155
|
+
# prior_term = array([[0, 4],[0, 4])
|
|
156
|
+
# term = array([[1, 2, 3],[1, 2, 3]])
|
|
157
|
+
#
|
|
158
|
+
#
|
|
159
|
+
# posn_diffs =
|
|
160
|
+
#
|
|
161
|
+
# array([[ term[0] - prior_term[0], term[0] - prior_term[1] ],
|
|
162
|
+
# [ term[1] - prior_term[0], ...
|
|
163
|
+
# [ term[2] - prior_term[0], ...
|
|
164
|
+
#
|
|
165
|
+
# or in our example
|
|
166
|
+
#
|
|
167
|
+
# array([[ 1, -3],
|
|
168
|
+
# [ 2, -2],
|
|
169
|
+
# [ 3, -1]])
|
|
170
|
+
#
|
|
171
|
+
# We care about all locations where posn == slop (or perhaps <= slop)
|
|
172
|
+
# that is term is slop away from prior_term. Usually slop == 1 (ie 1 posn away)
|
|
173
|
+
# for normal phrase matching
|
|
174
|
+
#
|
|
175
|
+
posn_diffs = term[:, :, np.newaxis] - prior_term[:, np.newaxis, :]
|
|
176
|
+
logger.info(f"posn_diffs took {perf_counter() - start:.4f} seconds")
|
|
177
|
+
|
|
178
|
+
# For > 2 terms, we need to connect a third term by making prior_term = term
|
|
179
|
+
# and repeating
|
|
180
|
+
#
|
|
181
|
+
# BUT
|
|
182
|
+
# we only want those parts of term that are adjacent to prior_term
|
|
183
|
+
# before continuing, so we don't accidentally get a partial phrase
|
|
184
|
+
# so we need to make sure to
|
|
185
|
+
# Pad out any rows in 'term' where posn diff != slop
|
|
186
|
+
# so they're not considered on subsequent iterations
|
|
187
|
+
term_mask = np.any(posn_diffs == 1, axis=2)
|
|
188
|
+
term[~term_mask] = -100
|
|
189
|
+
logger.info(f"term mask took {perf_counter() - start:.4f} seconds")
|
|
190
|
+
|
|
191
|
+
# Count how many times the row term is 1 away from the col term
|
|
192
|
+
per_doc_diffs = np.sum(posn_diffs == slop, axis=1, dtype=np.int8)
|
|
193
|
+
logger.info(f"count slops took {perf_counter() - start:.4f} seconds")
|
|
194
|
+
|
|
195
|
+
# Doc-wise sum to get a 'term freq' for the prior_term - term bigram
|
|
196
|
+
bigram_freqs = np.sum(per_doc_diffs == slop, axis=1)
|
|
197
|
+
if is_same_term:
|
|
198
|
+
satisfies_slop = per_doc_diffs == slop
|
|
199
|
+
consecutive_ones = satisfies_slop[:, 1:] & satisfies_slop[:, :-1]
|
|
200
|
+
consecutive_ones = np.sum(consecutive_ones, axis=1)
|
|
201
|
+
# ceiling divide?
|
|
202
|
+
# Really these show up as
|
|
203
|
+
# 1 1 1 0 1
|
|
204
|
+
# we need to treat the 2nd consecutive 1 as 'not a match'
|
|
205
|
+
# and also update 'term' to not include it
|
|
206
|
+
bigram_freqs -= -np.floor_divide(consecutive_ones, -2)
|
|
207
|
+
logger.info(f"same terms took {perf_counter() - start:.4f} seconds")
|
|
208
|
+
|
|
209
|
+
# Last loop, bigram_freqs is the full phrase term freq
|
|
210
|
+
|
|
211
|
+
# Update mask to eliminate any non-matches
|
|
212
|
+
phrase_freqs[to_compute] = bigram_freqs
|
|
213
|
+
|
|
214
|
+
# Should only keep positions of 'prior term' that are adjacent to the
|
|
215
|
+
# one prior to it...
|
|
216
|
+
prior_term = term
|
|
217
|
+
logger.info(f"Loop took {perf_counter() - start:.4f} seconds")
|
|
218
|
+
|
|
219
|
+
return phrase_freqs
|
|
220
|
+
|
|
221
|
+
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Phrase search by dumb numpy position differences subtraction, bigram by bigram."""
|
|
2
|
+
import numpy as np
|
|
3
|
+
from typing import List
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def vstack_with_mask(posns: List[np.ndarray], phrase_freqs: np.ndarray, width: int = 0, pad: int = -100):
|
|
7
|
+
"""Vertically stack arrays, only accepting those that fit within width.
|
|
8
|
+
|
|
9
|
+
Parameters
|
|
10
|
+
----------
|
|
11
|
+
posns: list of np.ndarray term positions for a given term across multiple docs
|
|
12
|
+
width: int, max number of positions we accept
|
|
13
|
+
pad: int, value to pad with when no position present
|
|
14
|
+
phrase_freqs: np.ndarray, phrase freqs for each doc.
|
|
15
|
+
-1 indicates "not yet processed" 0 means "defiiitely not a match"
|
|
16
|
+
|
|
17
|
+
Returns
|
|
18
|
+
-------
|
|
19
|
+
vstacked: np.ndarray, vertically stacked arrays of those <= width
|
|
20
|
+
mask: np.ndarray, boolean mask of which arrays were accepted (so you can recompute them a different way)
|
|
21
|
+
"""
|
|
22
|
+
vstacked = np.zeros((len(posns), width), dtype=posns[0].dtype) + pad
|
|
23
|
+
if not isinstance(posns[0], np.ndarray):
|
|
24
|
+
raise TypeError("posns must be a list of np.ndarrays")
|
|
25
|
+
for idx, array in enumerate(posns):
|
|
26
|
+
if len(array) >= width:
|
|
27
|
+
phrase_freqs[idx] = -2 # Mark as skip this round
|
|
28
|
+
else:
|
|
29
|
+
vstacked[idx, :len(array)] = array
|
|
30
|
+
return vstacked
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def stack_term_posns(term_posns: List[np.ndarray], phrase_freqs: np.ndarray, width: int = 10):
|
|
34
|
+
# Pad for easy difference computation
|
|
35
|
+
keep_term_posns = []
|
|
36
|
+
# keep_mask = np.ones(len(self), dtype=bool)
|
|
37
|
+
for term_posn in term_posns:
|
|
38
|
+
this_term_posns = vstack_with_mask(term_posn, phrase_freqs, width=width)
|
|
39
|
+
keep_term_posns.append(this_term_posns)
|
|
40
|
+
return keep_term_posns
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _compute_phrase_freqs(term_posns, phrase_freqs: np.ndarray, slop=1):
|
|
44
|
+
|
|
45
|
+
to_compute = phrase_freqs == -1
|
|
46
|
+
|
|
47
|
+
if not np.any(to_compute):
|
|
48
|
+
return phrase_freqs
|
|
49
|
+
|
|
50
|
+
# Only examine masked
|
|
51
|
+
term_posns = [term_posn[to_compute] for term_posn in term_posns]
|
|
52
|
+
|
|
53
|
+
prior_term = term_posns[0]
|
|
54
|
+
for term in term_posns[1:]:
|
|
55
|
+
is_same_term = (term.shape == prior_term.shape) and np.all(term == prior_term)
|
|
56
|
+
|
|
57
|
+
# Compute positional differences
|
|
58
|
+
#
|
|
59
|
+
# Each row of posn_diffs is a term posn diff matrix
|
|
60
|
+
# Where columns are prior_term posns, rows are term posns
|
|
61
|
+
# This shows every possible term diff
|
|
62
|
+
#
|
|
63
|
+
# Example:
|
|
64
|
+
# prior_term = array([[0, 4],[0, 4])
|
|
65
|
+
# term = array([[1, 2, 3],[1, 2, 3]])
|
|
66
|
+
#
|
|
67
|
+
#
|
|
68
|
+
# posn_diffs =
|
|
69
|
+
#
|
|
70
|
+
# array([[ term[0] - prior_term[0], term[0] - prior_term[1] ],
|
|
71
|
+
# [ term[1] - prior_term[0], ...
|
|
72
|
+
# [ term[2] - prior_term[0], ...
|
|
73
|
+
#
|
|
74
|
+
# or in our example
|
|
75
|
+
#
|
|
76
|
+
# array([[ 1, -3],
|
|
77
|
+
# [ 2, -2],
|
|
78
|
+
# [ 3, -1]])
|
|
79
|
+
#
|
|
80
|
+
# We care about all locations where posn == slop (or perhaps <= slop)
|
|
81
|
+
# that is term is slop away from prior_term. Usually slop == 1 (ie 1 posn away)
|
|
82
|
+
# for normal phrase matching
|
|
83
|
+
#
|
|
84
|
+
posn_diffs = term[:, :, np.newaxis] - prior_term[:, np.newaxis, :]
|
|
85
|
+
|
|
86
|
+
# For > 2 terms, we need to connect a third term by making prior_term = term
|
|
87
|
+
# and repeating
|
|
88
|
+
#
|
|
89
|
+
# BUT
|
|
90
|
+
# we only want those parts of term that are adjacent to prior_term
|
|
91
|
+
# before continuing, so we don't accidentally get a partial phrase
|
|
92
|
+
# so we need to make sure to
|
|
93
|
+
# Pad out any rows in 'term' where posn diff != slop
|
|
94
|
+
# so they're not considered on subsequent iterations
|
|
95
|
+
term_mask = np.any(posn_diffs == 1, axis=2)
|
|
96
|
+
term[~term_mask] = -100
|
|
97
|
+
|
|
98
|
+
# Count how many times the row term is 1 away from the col term
|
|
99
|
+
per_doc_diffs = np.sum(posn_diffs == slop, axis=1, dtype=np.int8)
|
|
100
|
+
|
|
101
|
+
# Doc-wise sum to get a 'term freq' for the prior_term - term bigram
|
|
102
|
+
bigram_freqs = np.sum(per_doc_diffs == slop, axis=1)
|
|
103
|
+
if is_same_term:
|
|
104
|
+
satisfies_slop = per_doc_diffs == slop
|
|
105
|
+
consecutive_ones = satisfies_slop[:, 1:] & satisfies_slop[:, :-1]
|
|
106
|
+
consecutive_ones = np.sum(consecutive_ones, axis=1)
|
|
107
|
+
# ceiling divide?
|
|
108
|
+
# Really these show up as
|
|
109
|
+
# 1 1 1 0 1
|
|
110
|
+
# we need to treat the 2nd consecutive 1 as 'not a match'
|
|
111
|
+
# and also update 'term' to not include it
|
|
112
|
+
bigram_freqs -= -np.floor_divide(consecutive_ones, -2)
|
|
113
|
+
|
|
114
|
+
# Last loop, bigram_freqs is the full phrase term freq
|
|
115
|
+
|
|
116
|
+
# Update mask to eliminate any non-matches
|
|
117
|
+
phrase_freqs[to_compute] = bigram_freqs
|
|
118
|
+
|
|
119
|
+
# Should only keep positions of 'prior term' that are adjacent to the
|
|
120
|
+
# one prior to it...
|
|
121
|
+
prior_term = term
|
|
122
|
+
|
|
123
|
+
return phrase_freqs
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def compute_phrase_freqs(term_posns, phrase_freqs, slop=1, width=10):
|
|
127
|
+
"""Compute phrase freq using matrix-diff method for docs up to width posns. Skip others.
|
|
128
|
+
|
|
129
|
+
Parameters
|
|
130
|
+
----------
|
|
131
|
+
term_posns: list of np.ndarray term positions for a given term across multiple docs
|
|
132
|
+
phrase_freqs: np.ndarray, phrase freqs for each doc present in term_posns
|
|
133
|
+
|
|
134
|
+
Returns
|
|
135
|
+
-------
|
|
136
|
+
phrase_freqs: np.ndarray, phrase freqs for each doc present in mask
|
|
137
|
+
|
|
138
|
+
See Also
|
|
139
|
+
--------
|
|
140
|
+
Colab notebook: https://colab.research.google.com/drive/1NRxeO8Ya8jSlFP5YwZaGh1-43kDH4OXG?authuser=1#scrollTo=5JZV8svpauYB
|
|
141
|
+
"""
|
|
142
|
+
if len(term_posns[0]) != len(phrase_freqs):
|
|
143
|
+
raise ValueError("term_posns and phrase_freqs must be same length")
|
|
144
|
+
stacked = stack_term_posns(term_posns, phrase_freqs, width=width)
|
|
145
|
+
phrase_freqs = _compute_phrase_freqs(stacked, phrase_freqs, slop=slop)
|
|
146
|
+
phrase_freqs[phrase_freqs == -2] = -1
|
|
147
|
+
return phrase_freqs
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from typing import List
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def advance_to_scan(arr, target, next_start, idx):
|
|
6
|
+
while idx < next_start and not arr[idx] > target:
|
|
7
|
+
idx += 1
|
|
8
|
+
return idx
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def advance_after_binsearch(arr, target, next_start, idx):
|
|
12
|
+
"""Scan arr from idx until just after target. Return next_start if exhausted.
|
|
13
|
+
|
|
14
|
+
Will manually create a skip index if next_start - idx is large
|
|
15
|
+
|
|
16
|
+
arr from idx -> next_start should be sorted
|
|
17
|
+
|
|
18
|
+
"""
|
|
19
|
+
beg = idx
|
|
20
|
+
end = next_start - 1
|
|
21
|
+
space = end - beg
|
|
22
|
+
if space < 1000:
|
|
23
|
+
advance_to_scan(arr, target, next_start, idx)
|
|
24
|
+
|
|
25
|
+
# print(f"Binsearching {target} in arr[{beg}:{end}] = {arr[beg:end]}")
|
|
26
|
+
if arr[idx] > target:
|
|
27
|
+
# print("Already past, returning idx")
|
|
28
|
+
return idx
|
|
29
|
+
|
|
30
|
+
mid = None
|
|
31
|
+
while True:
|
|
32
|
+
space = end - beg
|
|
33
|
+
if space <= 1000:
|
|
34
|
+
return advance_to_scan(arr, target, next_start, beg)
|
|
35
|
+
|
|
36
|
+
mid = beg + (space // 2)
|
|
37
|
+
if arr[mid] == target:
|
|
38
|
+
# print(f"Found exact match, returning {mid + 1}")
|
|
39
|
+
return mid + 1
|
|
40
|
+
elif arr[mid] > target and arr[mid - 1] <= target:
|
|
41
|
+
# print(f"Found one past, returning {mid}")
|
|
42
|
+
return mid
|
|
43
|
+
elif arr[mid] < target:
|
|
44
|
+
# print(f"Moving beg to {mid}")
|
|
45
|
+
# print(f" end is {end}")
|
|
46
|
+
# print(f"arr[{mid}] is {arr[mid]}")
|
|
47
|
+
beg = mid
|
|
48
|
+
else: # elif arr[mid] > target:
|
|
49
|
+
# print(f" beg is {beg}")
|
|
50
|
+
# print(f"Moving end to {mid}")
|
|
51
|
+
# print(f"arr[{mid}] is {arr[mid]}")
|
|
52
|
+
end = mid
|
|
53
|
+
|
|
54
|
+
# print(f"Exhausted binsearch, returning {next_start}")
|
|
55
|
+
return next_start
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def scan_merge(prior_posns: np.ndarray,
|
|
59
|
+
prior_starts: np.ndarray,
|
|
60
|
+
next_posns: np.ndarray,
|
|
61
|
+
next_starts: np.ndarray,
|
|
62
|
+
scan_algo=advance_to_scan,
|
|
63
|
+
slop=1):
|
|
64
|
+
"""Merge two term position lists together into a single list of bigrams.
|
|
65
|
+
|
|
66
|
+
Each position list is a flattened representation of multiple docs, ie
|
|
67
|
+
|
|
68
|
+
prior_posns: [0,2,1,4,5]
|
|
69
|
+
prior_starts: [0,2]
|
|
70
|
+
|
|
71
|
+
Points to the first position of each doc in the prior_posns list for this term
|
|
72
|
+
|
|
73
|
+
(same for next_posns)
|
|
74
|
+
|
|
75
|
+
This is intentionally written to be naive, C-like for later porting to C
|
|
76
|
+
|
|
77
|
+
See notebook:
|
|
78
|
+
https://colab.research.google.com/drive/10zjUYHGtwMfJMPXz-BHwHe_v6j4MUZzm?authuser=1#scrollTo=W6HuiFGaYCiX
|
|
79
|
+
|
|
80
|
+
"""
|
|
81
|
+
next_idx = 0
|
|
82
|
+
cont_nexts = []
|
|
83
|
+
cont_next = []
|
|
84
|
+
bigram_freq = 0
|
|
85
|
+
bigram_freqs = []
|
|
86
|
+
last_prior = -2
|
|
87
|
+
next_start = next_starts[0]
|
|
88
|
+
prior_start = prior_starts[0]
|
|
89
|
+
start_idx = 0
|
|
90
|
+
prior_idx = 0
|
|
91
|
+
while prior_idx < len(prior_posns):
|
|
92
|
+
# Scan next until just past prior
|
|
93
|
+
prior = prior_posns[prior_idx]
|
|
94
|
+
next_idx = scan_algo(next_posns, target=prior,
|
|
95
|
+
next_start=next_start,
|
|
96
|
+
idx=next_idx)
|
|
97
|
+
|
|
98
|
+
# Reset to head of next location,
|
|
99
|
+
# Re-advance next
|
|
100
|
+
if next_idx >= next_start or prior_idx >= prior_start:
|
|
101
|
+
next_idx = next_start
|
|
102
|
+
prior_idx = prior_start
|
|
103
|
+
last_prior = -2
|
|
104
|
+
|
|
105
|
+
start_idx += 1
|
|
106
|
+
if start_idx >= len(next_starts):
|
|
107
|
+
break
|
|
108
|
+
next_start = next_starts[start_idx]
|
|
109
|
+
prior_start = prior_starts[start_idx]
|
|
110
|
+
|
|
111
|
+
# Save and reset output
|
|
112
|
+
cont_nexts.append(np.array(cont_next))
|
|
113
|
+
bigram_freqs.append(bigram_freq)
|
|
114
|
+
|
|
115
|
+
cont_next = []
|
|
116
|
+
bigram_freq = 0
|
|
117
|
+
|
|
118
|
+
prior = prior_posns[prior_idx]
|
|
119
|
+
next_idx = scan_algo(next_posns, target=prior,
|
|
120
|
+
next_start=next_start, idx=next_idx)
|
|
121
|
+
|
|
122
|
+
if next_idx >= next_start:
|
|
123
|
+
continue
|
|
124
|
+
|
|
125
|
+
next_posn = next_posns[next_idx]
|
|
126
|
+
# Check if within slop
|
|
127
|
+
# And is not double counting 0->1->2 (only happens if prior/next identical)
|
|
128
|
+
dist = next_posn - prior
|
|
129
|
+
if dist <= slop and prior != (last_prior + 1):
|
|
130
|
+
cont_next.append(next_posn)
|
|
131
|
+
bigram_freq += 1
|
|
132
|
+
last_prior = prior
|
|
133
|
+
|
|
134
|
+
prior_idx += 1
|
|
135
|
+
|
|
136
|
+
# Save last output
|
|
137
|
+
cont_nexts.append(np.array(cont_next))
|
|
138
|
+
bigram_freqs.append(bigram_freq)
|
|
139
|
+
return bigram_freqs, cont_nexts
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def scan_merge_inplace(prior_posns: np.ndarray,
|
|
143
|
+
prior_starts: np.ndarray,
|
|
144
|
+
next_posns: np.ndarray,
|
|
145
|
+
next_starts: np.ndarray,
|
|
146
|
+
scan_algo=advance_to_scan,
|
|
147
|
+
slop=1):
|
|
148
|
+
"""Merge two term position lists together into a single list of bigrams.
|
|
149
|
+
|
|
150
|
+
Same as scan_merge, but doesn't need a dynamic array output, instead
|
|
151
|
+
overwrites prior* buffers
|
|
152
|
+
"""
|
|
153
|
+
next_idx = 0
|
|
154
|
+
bigram_freq = 0
|
|
155
|
+
bigram_freqs = [] # this could also be preallocated array output
|
|
156
|
+
last_prior = -2
|
|
157
|
+
output_idx = 0
|
|
158
|
+
next_start = next_starts[0]
|
|
159
|
+
prior_start = prior_starts[0]
|
|
160
|
+
start_idx = 0
|
|
161
|
+
prior_idx = 0
|
|
162
|
+
|
|
163
|
+
while prior_idx < prior_starts[-1]:
|
|
164
|
+
# Scan next until just past p
|
|
165
|
+
prior = prior_posns[prior_idx]
|
|
166
|
+
next_idx = scan_algo(next_posns,
|
|
167
|
+
target=prior,
|
|
168
|
+
next_start=next_start,
|
|
169
|
+
idx=next_idx)
|
|
170
|
+
|
|
171
|
+
# Reset to head of next location,
|
|
172
|
+
# Re-advance next
|
|
173
|
+
if next_idx >= next_start or prior_idx >= prior_start:
|
|
174
|
+
next_idx = next_start
|
|
175
|
+
prior_idx = prior_start
|
|
176
|
+
last_prior = -2
|
|
177
|
+
|
|
178
|
+
prior_starts[start_idx] = output_idx
|
|
179
|
+
|
|
180
|
+
start_idx += 1
|
|
181
|
+
if start_idx >= len(next_starts):
|
|
182
|
+
break
|
|
183
|
+
next_start = next_starts[start_idx]
|
|
184
|
+
prior_start = prior_starts[start_idx]
|
|
185
|
+
|
|
186
|
+
# Save and reset output
|
|
187
|
+
bigram_freqs.append(bigram_freq)
|
|
188
|
+
|
|
189
|
+
bigram_freq = 0
|
|
190
|
+
|
|
191
|
+
prior = prior_posns[prior_idx]
|
|
192
|
+
next_idx = scan_algo(next_posns,
|
|
193
|
+
target=prior,
|
|
194
|
+
next_start=next_start,
|
|
195
|
+
idx=next_idx)
|
|
196
|
+
if next_idx >= next_start:
|
|
197
|
+
continue
|
|
198
|
+
|
|
199
|
+
next_posn = next_posns[next_idx]
|
|
200
|
+
# Check if within slop
|
|
201
|
+
# And is not double counting 0->1->2 (only happens if prior/next identical)
|
|
202
|
+
dist = next_posn - prior
|
|
203
|
+
if dist <= slop and prior != (last_prior + 1):
|
|
204
|
+
prior_posns[output_idx] = next_posn
|
|
205
|
+
output_idx += 1
|
|
206
|
+
bigram_freq += 1
|
|
207
|
+
last_prior = prior
|
|
208
|
+
|
|
209
|
+
prior_idx += 1
|
|
210
|
+
|
|
211
|
+
# Save last output
|
|
212
|
+
bigram_freqs.append(bigram_freq)
|
|
213
|
+
if start_idx < len(next_starts):
|
|
214
|
+
prior_starts[start_idx] = output_idx
|
|
215
|
+
|
|
216
|
+
return bigram_freqs, prior_posns, prior_starts
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def scan_merge_bigram(prior_posns: List, next_posns: List, slop=1):
|
|
220
|
+
prior_starts = np.cumsum([lst.shape[0] for lst in prior_posns])
|
|
221
|
+
next_starts = np.cumsum([lst.shape[0] for lst in next_posns])
|
|
222
|
+
prior_posns = np.concatenate(prior_posns)
|
|
223
|
+
next_posns = np.concatenate(next_posns)
|
|
224
|
+
|
|
225
|
+
result = scan_merge(
|
|
226
|
+
prior_posns, prior_starts,
|
|
227
|
+
next_posns, next_starts,
|
|
228
|
+
slop=slop
|
|
229
|
+
)
|
|
230
|
+
return result
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""Phrase search as wide span detection."""
|
|
2
|
+
import numpy as np
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def advance_after(arr, target, idx, next_start):
|
|
6
|
+
"""Scan arr from idx until just after target. Return next_start if exhausted.
|
|
7
|
+
|
|
8
|
+
arr from idx -> next_start should be sorted
|
|
9
|
+
|
|
10
|
+
"""
|
|
11
|
+
while idx < next_start and not arr[idx] > target:
|
|
12
|
+
idx += 1
|
|
13
|
+
return idx
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def next_wide_span(posns, starts, start_idx, idxs):
|
|
17
|
+
"""Given N terms find possible acceptable phrases for a slop.
|
|
18
|
+
|
|
19
|
+
Parameters:
|
|
20
|
+
-----------
|
|
21
|
+
posns - N Numpy arrays, holding all matching document term positions
|
|
22
|
+
starts - each term N start boundaries, for all D docs
|
|
23
|
+
start_idx - where we are in the starts (aka doc positon)
|
|
24
|
+
idxs - where we are in each term
|
|
25
|
+
|
|
26
|
+
A wide span begins with the first term in a phrase, and finds the
|
|
27
|
+
first location after that where the last term occurs. Acceptable
|
|
28
|
+
wide spans are
|
|
29
|
+
|
|
30
|
+
Query - "foo bar baz"
|
|
31
|
+
posn 1 2 3 4
|
|
32
|
+
t1 foo foo
|
|
33
|
+
t2 bar
|
|
34
|
+
t3 baz
|
|
35
|
+
|
|
36
|
+
posn 1 2 3 ... N -1 N
|
|
37
|
+
t1 foo foo
|
|
38
|
+
t2 bar bar
|
|
39
|
+
t3 baz
|
|
40
|
+
|
|
41
|
+
To confirm these wide spans are acceptable phrase matches, we have
|
|
42
|
+
to walk backwards from the end and see if phrases occur in order within span
|
|
43
|
+
|
|
44
|
+
After consuming this wide span, the next phrase candidate can be had by advancing
|
|
45
|
+
T1 to N+1th position.
|
|
46
|
+
"""
|
|
47
|
+
# 30100000 20.651 0.000 33.100 0.000 wide_spans.py:16(next_wide_span)
|
|
48
|
+
# in-place
|
|
49
|
+
# 30100000 18.470 0.000 29.952 0.000 wide_spans.py:16(next_wide_span)
|
|
50
|
+
# 30100000 18.078 0.000 29.297 0.000 wide_spans.py:16(next_wide_span)
|
|
51
|
+
|
|
52
|
+
span_idx = idxs[0]
|
|
53
|
+
target = posns[0][span_idx]
|
|
54
|
+
|
|
55
|
+
term_idx = 1
|
|
56
|
+
for term in posns[1:]:
|
|
57
|
+
next_start = starts[term_idx][start_idx]
|
|
58
|
+
span_idx = advance_after(posns[term_idx],
|
|
59
|
+
target=target,
|
|
60
|
+
idx=idxs[term_idx],
|
|
61
|
+
next_start=next_start)
|
|
62
|
+
if span_idx == next_start:
|
|
63
|
+
return [start[start_idx] for start in starts]
|
|
64
|
+
|
|
65
|
+
target = posns[term_idx][span_idx]
|
|
66
|
+
idxs[term_idx] = span_idx
|
|
67
|
+
term_idx += 1
|
|
68
|
+
|
|
69
|
+
return idxs
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def get_back_span(posns, starts, start_idx, idxs):
|
|
73
|
+
"""Given N terms find minimal back span.
|
|
74
|
+
|
|
75
|
+
Parameters:
|
|
76
|
+
----------
|
|
77
|
+
posns - N Numpy arrays, holding all matching document term positions
|
|
78
|
+
starts - each term N start boundaries, for all D docs
|
|
79
|
+
start_idx - where we are in the starts (aka doc positon)
|
|
80
|
+
idxs - where we are in each term
|
|
81
|
+
|
|
82
|
+
Returns:
|
|
83
|
+
--------
|
|
84
|
+
Shortest span backwards, plus positional diff
|
|
85
|
+
|
|
86
|
+
A back span for 'foo bar baz' of a wide span, starts with end position
|
|
87
|
+
then goes backwards
|
|
88
|
+
|
|
89
|
+
Query - "foo bar baz"
|
|
90
|
+
posn 1 2 3 4
|
|
91
|
+
t1 foo* foo
|
|
92
|
+
t2 bar*
|
|
93
|
+
t3 baz*
|
|
94
|
+
|
|
95
|
+
posn 1 2 3 ... N -1 N
|
|
96
|
+
t1 foo* foo <- scan here
|
|
97
|
+
t2 bar* bar <- scan here
|
|
98
|
+
t3 baz
|
|
99
|
+
|
|
100
|
+
Walking backwards at idxs[-1] we scan idxs[-2] to find
|
|
101
|
+
the first occurence (going backwards)
|
|
102
|
+
"""
|
|
103
|
+
last_posn = posns[-1][idxs[-1]]
|
|
104
|
+
back_span_idxs = [idxs[-1]]
|
|
105
|
+
|
|
106
|
+
term_idx = len(posns) - 2
|
|
107
|
+
for posn in posns[::-1][1:]:
|
|
108
|
+
curr_posn = posn[idxs[term_idx]]
|
|
109
|
+
while curr_posn < last_posn:
|
|
110
|
+
idxs[term_idx] += 1
|
|
111
|
+
if idxs[term_idx] >= starts[term_idx][start_idx]:
|
|
112
|
+
break
|
|
113
|
+
curr_posn = posn[idxs[term_idx]]
|
|
114
|
+
back_span_idxs.append(idxs[term_idx] - 1)
|
|
115
|
+
last_posn = posn[back_span_idxs[-1]]
|
|
116
|
+
term_idx -= 1
|
|
117
|
+
back_span_idxs = back_span_idxs[::-1]
|
|
118
|
+
return back_span_idxs, (posns[-1][back_span_idxs[-1]] - posns[0][back_span_idxs[0]])
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def collect_span(posns, starts, start_idx, idxs, acceptable):
|
|
122
|
+
# Advance 0th beyond
|
|
123
|
+
beg_posn = posns[0][idxs[0]]
|
|
124
|
+
last_end_posn = posns[-1][idxs[-1]]
|
|
125
|
+
if (last_end_posn - beg_posn) <= acceptable:
|
|
126
|
+
return 1
|
|
127
|
+
else:
|
|
128
|
+
back_span, posn_diff = get_back_span(posns, starts, start_idx, idxs)
|
|
129
|
+
if posn_diff <= acceptable:
|
|
130
|
+
return 1
|
|
131
|
+
return 0
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def all_wide_spans_of_slop(posns, starts, slop=1):
|
|
135
|
+
"""Collect all wide spans of multiple docs for all posn terms.
|
|
136
|
+
|
|
137
|
+
Parameters:
|
|
138
|
+
-----------
|
|
139
|
+
posns - list of single concatenated array of all doc posns for each term
|
|
140
|
+
starts - doc start posn boundaries per term
|
|
141
|
+
slop - allowed (in order) moves of term
|
|
142
|
+
|
|
143
|
+
Returns:
|
|
144
|
+
--------
|
|
145
|
+
phrase frequency of terms in posns satisfying given slop
|
|
146
|
+
|
|
147
|
+
"""
|
|
148
|
+
start_idx = 0
|
|
149
|
+
span_idxs = np.zeros(len(starts), dtype=np.uint32)
|
|
150
|
+
phrases_per_doc = np.zeros(len(starts[0]), dtype=np.uint32)
|
|
151
|
+
# Acceptable posn difference (<=)
|
|
152
|
+
# 2-gram 3-gram
|
|
153
|
+
# slop
|
|
154
|
+
#
|
|
155
|
+
# 1 1 2
|
|
156
|
+
# 2 2 3
|
|
157
|
+
acceptable = len(posns) + (slop - 2)
|
|
158
|
+
first_term_start = starts[0]
|
|
159
|
+
num_starts = len(starts[0])
|
|
160
|
+
while True:
|
|
161
|
+
if start_idx >= num_starts:
|
|
162
|
+
break
|
|
163
|
+
elif span_idxs[0] == first_term_start[start_idx]:
|
|
164
|
+
span_idxs = [start[start_idx] for start in starts]
|
|
165
|
+
start_idx += 1
|
|
166
|
+
else:
|
|
167
|
+
span_idxs = next_wide_span(posns, starts, start_idx, span_idxs)
|
|
168
|
+
if span_idxs[0] == first_term_start[start_idx]:
|
|
169
|
+
# New doc / next start
|
|
170
|
+
continue
|
|
171
|
+
|
|
172
|
+
last_end_posn = posns[-1][span_idxs[-1]]
|
|
173
|
+
phrases_per_doc[start_idx] += collect_span(posns, starts, start_idx, span_idxs, acceptable)
|
|
174
|
+
|
|
175
|
+
span_idxs[0] = advance_after(posns[0],
|
|
176
|
+
target=last_end_posn,
|
|
177
|
+
idx=span_idxs[0],
|
|
178
|
+
next_start=first_term_start[start_idx])
|
|
179
|
+
|
|
180
|
+
if span_idxs[0] == first_term_start[-1]:
|
|
181
|
+
break
|
|
182
|
+
|
|
183
|
+
return phrases_per_doc
|
|
File without changes
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import numbers
|
|
2
|
+
from scipy.sparse import csr_matrix
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class RowViewableMatrix:
|
|
7
|
+
"""A slicable matrix that can return views without copying."""
|
|
8
|
+
|
|
9
|
+
def __init__(self, csr_mat: csr_matrix, rows: np.ndarray = None):
|
|
10
|
+
self.mat = csr_mat
|
|
11
|
+
self.col_cache = {}
|
|
12
|
+
self.cols_cached = []
|
|
13
|
+
if rows is None:
|
|
14
|
+
self.rows = np.arange(self.mat.shape[0])
|
|
15
|
+
elif isinstance(rows, numbers.Integral):
|
|
16
|
+
self.rows = np.array([rows])
|
|
17
|
+
else:
|
|
18
|
+
self.rows = rows
|
|
19
|
+
|
|
20
|
+
def slice(self, keys):
|
|
21
|
+
return RowViewableMatrix(self.mat, self.rows[keys])
|
|
22
|
+
|
|
23
|
+
def __setitem__(self, keys, values):
|
|
24
|
+
# Replace nan with 0
|
|
25
|
+
self.col_cache = {}
|
|
26
|
+
self.cols_cached = []
|
|
27
|
+
actual_keys = self.rows[keys]
|
|
28
|
+
if isinstance(actual_keys, numbers.Integral):
|
|
29
|
+
self.mat[actual_keys] = values
|
|
30
|
+
elif len(actual_keys) > 0:
|
|
31
|
+
self.mat[actual_keys] = values
|
|
32
|
+
|
|
33
|
+
def copy_row_at(self, row):
|
|
34
|
+
return self.mat[self.rows[row]]
|
|
35
|
+
|
|
36
|
+
def copy(self):
|
|
37
|
+
return RowViewableMatrix(self.mat.copy(), self.rows.copy())
|
|
38
|
+
|
|
39
|
+
def sum(self, axis=0):
|
|
40
|
+
return self.mat[self.rows].sum(axis=axis)
|
|
41
|
+
|
|
42
|
+
def copy_col_at(self, col):
|
|
43
|
+
if col not in self.col_cache:
|
|
44
|
+
self.col_cache[col] = self.mat[self.rows, col]
|
|
45
|
+
self.cols_cached.append(col)
|
|
46
|
+
if len(self.cols_cached) > 10:
|
|
47
|
+
del self.col_cache[self.cols_cached.pop(0)]
|
|
48
|
+
return self.col_cache[col]
|
|
49
|
+
|
|
50
|
+
def __getitem__(self, key):
|
|
51
|
+
if isinstance(key, numbers.Integral):
|
|
52
|
+
return self.copy_row_at(key)
|
|
53
|
+
else:
|
|
54
|
+
return self.slice(key)
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def nbytes(self):
|
|
58
|
+
return self.mat.data.nbytes + \
|
|
59
|
+
self.mat.indptr.nbytes + \
|
|
60
|
+
self.mat.indices.nbytes + \
|
|
61
|
+
self.rows.nbytes
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def shape(self):
|
|
65
|
+
return (len(self.rows), self.mat.shape[1])
|
|
66
|
+
|
|
67
|
+
def resize(self, shape):
|
|
68
|
+
self.mat.resize(shape)
|
|
69
|
+
|
|
70
|
+
def __len__(self):
|
|
71
|
+
return len(self.rows)
|
|
72
|
+
|
|
73
|
+
def __repr__(self):
|
|
74
|
+
return f"RowViewableMatrix({repr(self.mat)}, {repr(self.rows)})"
|
|
75
|
+
|
|
76
|
+
def __str__(self):
|
|
77
|
+
return f"RowViewableMatrix({str(self.mat)}, {str(self.rows)})"
|
|
@@ -12,6 +12,14 @@ searcharray.egg-info/dependency_links.txt
|
|
|
12
12
|
searcharray.egg-info/entry_points.txt
|
|
13
13
|
searcharray.egg-info/requires.txt
|
|
14
14
|
searcharray.egg-info/top_level.txt
|
|
15
|
+
searcharray/phrase/__init__.py
|
|
16
|
+
searcharray/phrase/middle_out.py
|
|
17
|
+
searcharray/phrase/padded_arrays.py
|
|
18
|
+
searcharray/phrase/posn_diffs.py
|
|
19
|
+
searcharray/phrase/scan_merge.py
|
|
20
|
+
searcharray/phrase/wide_spans.py
|
|
21
|
+
searcharray/utils/__init__.py
|
|
22
|
+
searcharray/utils/row_viewable_matrix.py
|
|
15
23
|
test/test_extension_array.py
|
|
16
24
|
test/test_msmarco.py
|
|
17
25
|
test/test_phrase_matches.py
|
|
@@ -6,7 +6,7 @@ https://github.com/pypa/sampleproject
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
# Always prefer setuptools over distutils
|
|
9
|
-
from setuptools import setup
|
|
9
|
+
from setuptools import setup, find_packages
|
|
10
10
|
import pathlib
|
|
11
11
|
|
|
12
12
|
here = pathlib.Path(__file__).parent.resolve()
|
|
@@ -36,7 +36,7 @@ setup(
|
|
|
36
36
|
# For a discussion on single-sourcing the version across setup.py and the
|
|
37
37
|
# project code, see
|
|
38
38
|
# https://packaging.python.org/guides/single-sourcing-package-version/
|
|
39
|
-
version="0.0.
|
|
39
|
+
version="0.0.14", # Required
|
|
40
40
|
# This is a one-line description or tagline of what your project does. This
|
|
41
41
|
# corresponds to the "Summary" metadata field:
|
|
42
42
|
# https://packaging.python.org/specifications/core-metadata/#summary
|
|
@@ -114,7 +114,7 @@ setup(
|
|
|
114
114
|
#
|
|
115
115
|
# py_modules=["my_module"],
|
|
116
116
|
#
|
|
117
|
-
packages=
|
|
117
|
+
packages=find_packages(),
|
|
118
118
|
# Specify which Python versions you support. In contrast to the
|
|
119
119
|
# 'Programming Language' classifiers above, 'pip install' will check this
|
|
120
120
|
# and refuse to install the project if the version does not match. See
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|