searcharray 0.0.54__tar.gz → 0.0.56__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.54 → searcharray-0.0.56}/PKG-INFO +1 -1
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/indexing.py +83 -28
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/phrase/bigram_freqs.py +41 -30
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/phrase/middle_out.py +62 -12
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/postings.py +32 -82
- searcharray-0.0.56/searcharray/roaringish/__init__.py +7 -0
- searcharray-0.0.54/searcharray/roaringish/snp_ops.c → searcharray-0.0.56/searcharray/roaringish/intersect.c +4203 -10360
- searcharray-0.0.56/searcharray/roaringish/intersect.pyi +25 -0
- searcharray-0.0.56/searcharray/roaringish/intersect.pyx +395 -0
- searcharray-0.0.56/searcharray/roaringish/merge.c +31524 -0
- searcharray-0.0.56/searcharray/roaringish/merge.pyi +8 -0
- searcharray-0.0.56/searcharray/roaringish/merge.pyx +160 -0
- searcharray-0.0.56/searcharray/roaringish/popcount.c +31835 -0
- searcharray-0.0.56/searcharray/roaringish/popcount.pyi +20 -0
- searcharray-0.0.56/searcharray/roaringish/popcount.pyx +113 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/roaringish.py +34 -3
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/roaringish_ops.c +300 -1076
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/roaringish_ops.pyx +1 -33
- searcharray-0.0.56/searcharray/roaringish/search.c +31761 -0
- searcharray-0.0.56/searcharray/roaringish/search.pyi +25 -0
- searcharray-0.0.56/searcharray/roaringish/search.pyx +141 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/snp_ops.pxd +0 -10
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/spans.c +242 -370
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/roaringish/spans.pyx +7 -1
- searcharray-0.0.56/searcharray/roaringish/unique.c +31772 -0
- searcharray-0.0.56/searcharray/roaringish/unique.pyi +6 -0
- searcharray-0.0.56/searcharray/roaringish/unique.pyx +147 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/similarity.py +15 -6
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/solr.py +86 -42
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/utils/mat_set.py +11 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/SOURCES.txt +15 -4
- {searcharray-0.0.54 → searcharray-0.0.56}/setup.py +3 -3
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_minmax_posns.py +1 -5
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_msmarco.py +105 -25
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_phrase_matches.py +76 -19
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_search.py +9 -6
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_snp_ops.py +65 -3
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_tmdb.py +77 -4
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_utils.py +12 -2
- searcharray-0.0.54/searcharray/phrase/posn_diffs.py +0 -147
- searcharray-0.0.54/searcharray/phrase/scan_merge.py +0 -84
- searcharray-0.0.54/searcharray/roaringish/__init__.py +0 -4
- searcharray-0.0.54/searcharray/roaringish/snp_ops.pyx +0 -664
- {searcharray-0.0.54 → searcharray-0.0.56}/README.md +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/pyproject.toml +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/__init__.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/phrase/spans.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray/utils/sort.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/setup.cfg +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_bitcount64.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_extension_array.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_similarity.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_slop_matches.py +0 -0
- {searcharray-0.0.54 → searcharray-0.0.56}/test/test_solr.py +0 -0
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import numpy as np
|
|
2
2
|
import math
|
|
3
3
|
import gc
|
|
4
|
-
|
|
4
|
+
import sys
|
|
5
|
+
from typing import Iterable, List
|
|
6
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
5
7
|
from itertools import islice
|
|
6
8
|
from searcharray.phrase.middle_out import MAX_POSN, PosnBitArrayFromFlatBuilder, PosnBitArrayBuilder, PosnBitArrayAlreadyEncBuilder
|
|
7
9
|
from searcharray.term_dict import TermDict
|
|
@@ -10,6 +12,15 @@ from searcharray.utils.row_viewable_matrix import RowViewableMatrix
|
|
|
10
12
|
|
|
11
13
|
import logging
|
|
12
14
|
logger = logging.getLogger(__name__)
|
|
15
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
16
|
+
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
17
|
+
logger.setLevel(logging.INFO)
|
|
18
|
+
handler.setFormatter(formatter)
|
|
19
|
+
logger.addHandler(handler)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# Set to stdout for debugging
|
|
23
|
+
# logging.basicConfig(level=logging.DEBUG)
|
|
13
24
|
|
|
14
25
|
|
|
15
26
|
def _compute_doc_lens(posns: np.ndarray, doc_ids: np.ndarray, num_docs: int) -> np.ndarray:
|
|
@@ -71,7 +82,7 @@ def _gather_tokens(array, tokenizer, term_dict, term_doc, start_doc_id=0, trunc_
|
|
|
71
82
|
del all_terms, all_docs, all_posns
|
|
72
83
|
gc.collect()
|
|
73
84
|
logger.info("Tokenization -- DONE")
|
|
74
|
-
return terms_w_posns,
|
|
85
|
+
return terms_w_posns, term_doc
|
|
75
86
|
|
|
76
87
|
|
|
77
88
|
def _lex_sort(terms_w_posns):
|
|
@@ -90,14 +101,16 @@ def _invert_docs_terms(terms_w_posns):
|
|
|
90
101
|
return terms_w_posns[:, lexsort]
|
|
91
102
|
|
|
92
103
|
|
|
93
|
-
def _tokenize_batch(array, tokenizer, term_dict,
|
|
104
|
+
def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate=False):
|
|
105
|
+
logger.info(f"{batch_beg} Batch Start tokenization")
|
|
94
106
|
trunc_posn = None
|
|
95
107
|
if truncate:
|
|
96
108
|
trunc_posn = MAX_POSN
|
|
97
109
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
110
|
+
term_doc = SparseMatSetBuilder()
|
|
111
|
+
terms_w_posns, term_doc = _gather_tokens(array, tokenizer,
|
|
112
|
+
term_dict, term_doc, start_doc_id=batch_beg,
|
|
113
|
+
trunc_posn=trunc_posn)
|
|
101
114
|
|
|
102
115
|
# Use posns to compute doc lens
|
|
103
116
|
doc_lens = _compute_doc_lens(posns=terms_w_posns[2, :],
|
|
@@ -114,12 +127,11 @@ def _tokenize_batch(array, tokenizer, term_dict, term_doc, batch_size, batch_beg
|
|
|
114
127
|
if np.any(doc_lens > MAX_POSN):
|
|
115
128
|
raise ValueError(f"Document length exceeds maximum of {MAX_POSN}")
|
|
116
129
|
|
|
117
|
-
return term_doc, bit_posns,
|
|
130
|
+
return batch_beg, term_doc, bit_posns, doc_lens
|
|
118
131
|
|
|
119
132
|
|
|
120
133
|
def batch_iterator(iterator, batch_size):
|
|
121
134
|
"""
|
|
122
|
-
Slices an iterator into batches of specified size and yields each batch until the iterator is exhausted.
|
|
123
135
|
|
|
124
136
|
:param iterator: The iterator to slice into batches.
|
|
125
137
|
:param batch_size: The size of each batch.
|
|
@@ -134,44 +146,87 @@ def batch_iterator(iterator, batch_size):
|
|
|
134
146
|
batch_beg += batch_size
|
|
135
147
|
|
|
136
148
|
|
|
137
|
-
def
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
for batch_beg, batch in batch_iterator(array, batch_size):
|
|
149
|
+
def _process_batches(term_doc, batch_size,
|
|
150
|
+
futures,
|
|
151
|
+
last_batch_beg_processed,
|
|
152
|
+
bit_posns=None,
|
|
153
|
+
doc_lens=None,
|
|
154
|
+
truncate=False):
|
|
155
|
+
batch_results = [None] * len(futures)
|
|
156
|
+
batch_beg = 0
|
|
157
|
+
for future in as_completed(futures):
|
|
148
158
|
try:
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
159
|
+
batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = future.result()
|
|
160
|
+
idx = (batch_beg - last_batch_beg_processed) // batch_size
|
|
161
|
+
assert batch_results[idx] is None
|
|
162
|
+
batch_results[idx] \
|
|
163
|
+
= (batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens)
|
|
154
164
|
except ValueError as e:
|
|
155
165
|
logger.error(e)
|
|
156
166
|
logger.error(f"Batch {batch_beg} failed to tokenize")
|
|
157
167
|
raise e
|
|
158
168
|
|
|
169
|
+
for result in batch_results:
|
|
170
|
+
assert result is not None
|
|
171
|
+
batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = result
|
|
172
|
+
term_doc.concat(batch_term_doc)
|
|
159
173
|
if bit_posns is None:
|
|
160
174
|
bit_posns = batch_bit_posns
|
|
161
175
|
else:
|
|
162
176
|
bit_posns.concat(batch_bit_posns)
|
|
163
177
|
|
|
164
178
|
doc_lens.append(batch_doc_lens)
|
|
179
|
+
return bit_posns
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
|
|
183
|
+
truncate=False, workers=4):
|
|
184
|
+
"""Build index directly from tokenizing docs (array of string)."""
|
|
185
|
+
term_dict = TermDict()
|
|
186
|
+
term_doc = SparseMatSetBuilder()
|
|
187
|
+
doc_lens: List[np.ndarray] = []
|
|
188
|
+
bit_posns = None
|
|
165
189
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
190
|
+
logger.info("Indexing begins")
|
|
191
|
+
futures = []
|
|
192
|
+
last_batch_beg_processed = 0
|
|
193
|
+
|
|
194
|
+
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
195
|
+
|
|
196
|
+
for batch_beg, batch in batch_iterator(array, batch_size):
|
|
197
|
+
futures.append(executor.submit(_tokenize_batch,
|
|
198
|
+
batch,
|
|
199
|
+
tokenizer,
|
|
200
|
+
term_dict,
|
|
201
|
+
batch_size,
|
|
202
|
+
batch_beg,
|
|
203
|
+
truncate=truncate))
|
|
204
|
+
|
|
205
|
+
if len(futures) >= workers:
|
|
206
|
+
logger.info(f"Collecting {len(futures)} futures")
|
|
207
|
+
bit_posns = _process_batches(term_doc, batch_size,
|
|
208
|
+
futures,
|
|
209
|
+
last_batch_beg_processed,
|
|
210
|
+
bit_posns, doc_lens, truncate=truncate)
|
|
211
|
+
logger.info(f"{batch_beg} Batch Complete")
|
|
212
|
+
logger.info(f"Roaringish NBytes -- {convert_size(bit_posns.nbytes)}")
|
|
213
|
+
logger.info(f"Term Dict Size -- {len(term_dict)}")
|
|
214
|
+
|
|
215
|
+
last_batch_beg_processed += len(futures) * batch_size
|
|
216
|
+
futures = []
|
|
217
|
+
if len(futures) > 0:
|
|
218
|
+
bit_posns = _process_batches(term_doc, batch_size,
|
|
219
|
+
futures,
|
|
220
|
+
last_batch_beg_processed,
|
|
221
|
+
bit_posns, doc_lens, truncate=truncate)
|
|
169
222
|
|
|
170
223
|
doc_lens = np.concatenate(doc_lens)
|
|
171
224
|
|
|
172
225
|
avg_doc_length = np.mean(doc_lens)
|
|
173
226
|
|
|
174
|
-
|
|
227
|
+
term_doc_built = RowViewableMatrix(term_doc.build())
|
|
228
|
+
logger.info("Indexing from tokenization complete")
|
|
229
|
+
return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
|
|
175
230
|
|
|
176
231
|
|
|
177
232
|
def build_index_from_terms_list(postings, Terms):
|
|
@@ -99,7 +99,10 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
|
|
|
99
99
|
return phrase_freqs, (lhs_cont, rhs_cont)
|
|
100
100
|
|
|
101
101
|
|
|
102
|
-
def _inner_bigram_freqs(lhs: np.ndarray,
|
|
102
|
+
def _inner_bigram_freqs(lhs: np.ndarray,
|
|
103
|
+
rhs: np.ndarray,
|
|
104
|
+
lhs_int: np.ndarray,
|
|
105
|
+
rhs_int: np.ndarray,
|
|
103
106
|
phrase_freqs: np.ndarray,
|
|
104
107
|
cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
105
108
|
"""Count bigram matches between two encoded arrays, within a 64 bit word with same MSBs.
|
|
@@ -108,6 +111,8 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
108
111
|
-----------
|
|
109
112
|
lhs: roaringish encoded posn array of left term
|
|
110
113
|
rhs: roaringish encoded posn array of right term
|
|
114
|
+
lhs_int: np.ndarray, intersected lhs
|
|
115
|
+
rhs_int: np.ndarray, intersected rhs
|
|
111
116
|
phrase_freqs: np.ndarray, preallocated phrase freqs for output
|
|
112
117
|
cont_rhs: bool, whether to continue matching on the rhs or lhs
|
|
113
118
|
|
|
@@ -117,7 +122,7 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
117
122
|
cont_next: the next (lhs or rhs) array to continue matching
|
|
118
123
|
|
|
119
124
|
"""
|
|
120
|
-
lhs_int, rhs_int = encoder.intersect(lhs, rhs)
|
|
125
|
+
# lhs_int, rhs_int = encoder.intersect(lhs, rhs)
|
|
121
126
|
lhs_doc_ids = encoder.keys(lhs_int)
|
|
122
127
|
if len(lhs_int) != len(rhs_int):
|
|
123
128
|
raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
|
|
@@ -149,6 +154,7 @@ def _inner_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
149
154
|
|
|
150
155
|
|
|
151
156
|
def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
157
|
+
lhs_adj: np.ndarray, rhs_adj: np.ndarray,
|
|
152
158
|
phrase_freqs: np.ndarray,
|
|
153
159
|
cont: Continuation = Continuation.RHS) -> Tuple[np.ndarray, Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
154
160
|
"""Count bigram matches between two encoded arrays where they occur in adjacent 64 bit words.
|
|
@@ -159,10 +165,9 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
159
165
|
rhs_next: the next rhs array to continue matching
|
|
160
166
|
|
|
161
167
|
"""
|
|
162
|
-
|
|
163
|
-
lhs_doc_ids = encoder.keys(lhs_int)
|
|
168
|
+
lhs_doc_ids = encoder.keys(lhs_adj)
|
|
164
169
|
# lhs lsb set and rhs lsb's most significant bit set
|
|
165
|
-
matches = ((
|
|
170
|
+
matches = ((lhs_adj & _upper_bit) != 0) & ((rhs_adj & _1) != 0)
|
|
166
171
|
unique, counts = np.unique(lhs_doc_ids[matches], return_counts=True)
|
|
167
172
|
phrase_freqs[unique] += counts
|
|
168
173
|
# Set lsb to 0 where no match, lsb to 1 where match
|
|
@@ -170,40 +175,40 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
|
170
175
|
lhs_next = None if cont == Continuation.RHS else np.asarray([], dtype=np.uint64)
|
|
171
176
|
if np.any(matches):
|
|
172
177
|
if cont in [Continuation.RHS, Continuation.BOTH]:
|
|
173
|
-
rhs_next =
|
|
178
|
+
rhs_next = rhs_adj[matches]
|
|
174
179
|
assert rhs_next is not None
|
|
175
|
-
rhs_next = encoder.header(
|
|
180
|
+
rhs_next = encoder.header(rhs_adj) | _1
|
|
176
181
|
elif cont in [Continuation.LHS, Continuation.BOTH]:
|
|
177
|
-
rhs_next =
|
|
182
|
+
rhs_next = rhs_adj[matches]
|
|
178
183
|
assert rhs_next is not None
|
|
179
184
|
rhs_next |= _1
|
|
180
|
-
rhs_next = encoder.header(
|
|
181
|
-
lhs_next =
|
|
182
|
-
lhs_next = encoder.header(
|
|
185
|
+
rhs_next = encoder.header(rhs_adj) | _1
|
|
186
|
+
lhs_next = lhs_adj[matches]
|
|
187
|
+
lhs_next = encoder.header(lhs_adj) | _upper_bit
|
|
183
188
|
rhs_next = None
|
|
184
189
|
return phrase_freqs, (lhs_next, rhs_next)
|
|
185
190
|
|
|
186
191
|
|
|
187
|
-
def _set_adjbit_at_header(
|
|
192
|
+
def _set_adjbit_at_header(next_inner: np.ndarray, next_adj: np.ndarray,
|
|
188
193
|
cont: Continuation = Continuation.RHS) -> np.ndarray:
|
|
189
194
|
"""Merge two encoded arrays on their headers."""
|
|
190
|
-
if len(
|
|
191
|
-
return
|
|
192
|
-
if len(
|
|
193
|
-
return
|
|
195
|
+
if len(next_inner) == 0:
|
|
196
|
+
return next_adj
|
|
197
|
+
if len(next_adj) == 0:
|
|
198
|
+
return next_inner
|
|
194
199
|
|
|
195
|
-
|
|
196
|
-
|
|
200
|
+
same_header_inner, same_header_adj = intersect(next_inner, next_adj,
|
|
201
|
+
mask=encoder.header_mask)
|
|
197
202
|
# Set _1 on intersection
|
|
198
|
-
ignore_mask = np.ones(len(
|
|
199
|
-
ignore_mask[
|
|
200
|
-
if len(
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
if len(
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
return merge(
|
|
203
|
+
ignore_mask = np.ones(len(next_adj), dtype=bool)
|
|
204
|
+
ignore_mask[same_header_adj] = False
|
|
205
|
+
if len(same_header_inner) > 0 and cont == Continuation.RHS:
|
|
206
|
+
next_inner[same_header_inner] |= _1
|
|
207
|
+
next_adj = next_adj[ignore_mask]
|
|
208
|
+
if len(same_header_adj) > 0 and cont == Continuation.LHS:
|
|
209
|
+
next_inner[same_header_inner] |= _upper_bit
|
|
210
|
+
next_adj = next_adj[ignore_mask]
|
|
211
|
+
return merge(next_inner, next_adj)
|
|
207
212
|
|
|
208
213
|
|
|
209
214
|
def bigram_freqs(lhs: np.ndarray,
|
|
@@ -242,9 +247,13 @@ def bigram_freqs(lhs: np.ndarray,
|
|
|
242
247
|
# rhs_term: 1234 101 0101000100
|
|
243
248
|
# ^^ ^^ <- inner matches, terms next to each other
|
|
244
249
|
#
|
|
245
|
-
|
|
246
|
-
= _inner_bigram_freqs(lhs, rhs, phrase_freqs, cont)
|
|
250
|
+
lhs_int, rhs_int, lhs_adj, rhs_adj = encoder.intersect_candidates(lhs, rhs)
|
|
247
251
|
|
|
252
|
+
phrase_freqs, (lhs_next_inner, rhs_next_inner)\
|
|
253
|
+
= _inner_bigram_freqs(lhs, rhs,
|
|
254
|
+
lhs_int, rhs_int,
|
|
255
|
+
phrase_freqs,
|
|
256
|
+
cont)
|
|
248
257
|
# ***************************************************************************
|
|
249
258
|
# "Adjacent" matches:
|
|
250
259
|
# Of course we also need to check for matches that span two 64 bit words,
|
|
@@ -259,7 +268,9 @@ def bigram_freqs(lhs: np.ndarray,
|
|
|
259
268
|
# 1 less than the posn_msb of rhs
|
|
260
269
|
# so we detect these differently
|
|
261
270
|
phrase_freqs, (lhs_next_adj, rhs_next_adj)\
|
|
262
|
-
= _adjacent_bigram_freqs(lhs, rhs,
|
|
271
|
+
= _adjacent_bigram_freqs(lhs, rhs,
|
|
272
|
+
lhs_adj, rhs_adj,
|
|
273
|
+
phrase_freqs, cont)
|
|
263
274
|
|
|
264
275
|
# ***************************************************************************
|
|
265
276
|
# rhs_next is where the bigram ends on the rhs side, we can use this
|
|
@@ -13,7 +13,7 @@ from searcharray.phrase.bigram_freqs import bigram_freqs, Continuation
|
|
|
13
13
|
from searcharray.phrase.spans import span_search
|
|
14
14
|
import numbers
|
|
15
15
|
import logging
|
|
16
|
-
from collections import defaultdict
|
|
16
|
+
from collections import defaultdict, abc
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
logger = logging.getLogger(__name__)
|
|
@@ -72,6 +72,7 @@ def trim_phrase_search(encoded_posns: List[np.ndarray],
|
|
|
72
72
|
|
|
73
73
|
def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
|
|
74
74
|
phrase_freqs: np.ndarray,
|
|
75
|
+
max_doc_id: np.uint64 = _0,
|
|
75
76
|
trim: bool = True) -> np.ndarray:
|
|
76
77
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
77
78
|
if len(encoded_posns) < 2:
|
|
@@ -86,7 +87,8 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
|
|
|
86
87
|
for rhs in encoded_posns[1:]:
|
|
87
88
|
# Only count the count of the last bigram (ignoring the ones where priors did not match)
|
|
88
89
|
phrase_freqs[mask] = 0
|
|
89
|
-
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
|
|
90
|
+
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
|
|
91
|
+
cont=Continuation.RHS)
|
|
90
92
|
assert conts[1] is not None
|
|
91
93
|
lhs = conts[1]
|
|
92
94
|
mask &= (phrase_freqs > 0)
|
|
@@ -99,6 +101,7 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
|
|
|
99
101
|
|
|
100
102
|
def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
|
|
101
103
|
phrase_freqs: np.ndarray,
|
|
104
|
+
max_doc_id: np.uint64 = _0,
|
|
102
105
|
trim: bool = True) -> np.ndarray:
|
|
103
106
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
104
107
|
if len(encoded_posns) < 2:
|
|
@@ -113,7 +116,8 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
|
|
|
113
116
|
for lhs in encoded_posns[-2::-1]:
|
|
114
117
|
# Only count the count of the last bigram (ignoring the ones where priors did not match)
|
|
115
118
|
phrase_freqs[mask] = 0
|
|
116
|
-
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
|
|
119
|
+
phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
|
|
120
|
+
cont=Continuation.LHS)
|
|
117
121
|
assert conts[0] is not None
|
|
118
122
|
rhs = conts[0]
|
|
119
123
|
mask &= (phrase_freqs > 0)
|
|
@@ -126,18 +130,19 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
|
|
|
126
130
|
|
|
127
131
|
def compute_phrase_freqs(encoded_posns: List[np.ndarray],
|
|
128
132
|
phrase_freqs: np.ndarray,
|
|
133
|
+
max_doc_id: np.uint64 = _0,
|
|
129
134
|
trim: bool = False) -> np.ndarray:
|
|
130
135
|
"""Compute phrase freqs from a set of encoded positions."""
|
|
131
136
|
shortest_len_index = min(enumerate(encoded_posns), key=lambda x: len(x[1]))[0]
|
|
132
137
|
if shortest_len_index <= 1:
|
|
133
|
-
return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim)
|
|
138
|
+
return _compute_phrase_freqs_left_to_right(encoded_posns, phrase_freqs, trim=trim, max_doc_id=max_doc_id)
|
|
134
139
|
elif shortest_len_index >= len(encoded_posns) - 2:
|
|
135
|
-
return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim)
|
|
140
|
+
return _compute_phrase_freqs_right_to_left(encoded_posns, phrase_freqs, trim=trim, max_doc_id=max_doc_id)
|
|
136
141
|
else:
|
|
137
142
|
# We optimize this case by going middle-out
|
|
138
143
|
# We can take the min of both directions phrase freqs
|
|
139
|
-
lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim)
|
|
140
|
-
rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim)
|
|
144
|
+
lhs = _compute_phrase_freqs_left_to_right(encoded_posns[:shortest_len_index], phrase_freqs, trim=trim, max_doc_id=max_doc_id)
|
|
145
|
+
rhs = _compute_phrase_freqs_right_to_left(encoded_posns[shortest_len_index:], phrase_freqs, trim=trim, max_doc_id=max_doc_id)
|
|
141
146
|
phrase_freqs = np.minimum(lhs, rhs)
|
|
142
147
|
return phrase_freqs
|
|
143
148
|
|
|
@@ -259,6 +264,35 @@ def index_range(rng, key):
|
|
|
259
264
|
return np.asarray(list(rng))[key]
|
|
260
265
|
|
|
261
266
|
|
|
267
|
+
class FilteredPosns(abc.Mapping):
|
|
268
|
+
"""When someone slices, this lets us avoid repeating constantly
|
|
269
|
+
slicing the keys out of the encoded positions."""
|
|
270
|
+
|
|
271
|
+
def __init__(self, base, doc_ids):
|
|
272
|
+
self.base = base
|
|
273
|
+
self.sliced = {}
|
|
274
|
+
self.doc_ids = doc_ids
|
|
275
|
+
|
|
276
|
+
def __getitem__(self, key):
|
|
277
|
+
"""Slice with encoder, cache, and return."""
|
|
278
|
+
if key in self.sliced:
|
|
279
|
+
return self.sliced[key]
|
|
280
|
+
|
|
281
|
+
sliced = encoder.slice(self.base[key],
|
|
282
|
+
keys=self.doc_ids)
|
|
283
|
+
self.sliced[key] = sliced
|
|
284
|
+
return sliced
|
|
285
|
+
|
|
286
|
+
def __setitem__(self, key, value):
|
|
287
|
+
self.base[key] = value
|
|
288
|
+
|
|
289
|
+
def __iter__(self):
|
|
290
|
+
return iter(self.doc_ids)
|
|
291
|
+
|
|
292
|
+
def __len__(self):
|
|
293
|
+
return len(self.doc_ids)
|
|
294
|
+
|
|
295
|
+
|
|
262
296
|
class PosnBitArray:
|
|
263
297
|
|
|
264
298
|
def __init__(self, encoded_term_posns, max_doc_id: int):
|
|
@@ -274,9 +308,21 @@ class PosnBitArray:
|
|
|
274
308
|
self.docfreq(term_id)
|
|
275
309
|
self.termfreqs(term_id)
|
|
276
310
|
|
|
311
|
+
def filter(self, doc_ids) -> "PosnBitArray":
|
|
312
|
+
"""Filter my doc ids to only those in doc_ids."""
|
|
313
|
+
enc_term_posns = self.encoded_term_posns
|
|
314
|
+
if isinstance(enc_term_posns, FilteredPosns):
|
|
315
|
+
self.encoded_term_posns = enc_term_posns.base
|
|
316
|
+
return PosnBitArray(FilteredPosns(self.encoded_term_posns, doc_ids), self.max_doc_id)
|
|
317
|
+
|
|
318
|
+
def _reset_filter(self):
|
|
319
|
+
if isinstance(self.encoded_term_posns, FilteredPosns):
|
|
320
|
+
self.encoded_term_posns = self.encoded_term_posns.base
|
|
321
|
+
|
|
277
322
|
def clear_cache(self):
|
|
278
323
|
self.docfreq_cache = {}
|
|
279
324
|
self.termfreq_cache = {}
|
|
325
|
+
self._reset_filter()
|
|
280
326
|
|
|
281
327
|
def copy(self):
|
|
282
328
|
new = PosnBitArray(deepcopy(self.encoded_term_posns), self.max_doc_id)
|
|
@@ -335,18 +381,22 @@ class PosnBitArray:
|
|
|
335
381
|
keys=np.asarray([doc_id], dtype=np.uint64))
|
|
336
382
|
return term_posns
|
|
337
383
|
|
|
338
|
-
def
|
|
339
|
-
|
|
384
|
+
def empty_buffer(self):
|
|
385
|
+
return np.zeros(int(self.max_doc_id + 1), dtype=np.float64)
|
|
386
|
+
|
|
387
|
+
def phrase_freqs(self, term_ids: List[int],
|
|
340
388
|
slop: int = 0,
|
|
389
|
+
doc_ids: Optional[np.ndarray] = None,
|
|
341
390
|
min_posn: Optional[int] = None,
|
|
342
391
|
max_posn: Optional[int] = None) -> np.ndarray:
|
|
392
|
+
phrase_freqs = self.empty_buffer()
|
|
343
393
|
if len(term_ids) < 2:
|
|
344
394
|
raise ValueError("Must have at least two terms")
|
|
345
|
-
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None:
|
|
395
|
+
if phrase_freqs.shape[0] == self.max_doc_id + 1 and min_posn is None and max_posn is None and doc_ids is None:
|
|
346
396
|
enc_term_posns = [self.encoded_term_posns[term_id] for term_id in term_ids]
|
|
347
397
|
else:
|
|
348
398
|
keys = None
|
|
349
|
-
if
|
|
399
|
+
if doc_ids is not None:
|
|
350
400
|
keys = doc_ids.view(np.uint64)
|
|
351
401
|
|
|
352
402
|
enc_term_posns = [encoder.slice(self.encoded_term_posns[term_id],
|
|
@@ -355,7 +405,7 @@ class PosnBitArray:
|
|
|
355
405
|
max_payload=max_posn) for term_id in term_ids]
|
|
356
406
|
|
|
357
407
|
if slop == 0:
|
|
358
|
-
return compute_phrase_freqs(enc_term_posns, phrase_freqs)
|
|
408
|
+
return compute_phrase_freqs(enc_term_posns, phrase_freqs, max_doc_id=np.uint64(self.max_doc_id))
|
|
359
409
|
else:
|
|
360
410
|
return span_search(enc_term_posns, phrase_freqs, slop)
|
|
361
411
|
|