searcharray 0.0.13a0__tar.gz → 0.0.14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. {searcharray-0.0.13a0 → searcharray-0.0.14}/PKG-INFO +1 -1
  2. searcharray-0.0.14/searcharray/phrase/__init__.py +0 -0
  3. searcharray-0.0.14/searcharray/phrase/middle_out.py +326 -0
  4. searcharray-0.0.14/searcharray/phrase/padded_arrays.py +221 -0
  5. searcharray-0.0.14/searcharray/phrase/posn_diffs.py +147 -0
  6. searcharray-0.0.14/searcharray/phrase/scan_merge.py +230 -0
  7. searcharray-0.0.14/searcharray/phrase/wide_spans.py +183 -0
  8. searcharray-0.0.14/searcharray/utils/__init__.py +0 -0
  9. searcharray-0.0.14/searcharray/utils/row_viewable_matrix.py +77 -0
  10. {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/PKG-INFO +1 -1
  11. {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/SOURCES.txt +8 -0
  12. {searcharray-0.0.13a0 → searcharray-0.0.14}/setup.py +3 -3
  13. {searcharray-0.0.13a0 → searcharray-0.0.14}/README.md +0 -0
  14. {searcharray-0.0.13a0 → searcharray-0.0.14}/pyproject.toml +0 -0
  15. {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray/__init__.py +0 -0
  16. {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray/postings.py +0 -0
  17. {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray/stemmers.py +0 -0
  18. {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray/term_dict.py +0 -0
  19. {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/dependency_links.txt +0 -0
  20. {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/entry_points.txt +0 -0
  21. {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/requires.txt +0 -0
  22. {searcharray-0.0.13a0 → searcharray-0.0.14}/searcharray.egg-info/top_level.txt +0 -0
  23. {searcharray-0.0.13a0 → searcharray-0.0.14}/setup.cfg +0 -0
  24. {searcharray-0.0.13a0 → searcharray-0.0.14}/test/test_extension_array.py +0 -0
  25. {searcharray-0.0.13a0 → searcharray-0.0.14}/test/test_msmarco.py +0 -0
  26. {searcharray-0.0.13a0 → searcharray-0.0.14}/test/test_phrase_matches.py +0 -0
  27. {searcharray-0.0.13a0 → searcharray-0.0.14}/test/test_tmdb.py +0 -0
  28. {searcharray-0.0.13a0 → searcharray-0.0.14}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.13a0
3
+ Version: 0.0.14
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
File without changes
@@ -0,0 +1,326 @@
1
+ """Encode positions in bits along with some neighboring information for wrapping.
2
+
3
+ See this notebook for motivation:
4
+
5
+ https://colab.research.google.com/drive/10tIEkdlCE_1J_CcgEcV0jkLfBc-0H4am?authuser=1#scrollTo=XWzy-n9dF3PG
6
+
7
+ """
8
+ import numpy as np
9
+ import sortednp as snp
10
+ from copy import deepcopy
11
+ from typing import List, Optional
12
+ import numbers
13
+ import logging
14
+ from time import perf_counter
15
+
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ # When running in pytest
20
+ import sys # noqa
21
+ handler = logging.StreamHandler(sys.stdout)
22
+ handler.setLevel(logging.ERROR)
23
+ formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
24
+ handler.setFormatter(formatter)
25
+ logger.addHandler(handler)
26
+ logger.setLevel(logging.ERROR)
27
+
28
+
29
+ DOC_ID_MASK = 0xFFFFFFF000000000
30
+ DOC_ID_BITS = 28
31
+ POSN_MSB_MASK = 0x0000000FFFFC0000
32
+ POSN_MSB_BITS = 18
33
+ POSN_LSB_MASK = 0x000000000003FFFF
34
+ POSN_LSB_BITS = 18
35
+
36
+ assert DOC_ID_BITS + POSN_MSB_BITS + POSN_LSB_BITS == 64
37
+ assert DOC_ID_MASK | POSN_MSB_MASK | POSN_LSB_MASK == 0xFFFFFFFFFFFFFFFF
38
+
39
+
40
+ def validate_posns(posns: np.ndarray):
41
+ if not np.all(posns < 2**POSN_LSB_BITS):
42
+ raise ValueError(f"Positions must be less than {2**POSN_LSB_BITS}")
43
+
44
+
45
+ def encode_posns(posns: np.ndarray, doc_ids: Optional[np.ndarray] = None):
46
+ """Pack a sorted array of positions into compact bit array.
47
+
48
+ each returned array represents a single term, with doc_id as 32 MSBs
49
+
50
+ | 32 MSBs | 16 LSBs | 16 LSBs |
51
+ doc_id bits msbs posns
52
+
53
+ for later easy intersection of 32+16 msbs, then checking for adjacent
54
+ positions
55
+
56
+ """
57
+ validate_posns(posns)
58
+ cols = posns // POSN_LSB_BITS # Header of bit to use
59
+ cols = cols.astype(np.uint64) << POSN_MSB_BITS
60
+ if doc_ids is not None:
61
+ cols |= doc_ids.astype(np.uint64) << (64 - DOC_ID_BITS)
62
+ values = posns % POSN_LSB_BITS # Value to encode
63
+
64
+ change_indices = np.nonzero(np.diff(cols))[0] + 1
65
+ change_indices = np.insert(change_indices, 0, 0)
66
+
67
+ encoded = cols | (1 << values)
68
+ if len(encoded) == 0:
69
+ return encoded
70
+ return np.bitwise_or.reduceat(encoded, change_indices)
71
+
72
+
73
+ def decode_posns(encoded):
74
+ start = perf_counter()
75
+ doc_ids = (encoded & DOC_ID_MASK) >> (64 - DOC_ID_BITS)
76
+ msbs = (encoded & POSN_MSB_MASK) >> POSN_MSB_BITS
77
+ to_concat = []
78
+ for bit in range(POSN_LSB_BITS):
79
+ mask = 1 << bit
80
+ lsbs = encoded & mask
81
+ set_lsbs = (lsbs != 0)
82
+ this_doc_ids = doc_ids[set_lsbs]
83
+ posns = bit + (msbs[set_lsbs] * POSN_LSB_BITS)
84
+ doc_with_posn = np.dstack([this_doc_ids, posns])[0]
85
+ to_concat.append(doc_with_posn)
86
+
87
+ logger.debug(f"loop took {perf_counter() - start:.2f} seconds")
88
+ stacked = np.vstack(to_concat)
89
+ # Sort by doc_id, then posn
90
+ sorted_posns = stacked[np.lexsort((stacked[:, 1], stacked[:, 0]))]
91
+ doc_ids, idx = np.unique(sorted_posns[:, 0], return_index=True)
92
+ grouped = np.split(sorted_posns[:, 1], idx[1:])
93
+ as_list = list(zip(doc_ids, grouped))
94
+
95
+ logger.debug(f"groupby took {perf_counter() - start:.2f} seconds | got {len(as_list)}")
96
+ return as_list
97
+
98
+
99
+ def intersect_msbs(lhs: np.ndarray, rhs: np.ndarray):
100
+ """Return the MSBs that are common to both lhs and rhs."""
101
+ # common = np.intersect1d(lhs_msbs, rhs_msbs)
102
+ _, (lhs_idx, rhs_idx) = snp.intersect(lhs >> POSN_LSB_BITS, rhs >> POSN_LSB_BITS, indices=True)
103
+ # With large arrays np.isin becomes a bottleneck
104
+ return lhs[lhs_idx], rhs[rhs_idx]
105
+
106
+
107
+ def convert_doc_ids(doc_ids):
108
+ if isinstance(doc_ids, numbers.Number):
109
+ return np.asarray([doc_ids], dtype=np.uint64)
110
+ elif isinstance(doc_ids, list):
111
+ return np.asarray(doc_ids, dtype=np.uint64)
112
+ elif isinstance(doc_ids, np.ndarray):
113
+ return doc_ids.astype(np.uint64)
114
+ elif isinstance(doc_ids, range):
115
+ return np.asarray(doc_ids, dtype=np.uint64) # UNFORTUNATE COPY
116
+
117
+
118
+ def get_docs(encoded: np.ndarray, doc_ids: np.ndarray):
119
+ """Get list of encoded that have positions in doc_ids."""
120
+ doc_ids = convert_doc_ids(doc_ids)
121
+ assert len(doc_ids.shape) == 1
122
+ assert len(encoded.shape) == 1
123
+ encoded_doc_ids = encoded.astype(np.uint64) >> (64 - DOC_ID_BITS)
124
+ empty = doc_ids << (64 - DOC_ID_BITS)
125
+ _, (idx_docs, idx_enc) = snp.intersect(doc_ids, encoded_doc_ids, indices=True,
126
+ duplicates=snp.KEEP_MAX_N)
127
+
128
+ found = encoded[idx_enc]
129
+ empties = empty[np.isin(doc_ids, found, invert=True)]
130
+
131
+ merged = snp.merge(found, empties, duplicates=snp.DROP)
132
+ return merged
133
+
134
+
135
+ def inner_bigram_match(lhs, rhs):
136
+ """Count bigram matches between two encoded arrays."""
137
+ lhs, rhs = intersect_msbs(lhs, rhs)
138
+ if len(lhs) != len(rhs):
139
+ raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
140
+ rhs_next = (rhs & DOC_ID_MASK)
141
+ rhs_next_mask = np.zeros(len(rhs), dtype=bool)
142
+ counts = []
143
+ # With popcount soon to be in numpy, this could potentially
144
+ # be simply a left shift of the RHS LSB poppcount, and and a popcount
145
+ # to count the overlaps
146
+ for bit in range(1, POSN_LSB_BITS - 1):
147
+ lhs_mask = 1 << bit
148
+ rhs_mask = 1 << (bit + 1)
149
+ lhs_set = (lhs & lhs_mask) != 0
150
+ rhs_set = (rhs & rhs_mask) != 0
151
+
152
+ matches = lhs_set & rhs_set
153
+ rhs_next[matches] |= rhs_mask
154
+ rhs_next_mask |= matches
155
+ counts.append(np.count_nonzero(matches))
156
+ return np.sum(counts), rhs_next[rhs_next_mask]
157
+
158
+
159
+ class PosnBitArrayBuilder:
160
+
161
+ def __init__(self):
162
+ self.term_posns = {}
163
+ self.term_posn_doc_ids = {}
164
+ self.max_doc_id = 0
165
+
166
+ def add_posns(self, doc_id: int, term_id: int, posns):
167
+ if len(posns.shape) != 1:
168
+ raise ValueError("posns must be a 1D array")
169
+ if term_id not in self.term_posns:
170
+ self.term_posns[term_id] = []
171
+ self.term_posn_doc_ids[term_id] = []
172
+ doc_ids = [doc_id] * len(posns)
173
+ self.term_posns[term_id].extend(posns.tolist())
174
+ self.term_posn_doc_ids[term_id].extend(doc_ids)
175
+ self.max_doc_id = max(self.max_doc_id, doc_id)
176
+
177
+ def ensure_capacity(self, doc_id):
178
+ self.max_doc_id = max(self.max_doc_id, doc_id)
179
+
180
+ def build(self, check=False):
181
+ encoded_term_posns = {}
182
+ for term_id, posns in self.term_posns.items():
183
+ if len(posns) == 0:
184
+ posns = np.asarray([], dtype=np.uint32).flatten()
185
+ elif isinstance(posns, list):
186
+ posns_arr = np.asarray(posns, dtype=np.uint32)
187
+ assert len(posns_arr.shape) == 1
188
+ posns = posns_arr
189
+ doc_ids = self.term_posn_doc_ids[term_id]
190
+ if isinstance(doc_ids, list):
191
+ doc_ids = np.asarray(doc_ids, dtype=np.uint32)
192
+ encoded = encode_posns(doc_ids=doc_ids, posns=posns)
193
+ decode_again = decode_posns(encoded)
194
+ if check:
195
+ docs_to_posns = dict(decode_again)
196
+ doc_ids_again = []
197
+ posns_again = []
198
+ for doc_id, posns_dec in docs_to_posns.items():
199
+ for posn in posns_dec:
200
+ doc_ids_again.append(doc_id)
201
+ posns_again.append(posn)
202
+ assert np.array_equal(doc_ids_again, doc_ids)
203
+ assert np.array_equal(posns, posns_again)
204
+ encoded_term_posns[term_id] = encoded
205
+
206
+ return PosnBitArray(encoded_term_posns, range(0, self.max_doc_id + 1))
207
+
208
+
209
+ def index_range(rng, key):
210
+ if key is None:
211
+ return rng
212
+ if isinstance(rng, np.ndarray):
213
+ return rng[key]
214
+
215
+ if isinstance(key, slice):
216
+ return rng[key]
217
+ elif isinstance(key, numbers.Number):
218
+ return rng[key]
219
+ elif isinstance(key, np.ndarray):
220
+ try:
221
+ # UNFORTUNATE COPY
222
+ r_val = np.asarray(list(rng))[key]
223
+ return r_val
224
+ except IndexError as e:
225
+ raise e
226
+ # Last resort
227
+ # UNFORTUNATE COPY
228
+ # Here probably elipses or a tuple of various things
229
+ return np.asarray(list(rng))[key]
230
+
231
+
232
+ class PosnBitArray:
233
+
234
+ def __init__(self, encoded_term_posns, doc_ids):
235
+ self.encoded_term_posns = encoded_term_posns
236
+ self.doc_ids = doc_ids
237
+
238
+ def copy(self):
239
+ new = PosnBitArray(deepcopy(self.encoded_term_posns),
240
+ self.doc_ids)
241
+ return new
242
+
243
+ def slice(self, key):
244
+ sliced_term_posns = {}
245
+ doc_ids = index_range(self.doc_ids, key)
246
+ for term_id, posns in self.encoded_term_posns.items():
247
+ encoded = self.encoded_term_posns[term_id]
248
+ assert len(encoded.shape) == 1
249
+ sliced_term_posns[term_id] = get_docs(encoded, doc_ids=doc_ids)
250
+
251
+ return PosnBitArray(sliced_term_posns, doc_ids=doc_ids)
252
+
253
+ def __getitem__(self, key):
254
+ return self.slice(key)
255
+
256
+ def merge(self, other):
257
+ for term_id, posns in self.encoded_term_posns.items():
258
+ try:
259
+ posns_other = other.encoded_term_posns[term_id]
260
+ self.encoded_term_posns[term_id] = snp.merge(posns, posns_other)
261
+ except KeyError:
262
+ pass
263
+
264
+ def positions(self, term_id: int, key) -> List:
265
+ # Check if key is in doc ids?
266
+ start = perf_counter()
267
+ doc_ids = index_range(self.doc_ids, key)
268
+ logger.debug(f"index_range took {perf_counter() - start} seconds")
269
+ if isinstance(doc_ids, numbers.Number):
270
+ doc_ids = np.asarray([doc_ids])
271
+ try:
272
+ term_posns = get_docs(self.encoded_term_posns[term_id],
273
+ doc_ids=doc_ids)
274
+ logger.debug(f"get_docs took {perf_counter() - start} seconds")
275
+ except KeyError:
276
+ r_val = [np.array([], dtype=np.uint32) for doc_id in doc_ids]
277
+ if len(r_val) == 1 and isinstance(key, numbers.Number):
278
+ r_val = r_val[0]
279
+ logger.debug(f"positions exit(1) took {perf_counter() - start} seconds")
280
+ return r_val
281
+
282
+ decoded = decode_posns(term_posns)
283
+ logger.debug(f"decode took {perf_counter() - start} seconds")
284
+
285
+ if len(decoded) == 0:
286
+ return np.array([], dtype=np.uint32)
287
+ logger.debug(f"positions exit(2) took {perf_counter() - start} seconds")
288
+ if len(decoded) != len(doc_ids):
289
+ # Fill non matches
290
+ as_dict = dict(decoded)
291
+ decs = []
292
+ for doc_id in doc_ids:
293
+ if doc_id in as_dict:
294
+ decs.append(as_dict[doc_id])
295
+ else:
296
+ decs.append(np.array([], dtype=np.uint32))
297
+ logger.debug(f"positions exit(3) took {perf_counter() - start} seconds")
298
+ return decs
299
+ else:
300
+ decs = [dec[1] for dec in decoded]
301
+ if len(decs) == 1 and isinstance(key, numbers.Number):
302
+ decs = decs[0]
303
+ logger.debug(f"positions exit(4) took {perf_counter() - start} seconds")
304
+ return decs
305
+
306
+ def insert(self, key, term_ids_to_posns):
307
+ new_posns = PosnBitArrayBuilder()
308
+ max_doc_id = 0
309
+ for doc_id, new_posns_row in enumerate(term_ids_to_posns):
310
+ for term_id, positions in new_posns_row:
311
+ new_posns.add_posns(doc_id, term_id, positions)
312
+ max_doc_id = max(doc_id, max_doc_id)
313
+ new_posns.max_doc_id = max_doc_id
314
+ ins_arr = new_posns.build()
315
+ self.merge(ins_arr)
316
+
317
+ @property
318
+ def nbytes(self):
319
+ arr_bytes = 0
320
+ for doc_id, posn in self.encoded_term_posns.items():
321
+ arr_bytes += posn.nbytes
322
+ return arr_bytes
323
+
324
+ def phrase_freqs(self, terms: List[int]):
325
+ """Return the phrase frequencies for a list of terms."""
326
+ raise NotImplementedError("Not yet implemented")
@@ -0,0 +1,221 @@
1
+ """Query-doc term positions implemented as a padded array."""
2
+ from collections import defaultdict
3
+ from searcharray.utils.row_viewable_matrix import RowViewableMatrix
4
+ from scipy.sparse import dok_matrix
5
+ from time import perf_counter
6
+ import logging
7
+ import numpy as np
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+ # When running in pytest
12
+ import sys # noqa
13
+ handler = logging.StreamHandler(sys.stdout)
14
+ handler.setLevel(logging.INFO)
15
+ formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
16
+ handler.setFormatter(formatter)
17
+ logger.addHandler(handler)
18
+ logger.setLevel(logging.INFO)
19
+
20
+
21
+ def _pad_to_len(arr, fill, every=10):
22
+ """Pad an array to a multiple of every."""
23
+ if len(arr) % every != 0:
24
+ arr = np.pad(arr, (0, every - len(arr) % every),
25
+ 'constant', constant_values=fill)
26
+ return arr
27
+
28
+
29
+ def mat_index_from_width(width, mats):
30
+ """Return the index of the matrix to use for a given width."""
31
+ if width < len(mats) - 1:
32
+ return width // 10
33
+ else:
34
+ return len(mats) - 1
35
+
36
+
37
+ class PaddedPosnArraysBuilder():
38
+ def __init__(self, pad=-100):
39
+ # Matrix per 10 positions
40
+ self.posns_mats = [defaultdict(int), defaultdict(int),
41
+ defaultdict(int), defaultdict(int),
42
+ defaultdict(int)]
43
+
44
+ self.posns_lookup = [np.array([]), ] # 0th is invalid
45
+ self.pad = pad
46
+
47
+ def add_posns(self, doc_id, term_id, positions):
48
+ idx = len(self.posns_lookup)
49
+ positions = np.array(positions)
50
+
51
+ # PAD TO LENGTH % 10
52
+ positions = _pad_to_len(positions, self.pad)
53
+
54
+ self.posns_lookup.append(positions)
55
+ mat_to_use = mat_index_from_width(len(positions), self.posns_mats)
56
+ self.posns_mats[mat_to_use][doc_id, term_id] += idx
57
+
58
+ def _build_mat(self, mat_dict, num_postings, num_terms):
59
+ posns_dok = dok_matrix((num_postings, num_terms), dtype=np.uint32)
60
+ dict.update(posns_dok, mat_dict)
61
+ posns_csr = posns_dok.tocsr()
62
+ posns_mat = RowViewableMatrix(posns_csr)
63
+ return posns_mat
64
+
65
+ def build(self, num_postings, num_terms):
66
+ mats = [self._build_mat(mat_dict, num_postings, num_terms)
67
+ for mat_dict in self.posns_mats]
68
+
69
+ padded_posn_arrays = PaddedPosnArrays()
70
+ padded_posn_arrays.posns_mats = mats
71
+ padded_posn_arrays.posns_lookup = self.posns_lookup
72
+ return padded_posn_arrays
73
+
74
+
75
+ class PaddedPosnArrays():
76
+
77
+ def __init__(self, pad=-100):
78
+ self.posns_mats = None
79
+ self.posns_lookup = [np.array([])] # 0th is invalid
80
+ self.pad = pad
81
+
82
+ def ensure_capacity(self, num_postings, num_terms):
83
+ for mat in self.posns_mats:
84
+ mat.resize((num_postings, num_terms))
85
+ max_lookup = max(mat.mat.max() for mat in self.posns_mats)
86
+ if max_lookup > len(self.posns_lookup):
87
+ self.posns_lookup = np.resize(self.posns_lookup, max_lookup + 1)
88
+
89
+ @property
90
+ def nbytes(self):
91
+ posns_lookup_bytes = sum(x.nbytes for x in self.posns_lookup)
92
+ posns_mats_bytes = sum(x.nbytes for x in self.posns_mats)
93
+ return posns_mats_bytes + posns_lookup_bytes
94
+
95
+ def insert(self, key, term_ids_to_posns):
96
+ for update_doc_idx, new_posns_row in enumerate(term_ids_to_posns):
97
+ for term_id, positions in new_posns_row:
98
+ mat_idx = mat_index_from_width(len(positions), self.posns_mats)
99
+ update_docs = self.posns_mats[mat_idx][key]
100
+ lookup_location = update_docs[update_doc_idx][0, term_id]
101
+ self.posns_lookup[lookup_location] = positions
102
+
103
+ def positions(self, term_id, key=None, padded=False, width_hint=None):
104
+
105
+ if width_hint is not None:
106
+ mat_to_use = mat_index_from_width(width_hint, self.posns_mats)
107
+ posns_to_lookup = self.posns_mats[mat_to_use].copy_col_at(term_id)
108
+ else:
109
+ for mat in self.posns_mats:
110
+ if key is not None:
111
+ posns_to_lookup = mat.copy_col_at(term_id)[key]
112
+ else:
113
+ posns_to_lookup = self.posns_mat.copy_col_at(term_id)
114
+ if posns_to_lookup.nnz > 0:
115
+ break
116
+
117
+ # This could be faster if posns_lookup was more row slicable
118
+ posns_to_lookup = posns_to_lookup.toarray().flatten()
119
+ if padded:
120
+ posns = [self.posns_lookup[lookup] for lookup in posns_to_lookup]
121
+ else:
122
+ posns = [self.posns_lookup[lookup][self.posns_lookup[lookup] != self.pad] for lookup in posns_to_lookup]
123
+ return posns
124
+
125
+ def copy(self):
126
+ new = PaddedPosnArrays()
127
+ new.posns_mats = [mat.copy() for mat in self.posns_mats]
128
+ new.posns_lookup = self.posns_lookup.copy()
129
+ return new
130
+
131
+ def slice(self, key):
132
+ new = PaddedPosnArrays()
133
+ new.posns_mats = [mat.slice(key) for mat in self.posns_mats]
134
+ new.posns_lookup = self.posns_lookup
135
+ return new
136
+
137
+ def __getitem__(self, key):
138
+ return self.posns_mat[key]
139
+
140
+ def _compute_phrase_freqs(term_posns, slop=1):
141
+ start = perf_counter()
142
+
143
+ prior_term = term_posns[0]
144
+ logger.info(f"Preamble took {perf_counter() - start:.4f} seconds")
145
+ for term in term_posns[1:]:
146
+ is_same_term = (term.shape == prior_term.shape) and np.all(term[0] == prior_term[0])
147
+
148
+ # Compute positional differences
149
+ #
150
+ # Each row of posn_diffs is a term posn diff matrix
151
+ # Where columns are prior_term posns, rows are term posns
152
+ # This shows every possible term diff
153
+ #
154
+ # Example:
155
+ # prior_term = array([[0, 4],[0, 4])
156
+ # term = array([[1, 2, 3],[1, 2, 3]])
157
+ #
158
+ #
159
+ # posn_diffs =
160
+ #
161
+ # array([[ term[0] - prior_term[0], term[0] - prior_term[1] ],
162
+ # [ term[1] - prior_term[0], ...
163
+ # [ term[2] - prior_term[0], ...
164
+ #
165
+ # or in our example
166
+ #
167
+ # array([[ 1, -3],
168
+ # [ 2, -2],
169
+ # [ 3, -1]])
170
+ #
171
+ # We care about all locations where posn == slop (or perhaps <= slop)
172
+ # that is term is slop away from prior_term. Usually slop == 1 (ie 1 posn away)
173
+ # for normal phrase matching
174
+ #
175
+ posn_diffs = term[:, :, np.newaxis] - prior_term[:, np.newaxis, :]
176
+ logger.info(f"posn_diffs took {perf_counter() - start:.4f} seconds")
177
+
178
+ # For > 2 terms, we need to connect a third term by making prior_term = term
179
+ # and repeating
180
+ #
181
+ # BUT
182
+ # we only want those parts of term that are adjacent to prior_term
183
+ # before continuing, so we don't accidentally get a partial phrase
184
+ # so we need to make sure to
185
+ # Pad out any rows in 'term' where posn diff != slop
186
+ # so they're not considered on subsequent iterations
187
+ term_mask = np.any(posn_diffs == 1, axis=2)
188
+ term[~term_mask] = -100
189
+ logger.info(f"term mask took {perf_counter() - start:.4f} seconds")
190
+
191
+ # Count how many times the row term is 1 away from the col term
192
+ per_doc_diffs = np.sum(posn_diffs == slop, axis=1, dtype=np.int8)
193
+ logger.info(f"count slops took {perf_counter() - start:.4f} seconds")
194
+
195
+ # Doc-wise sum to get a 'term freq' for the prior_term - term bigram
196
+ bigram_freqs = np.sum(per_doc_diffs == slop, axis=1)
197
+ if is_same_term:
198
+ satisfies_slop = per_doc_diffs == slop
199
+ consecutive_ones = satisfies_slop[:, 1:] & satisfies_slop[:, :-1]
200
+ consecutive_ones = np.sum(consecutive_ones, axis=1)
201
+ # ceiling divide?
202
+ # Really these show up as
203
+ # 1 1 1 0 1
204
+ # we need to treat the 2nd consecutive 1 as 'not a match'
205
+ # and also update 'term' to not include it
206
+ bigram_freqs -= -np.floor_divide(consecutive_ones, -2)
207
+ logger.info(f"same terms took {perf_counter() - start:.4f} seconds")
208
+
209
+ # Last loop, bigram_freqs is the full phrase term freq
210
+
211
+ # Update mask to eliminate any non-matches
212
+ phrase_freqs[to_compute] = bigram_freqs
213
+
214
+ # Should only keep positions of 'prior term' that are adjacent to the
215
+ # one prior to it...
216
+ prior_term = term
217
+ logger.info(f"Loop took {perf_counter() - start:.4f} seconds")
218
+
219
+ return phrase_freqs
220
+
221
+
@@ -0,0 +1,147 @@
1
+ """Phrase search by dumb numpy position differences subtraction, bigram by bigram."""
2
+ import numpy as np
3
+ from typing import List
4
+
5
+
6
+ def vstack_with_mask(posns: List[np.ndarray], phrase_freqs: np.ndarray, width: int = 0, pad: int = -100):
7
+ """Vertically stack arrays, only accepting those that fit within width.
8
+
9
+ Parameters
10
+ ----------
11
+ posns: list of np.ndarray term positions for a given term across multiple docs
12
+ width: int, max number of positions we accept
13
+ pad: int, value to pad with when no position present
14
+ phrase_freqs: np.ndarray, phrase freqs for each doc.
15
+ -1 indicates "not yet processed" 0 means "defiiitely not a match"
16
+
17
+ Returns
18
+ -------
19
+ vstacked: np.ndarray, vertically stacked arrays of those <= width
20
+ mask: np.ndarray, boolean mask of which arrays were accepted (so you can recompute them a different way)
21
+ """
22
+ vstacked = np.zeros((len(posns), width), dtype=posns[0].dtype) + pad
23
+ if not isinstance(posns[0], np.ndarray):
24
+ raise TypeError("posns must be a list of np.ndarrays")
25
+ for idx, array in enumerate(posns):
26
+ if len(array) >= width:
27
+ phrase_freqs[idx] = -2 # Mark as skip this round
28
+ else:
29
+ vstacked[idx, :len(array)] = array
30
+ return vstacked
31
+
32
+
33
+ def stack_term_posns(term_posns: List[np.ndarray], phrase_freqs: np.ndarray, width: int = 10):
34
+ # Pad for easy difference computation
35
+ keep_term_posns = []
36
+ # keep_mask = np.ones(len(self), dtype=bool)
37
+ for term_posn in term_posns:
38
+ this_term_posns = vstack_with_mask(term_posn, phrase_freqs, width=width)
39
+ keep_term_posns.append(this_term_posns)
40
+ return keep_term_posns
41
+
42
+
43
+ def _compute_phrase_freqs(term_posns, phrase_freqs: np.ndarray, slop=1):
44
+
45
+ to_compute = phrase_freqs == -1
46
+
47
+ if not np.any(to_compute):
48
+ return phrase_freqs
49
+
50
+ # Only examine masked
51
+ term_posns = [term_posn[to_compute] for term_posn in term_posns]
52
+
53
+ prior_term = term_posns[0]
54
+ for term in term_posns[1:]:
55
+ is_same_term = (term.shape == prior_term.shape) and np.all(term == prior_term)
56
+
57
+ # Compute positional differences
58
+ #
59
+ # Each row of posn_diffs is a term posn diff matrix
60
+ # Where columns are prior_term posns, rows are term posns
61
+ # This shows every possible term diff
62
+ #
63
+ # Example:
64
+ # prior_term = array([[0, 4],[0, 4])
65
+ # term = array([[1, 2, 3],[1, 2, 3]])
66
+ #
67
+ #
68
+ # posn_diffs =
69
+ #
70
+ # array([[ term[0] - prior_term[0], term[0] - prior_term[1] ],
71
+ # [ term[1] - prior_term[0], ...
72
+ # [ term[2] - prior_term[0], ...
73
+ #
74
+ # or in our example
75
+ #
76
+ # array([[ 1, -3],
77
+ # [ 2, -2],
78
+ # [ 3, -1]])
79
+ #
80
+ # We care about all locations where posn == slop (or perhaps <= slop)
81
+ # that is term is slop away from prior_term. Usually slop == 1 (ie 1 posn away)
82
+ # for normal phrase matching
83
+ #
84
+ posn_diffs = term[:, :, np.newaxis] - prior_term[:, np.newaxis, :]
85
+
86
+ # For > 2 terms, we need to connect a third term by making prior_term = term
87
+ # and repeating
88
+ #
89
+ # BUT
90
+ # we only want those parts of term that are adjacent to prior_term
91
+ # before continuing, so we don't accidentally get a partial phrase
92
+ # so we need to make sure to
93
+ # Pad out any rows in 'term' where posn diff != slop
94
+ # so they're not considered on subsequent iterations
95
+ term_mask = np.any(posn_diffs == 1, axis=2)
96
+ term[~term_mask] = -100
97
+
98
+ # Count how many times the row term is 1 away from the col term
99
+ per_doc_diffs = np.sum(posn_diffs == slop, axis=1, dtype=np.int8)
100
+
101
+ # Doc-wise sum to get a 'term freq' for the prior_term - term bigram
102
+ bigram_freqs = np.sum(per_doc_diffs == slop, axis=1)
103
+ if is_same_term:
104
+ satisfies_slop = per_doc_diffs == slop
105
+ consecutive_ones = satisfies_slop[:, 1:] & satisfies_slop[:, :-1]
106
+ consecutive_ones = np.sum(consecutive_ones, axis=1)
107
+ # ceiling divide?
108
+ # Really these show up as
109
+ # 1 1 1 0 1
110
+ # we need to treat the 2nd consecutive 1 as 'not a match'
111
+ # and also update 'term' to not include it
112
+ bigram_freqs -= -np.floor_divide(consecutive_ones, -2)
113
+
114
+ # Last loop, bigram_freqs is the full phrase term freq
115
+
116
+ # Update mask to eliminate any non-matches
117
+ phrase_freqs[to_compute] = bigram_freqs
118
+
119
+ # Should only keep positions of 'prior term' that are adjacent to the
120
+ # one prior to it...
121
+ prior_term = term
122
+
123
+ return phrase_freqs
124
+
125
+
126
+ def compute_phrase_freqs(term_posns, phrase_freqs, slop=1, width=10):
127
+ """Compute phrase freq using matrix-diff method for docs up to width posns. Skip others.
128
+
129
+ Parameters
130
+ ----------
131
+ term_posns: list of np.ndarray term positions for a given term across multiple docs
132
+ phrase_freqs: np.ndarray, phrase freqs for each doc present in term_posns
133
+
134
+ Returns
135
+ -------
136
+ phrase_freqs: np.ndarray, phrase freqs for each doc present in mask
137
+
138
+ See Also
139
+ --------
140
+ Colab notebook: https://colab.research.google.com/drive/1NRxeO8Ya8jSlFP5YwZaGh1-43kDH4OXG?authuser=1#scrollTo=5JZV8svpauYB
141
+ """
142
+ if len(term_posns[0]) != len(phrase_freqs):
143
+ raise ValueError("term_posns and phrase_freqs must be same length")
144
+ stacked = stack_term_posns(term_posns, phrase_freqs, width=width)
145
+ phrase_freqs = _compute_phrase_freqs(stacked, phrase_freqs, slop=slop)
146
+ phrase_freqs[phrase_freqs == -2] = -1
147
+ return phrase_freqs
@@ -0,0 +1,230 @@
1
+ import numpy as np
2
+ from typing import List
3
+
4
+
5
+ def advance_to_scan(arr, target, next_start, idx):
6
+ while idx < next_start and not arr[idx] > target:
7
+ idx += 1
8
+ return idx
9
+
10
+
11
+ def advance_after_binsearch(arr, target, next_start, idx):
12
+ """Scan arr from idx until just after target. Return next_start if exhausted.
13
+
14
+ Will manually create a skip index if next_start - idx is large
15
+
16
+ arr from idx -> next_start should be sorted
17
+
18
+ """
19
+ beg = idx
20
+ end = next_start - 1
21
+ space = end - beg
22
+ if space < 1000:
23
+ advance_to_scan(arr, target, next_start, idx)
24
+
25
+ # print(f"Binsearching {target} in arr[{beg}:{end}] = {arr[beg:end]}")
26
+ if arr[idx] > target:
27
+ # print("Already past, returning idx")
28
+ return idx
29
+
30
+ mid = None
31
+ while True:
32
+ space = end - beg
33
+ if space <= 1000:
34
+ return advance_to_scan(arr, target, next_start, beg)
35
+
36
+ mid = beg + (space // 2)
37
+ if arr[mid] == target:
38
+ # print(f"Found exact match, returning {mid + 1}")
39
+ return mid + 1
40
+ elif arr[mid] > target and arr[mid - 1] <= target:
41
+ # print(f"Found one past, returning {mid}")
42
+ return mid
43
+ elif arr[mid] < target:
44
+ # print(f"Moving beg to {mid}")
45
+ # print(f" end is {end}")
46
+ # print(f"arr[{mid}] is {arr[mid]}")
47
+ beg = mid
48
+ else: # elif arr[mid] > target:
49
+ # print(f" beg is {beg}")
50
+ # print(f"Moving end to {mid}")
51
+ # print(f"arr[{mid}] is {arr[mid]}")
52
+ end = mid
53
+
54
+ # print(f"Exhausted binsearch, returning {next_start}")
55
+ return next_start
56
+
57
+
58
+ def scan_merge(prior_posns: np.ndarray,
59
+ prior_starts: np.ndarray,
60
+ next_posns: np.ndarray,
61
+ next_starts: np.ndarray,
62
+ scan_algo=advance_to_scan,
63
+ slop=1):
64
+ """Merge two term position lists together into a single list of bigrams.
65
+
66
+ Each position list is a flattened representation of multiple docs, ie
67
+
68
+ prior_posns: [0,2,1,4,5]
69
+ prior_starts: [0,2]
70
+
71
+ Points to the first position of each doc in the prior_posns list for this term
72
+
73
+ (same for next_posns)
74
+
75
+ This is intentionally written to be naive, C-like for later porting to C
76
+
77
+ See notebook:
78
+ https://colab.research.google.com/drive/10zjUYHGtwMfJMPXz-BHwHe_v6j4MUZzm?authuser=1#scrollTo=W6HuiFGaYCiX
79
+
80
+ """
81
+ next_idx = 0
82
+ cont_nexts = []
83
+ cont_next = []
84
+ bigram_freq = 0
85
+ bigram_freqs = []
86
+ last_prior = -2
87
+ next_start = next_starts[0]
88
+ prior_start = prior_starts[0]
89
+ start_idx = 0
90
+ prior_idx = 0
91
+ while prior_idx < len(prior_posns):
92
+ # Scan next until just past prior
93
+ prior = prior_posns[prior_idx]
94
+ next_idx = scan_algo(next_posns, target=prior,
95
+ next_start=next_start,
96
+ idx=next_idx)
97
+
98
+ # Reset to head of next location,
99
+ # Re-advance next
100
+ if next_idx >= next_start or prior_idx >= prior_start:
101
+ next_idx = next_start
102
+ prior_idx = prior_start
103
+ last_prior = -2
104
+
105
+ start_idx += 1
106
+ if start_idx >= len(next_starts):
107
+ break
108
+ next_start = next_starts[start_idx]
109
+ prior_start = prior_starts[start_idx]
110
+
111
+ # Save and reset output
112
+ cont_nexts.append(np.array(cont_next))
113
+ bigram_freqs.append(bigram_freq)
114
+
115
+ cont_next = []
116
+ bigram_freq = 0
117
+
118
+ prior = prior_posns[prior_idx]
119
+ next_idx = scan_algo(next_posns, target=prior,
120
+ next_start=next_start, idx=next_idx)
121
+
122
+ if next_idx >= next_start:
123
+ continue
124
+
125
+ next_posn = next_posns[next_idx]
126
+ # Check if within slop
127
+ # And is not double counting 0->1->2 (only happens if prior/next identical)
128
+ dist = next_posn - prior
129
+ if dist <= slop and prior != (last_prior + 1):
130
+ cont_next.append(next_posn)
131
+ bigram_freq += 1
132
+ last_prior = prior
133
+
134
+ prior_idx += 1
135
+
136
+ # Save last output
137
+ cont_nexts.append(np.array(cont_next))
138
+ bigram_freqs.append(bigram_freq)
139
+ return bigram_freqs, cont_nexts
140
+
141
+
142
+ def scan_merge_inplace(prior_posns: np.ndarray,
143
+ prior_starts: np.ndarray,
144
+ next_posns: np.ndarray,
145
+ next_starts: np.ndarray,
146
+ scan_algo=advance_to_scan,
147
+ slop=1):
148
+ """Merge two term position lists together into a single list of bigrams.
149
+
150
+ Same as scan_merge, but doesn't need a dynamic array output, instead
151
+ overwrites prior* buffers
152
+ """
153
+ next_idx = 0
154
+ bigram_freq = 0
155
+ bigram_freqs = [] # this could also be preallocated array output
156
+ last_prior = -2
157
+ output_idx = 0
158
+ next_start = next_starts[0]
159
+ prior_start = prior_starts[0]
160
+ start_idx = 0
161
+ prior_idx = 0
162
+
163
+ while prior_idx < prior_starts[-1]:
164
+ # Scan next until just past p
165
+ prior = prior_posns[prior_idx]
166
+ next_idx = scan_algo(next_posns,
167
+ target=prior,
168
+ next_start=next_start,
169
+ idx=next_idx)
170
+
171
+ # Reset to head of next location,
172
+ # Re-advance next
173
+ if next_idx >= next_start or prior_idx >= prior_start:
174
+ next_idx = next_start
175
+ prior_idx = prior_start
176
+ last_prior = -2
177
+
178
+ prior_starts[start_idx] = output_idx
179
+
180
+ start_idx += 1
181
+ if start_idx >= len(next_starts):
182
+ break
183
+ next_start = next_starts[start_idx]
184
+ prior_start = prior_starts[start_idx]
185
+
186
+ # Save and reset output
187
+ bigram_freqs.append(bigram_freq)
188
+
189
+ bigram_freq = 0
190
+
191
+ prior = prior_posns[prior_idx]
192
+ next_idx = scan_algo(next_posns,
193
+ target=prior,
194
+ next_start=next_start,
195
+ idx=next_idx)
196
+ if next_idx >= next_start:
197
+ continue
198
+
199
+ next_posn = next_posns[next_idx]
200
+ # Check if within slop
201
+ # And is not double counting 0->1->2 (only happens if prior/next identical)
202
+ dist = next_posn - prior
203
+ if dist <= slop and prior != (last_prior + 1):
204
+ prior_posns[output_idx] = next_posn
205
+ output_idx += 1
206
+ bigram_freq += 1
207
+ last_prior = prior
208
+
209
+ prior_idx += 1
210
+
211
+ # Save last output
212
+ bigram_freqs.append(bigram_freq)
213
+ if start_idx < len(next_starts):
214
+ prior_starts[start_idx] = output_idx
215
+
216
+ return bigram_freqs, prior_posns, prior_starts
217
+
218
+
219
+ def scan_merge_bigram(prior_posns: List, next_posns: List, slop=1):
220
+ prior_starts = np.cumsum([lst.shape[0] for lst in prior_posns])
221
+ next_starts = np.cumsum([lst.shape[0] for lst in next_posns])
222
+ prior_posns = np.concatenate(prior_posns)
223
+ next_posns = np.concatenate(next_posns)
224
+
225
+ result = scan_merge(
226
+ prior_posns, prior_starts,
227
+ next_posns, next_starts,
228
+ slop=slop
229
+ )
230
+ return result
@@ -0,0 +1,183 @@
1
+ """Phrase search as wide span detection."""
2
+ import numpy as np
3
+
4
+
5
+ def advance_after(arr, target, idx, next_start):
6
+ """Scan arr from idx until just after target. Return next_start if exhausted.
7
+
8
+ arr from idx -> next_start should be sorted
9
+
10
+ """
11
+ while idx < next_start and not arr[idx] > target:
12
+ idx += 1
13
+ return idx
14
+
15
+
16
+ def next_wide_span(posns, starts, start_idx, idxs):
17
+ """Given N terms find possible acceptable phrases for a slop.
18
+
19
+ Parameters:
20
+ -----------
21
+ posns - N Numpy arrays, holding all matching document term positions
22
+ starts - each term N start boundaries, for all D docs
23
+ start_idx - where we are in the starts (aka doc positon)
24
+ idxs - where we are in each term
25
+
26
+ A wide span begins with the first term in a phrase, and finds the
27
+ first location after that where the last term occurs. Acceptable
28
+ wide spans are
29
+
30
+ Query - "foo bar baz"
31
+ posn 1 2 3 4
32
+ t1 foo foo
33
+ t2 bar
34
+ t3 baz
35
+
36
+ posn 1 2 3 ... N -1 N
37
+ t1 foo foo
38
+ t2 bar bar
39
+ t3 baz
40
+
41
+ To confirm these wide spans are acceptable phrase matches, we have
42
+ to walk backwards from the end and see if phrases occur in order within span
43
+
44
+ After consuming this wide span, the next phrase candidate can be had by advancing
45
+ T1 to N+1th position.
46
+ """
47
+ # 30100000 20.651 0.000 33.100 0.000 wide_spans.py:16(next_wide_span)
48
+ # in-place
49
+ # 30100000 18.470 0.000 29.952 0.000 wide_spans.py:16(next_wide_span)
50
+ # 30100000 18.078 0.000 29.297 0.000 wide_spans.py:16(next_wide_span)
51
+
52
+ span_idx = idxs[0]
53
+ target = posns[0][span_idx]
54
+
55
+ term_idx = 1
56
+ for term in posns[1:]:
57
+ next_start = starts[term_idx][start_idx]
58
+ span_idx = advance_after(posns[term_idx],
59
+ target=target,
60
+ idx=idxs[term_idx],
61
+ next_start=next_start)
62
+ if span_idx == next_start:
63
+ return [start[start_idx] for start in starts]
64
+
65
+ target = posns[term_idx][span_idx]
66
+ idxs[term_idx] = span_idx
67
+ term_idx += 1
68
+
69
+ return idxs
70
+
71
+
72
+ def get_back_span(posns, starts, start_idx, idxs):
73
+ """Given N terms find minimal back span.
74
+
75
+ Parameters:
76
+ ----------
77
+ posns - N Numpy arrays, holding all matching document term positions
78
+ starts - each term N start boundaries, for all D docs
79
+ start_idx - where we are in the starts (aka doc positon)
80
+ idxs - where we are in each term
81
+
82
+ Returns:
83
+ --------
84
+ Shortest span backwards, plus positional diff
85
+
86
+ A back span for 'foo bar baz' of a wide span, starts with end position
87
+ then goes backwards
88
+
89
+ Query - "foo bar baz"
90
+ posn 1 2 3 4
91
+ t1 foo* foo
92
+ t2 bar*
93
+ t3 baz*
94
+
95
+ posn 1 2 3 ... N -1 N
96
+ t1 foo* foo <- scan here
97
+ t2 bar* bar <- scan here
98
+ t3 baz
99
+
100
+ Walking backwards at idxs[-1] we scan idxs[-2] to find
101
+ the first occurence (going backwards)
102
+ """
103
+ last_posn = posns[-1][idxs[-1]]
104
+ back_span_idxs = [idxs[-1]]
105
+
106
+ term_idx = len(posns) - 2
107
+ for posn in posns[::-1][1:]:
108
+ curr_posn = posn[idxs[term_idx]]
109
+ while curr_posn < last_posn:
110
+ idxs[term_idx] += 1
111
+ if idxs[term_idx] >= starts[term_idx][start_idx]:
112
+ break
113
+ curr_posn = posn[idxs[term_idx]]
114
+ back_span_idxs.append(idxs[term_idx] - 1)
115
+ last_posn = posn[back_span_idxs[-1]]
116
+ term_idx -= 1
117
+ back_span_idxs = back_span_idxs[::-1]
118
+ return back_span_idxs, (posns[-1][back_span_idxs[-1]] - posns[0][back_span_idxs[0]])
119
+
120
+
121
+ def collect_span(posns, starts, start_idx, idxs, acceptable):
122
+ # Advance 0th beyond
123
+ beg_posn = posns[0][idxs[0]]
124
+ last_end_posn = posns[-1][idxs[-1]]
125
+ if (last_end_posn - beg_posn) <= acceptable:
126
+ return 1
127
+ else:
128
+ back_span, posn_diff = get_back_span(posns, starts, start_idx, idxs)
129
+ if posn_diff <= acceptable:
130
+ return 1
131
+ return 0
132
+
133
+
134
+ def all_wide_spans_of_slop(posns, starts, slop=1):
135
+ """Collect all wide spans of multiple docs for all posn terms.
136
+
137
+ Parameters:
138
+ -----------
139
+ posns - list of single concatenated array of all doc posns for each term
140
+ starts - doc start posn boundaries per term
141
+ slop - allowed (in order) moves of term
142
+
143
+ Returns:
144
+ --------
145
+ phrase frequency of terms in posns satisfying given slop
146
+
147
+ """
148
+ start_idx = 0
149
+ span_idxs = np.zeros(len(starts), dtype=np.uint32)
150
+ phrases_per_doc = np.zeros(len(starts[0]), dtype=np.uint32)
151
+ # Acceptable posn difference (<=)
152
+ # 2-gram 3-gram
153
+ # slop
154
+ #
155
+ # 1 1 2
156
+ # 2 2 3
157
+ acceptable = len(posns) + (slop - 2)
158
+ first_term_start = starts[0]
159
+ num_starts = len(starts[0])
160
+ while True:
161
+ if start_idx >= num_starts:
162
+ break
163
+ elif span_idxs[0] == first_term_start[start_idx]:
164
+ span_idxs = [start[start_idx] for start in starts]
165
+ start_idx += 1
166
+ else:
167
+ span_idxs = next_wide_span(posns, starts, start_idx, span_idxs)
168
+ if span_idxs[0] == first_term_start[start_idx]:
169
+ # New doc / next start
170
+ continue
171
+
172
+ last_end_posn = posns[-1][span_idxs[-1]]
173
+ phrases_per_doc[start_idx] += collect_span(posns, starts, start_idx, span_idxs, acceptable)
174
+
175
+ span_idxs[0] = advance_after(posns[0],
176
+ target=last_end_posn,
177
+ idx=span_idxs[0],
178
+ next_start=first_term_start[start_idx])
179
+
180
+ if span_idxs[0] == first_term_start[-1]:
181
+ break
182
+
183
+ return phrases_per_doc
File without changes
@@ -0,0 +1,77 @@
1
+ import numbers
2
+ from scipy.sparse import csr_matrix
3
+ import numpy as np
4
+
5
+
6
+ class RowViewableMatrix:
7
+ """A slicable matrix that can return views without copying."""
8
+
9
+ def __init__(self, csr_mat: csr_matrix, rows: np.ndarray = None):
10
+ self.mat = csr_mat
11
+ self.col_cache = {}
12
+ self.cols_cached = []
13
+ if rows is None:
14
+ self.rows = np.arange(self.mat.shape[0])
15
+ elif isinstance(rows, numbers.Integral):
16
+ self.rows = np.array([rows])
17
+ else:
18
+ self.rows = rows
19
+
20
+ def slice(self, keys):
21
+ return RowViewableMatrix(self.mat, self.rows[keys])
22
+
23
+ def __setitem__(self, keys, values):
24
+ # Replace nan with 0
25
+ self.col_cache = {}
26
+ self.cols_cached = []
27
+ actual_keys = self.rows[keys]
28
+ if isinstance(actual_keys, numbers.Integral):
29
+ self.mat[actual_keys] = values
30
+ elif len(actual_keys) > 0:
31
+ self.mat[actual_keys] = values
32
+
33
+ def copy_row_at(self, row):
34
+ return self.mat[self.rows[row]]
35
+
36
+ def copy(self):
37
+ return RowViewableMatrix(self.mat.copy(), self.rows.copy())
38
+
39
+ def sum(self, axis=0):
40
+ return self.mat[self.rows].sum(axis=axis)
41
+
42
+ def copy_col_at(self, col):
43
+ if col not in self.col_cache:
44
+ self.col_cache[col] = self.mat[self.rows, col]
45
+ self.cols_cached.append(col)
46
+ if len(self.cols_cached) > 10:
47
+ del self.col_cache[self.cols_cached.pop(0)]
48
+ return self.col_cache[col]
49
+
50
+ def __getitem__(self, key):
51
+ if isinstance(key, numbers.Integral):
52
+ return self.copy_row_at(key)
53
+ else:
54
+ return self.slice(key)
55
+
56
+ @property
57
+ def nbytes(self):
58
+ return self.mat.data.nbytes + \
59
+ self.mat.indptr.nbytes + \
60
+ self.mat.indices.nbytes + \
61
+ self.rows.nbytes
62
+
63
+ @property
64
+ def shape(self):
65
+ return (len(self.rows), self.mat.shape[1])
66
+
67
+ def resize(self, shape):
68
+ self.mat.resize(shape)
69
+
70
+ def __len__(self):
71
+ return len(self.rows)
72
+
73
+ def __repr__(self):
74
+ return f"RowViewableMatrix({repr(self.mat)}, {repr(self.rows)})"
75
+
76
+ def __str__(self):
77
+ return f"RowViewableMatrix({str(self.mat)}, {str(self.rows)})"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.13a0
3
+ Version: 0.0.14
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -12,6 +12,14 @@ searcharray.egg-info/dependency_links.txt
12
12
  searcharray.egg-info/entry_points.txt
13
13
  searcharray.egg-info/requires.txt
14
14
  searcharray.egg-info/top_level.txt
15
+ searcharray/phrase/__init__.py
16
+ searcharray/phrase/middle_out.py
17
+ searcharray/phrase/padded_arrays.py
18
+ searcharray/phrase/posn_diffs.py
19
+ searcharray/phrase/scan_merge.py
20
+ searcharray/phrase/wide_spans.py
21
+ searcharray/utils/__init__.py
22
+ searcharray/utils/row_viewable_matrix.py
15
23
  test/test_extension_array.py
16
24
  test/test_msmarco.py
17
25
  test/test_phrase_matches.py
@@ -6,7 +6,7 @@ https://github.com/pypa/sampleproject
6
6
  """
7
7
 
8
8
  # Always prefer setuptools over distutils
9
- from setuptools import setup
9
+ from setuptools import setup, find_packages
10
10
  import pathlib
11
11
 
12
12
  here = pathlib.Path(__file__).parent.resolve()
@@ -36,7 +36,7 @@ setup(
36
36
  # For a discussion on single-sourcing the version across setup.py and the
37
37
  # project code, see
38
38
  # https://packaging.python.org/guides/single-sourcing-package-version/
39
- version="0.0.13a", # Required
39
+ version="0.0.14", # Required
40
40
  # This is a one-line description or tagline of what your project does. This
41
41
  # corresponds to the "Summary" metadata field:
42
42
  # https://packaging.python.org/specifications/core-metadata/#summary
@@ -114,7 +114,7 @@ setup(
114
114
  #
115
115
  # py_modules=["my_module"],
116
116
  #
117
- packages=["searcharray"], # Required
117
+ packages=find_packages(),
118
118
  # Specify which Python versions you support. In contrast to the
119
119
  # 'Programming Language' classifiers above, 'pip install' will check this
120
120
  # and refuse to install the project if the version does not match. See
File without changes
File without changes