searcharray 0.0.73__cp313-cp313-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- searcharray/__init__.py +2 -0
- searcharray/bm25/__init__.py +1 -0
- searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
- searcharray/bm25/bm25.pyi +11 -0
- searcharray/bm25/bm25.pyx +41 -0
- searcharray/indexing.py +342 -0
- searcharray/phrase/__init__.py +0 -0
- searcharray/phrase/bigram_freqs.py +307 -0
- searcharray/phrase/memmap_arrays.py +208 -0
- searcharray/phrase/middle_out.py +553 -0
- searcharray/phrase/spans.py +187 -0
- searcharray/postings.py +708 -0
- searcharray/roaringish/__init__.py +7 -0
- searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/intersect.pyi +25 -0
- searcharray/roaringish/intersect.pyx +390 -0
- searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/merge.pyi +16 -0
- searcharray/roaringish/merge.pyx +232 -0
- searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/popcount.pyi +36 -0
- searcharray/roaringish/popcount.pyx +278 -0
- searcharray/roaringish/roaringish.py +298 -0
- searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/roaringish_ops.pyx +98 -0
- searcharray/roaringish/scatter_assign.h +31 -0
- searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/search.pyi +25 -0
- searcharray/roaringish/search.pyx +140 -0
- searcharray/roaringish/snp_ops.pxd +13 -0
- searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/spans.pyx +330 -0
- searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/unique.pyi +6 -0
- searcharray/roaringish/unique.pyx +145 -0
- searcharray/similarity.py +92 -0
- searcharray/solr.py +355 -0
- searcharray/term_dict.py +59 -0
- searcharray/utils/__init__.py +0 -0
- searcharray/utils/bitcount.py +34 -0
- searcharray/utils/mat_set.py +170 -0
- searcharray/utils/row_viewable_matrix.py +91 -0
- searcharray/utils/sort.py +45 -0
- searcharray-0.0.73.dist-info/METADATA +245 -0
- searcharray-0.0.73.dist-info/RECORD +48 -0
- searcharray-0.0.73.dist-info/WHEEL +5 -0
- searcharray-0.0.73.dist-info/entry_points.txt +2 -0
- searcharray-0.0.73.dist-info/top_level.txt +1 -0
searcharray/postings.py
ADDED
|
@@ -0,0 +1,708 @@
|
|
|
1
|
+
"""Tokenized, searchable text as a pandas dtype."""
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import numbers
|
|
4
|
+
from pandas.api.extensions import ExtensionDtype, ExtensionArray, register_extension_dtype
|
|
5
|
+
from pandas.api.types import is_list_like
|
|
6
|
+
from pandas.api.extensions import take
|
|
7
|
+
import json
|
|
8
|
+
from collections import Counter
|
|
9
|
+
import warnings
|
|
10
|
+
import logging
|
|
11
|
+
from typing import List, Union, Optional, Iterable
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
from searcharray.phrase.middle_out import PosnBitArray
|
|
16
|
+
from searcharray.similarity import Similarity, default_bm25
|
|
17
|
+
from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
|
|
18
|
+
from searcharray.term_dict import TermMissingError
|
|
19
|
+
from searcharray.roaringish.roaringish_ops import as_dense
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
# When running in pytest
|
|
25
|
+
import sys # noqa
|
|
26
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
27
|
+
handler.setLevel(logging.ERROR)
|
|
28
|
+
formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
|
|
29
|
+
handler.setFormatter(formatter)
|
|
30
|
+
logger.addHandler(handler)
|
|
31
|
+
logger.setLevel(logging.ERROR)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def bytes_to_human_readable(num_bytes):
|
|
35
|
+
"""
|
|
36
|
+
Convert a number of bytes into a human-readable format (KB, MB, GB, etc.)
|
|
37
|
+
|
|
38
|
+
:param num_bytes: Number of bytes to be converted
|
|
39
|
+
:type num_bytes: int or float
|
|
40
|
+
:return: Human-readable string
|
|
41
|
+
:rtype: str
|
|
42
|
+
"""
|
|
43
|
+
# Define the suffixes for byte conversion
|
|
44
|
+
suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
|
|
45
|
+
|
|
46
|
+
if num_bytes == 0:
|
|
47
|
+
return "0B"
|
|
48
|
+
|
|
49
|
+
i = 0
|
|
50
|
+
while num_bytes >= 1024 and i < len(suffixes) - 1:
|
|
51
|
+
num_bytes /= 1024.
|
|
52
|
+
i += 1
|
|
53
|
+
|
|
54
|
+
return f"{num_bytes:.2f} {suffixes[i]}"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class Terms:
|
|
58
|
+
"""An indexed search doc - a single bag of tokenized words and positions."""
|
|
59
|
+
|
|
60
|
+
def __init__(self,
|
|
61
|
+
postings,
|
|
62
|
+
doc_len: int = 0,
|
|
63
|
+
posns: Optional[dict] = None,
|
|
64
|
+
encoded=False):
|
|
65
|
+
self.postings = postings
|
|
66
|
+
self.posns = None
|
|
67
|
+
self.encoded = encoded
|
|
68
|
+
self.doc_len = doc_len
|
|
69
|
+
self.posns = posns
|
|
70
|
+
|
|
71
|
+
def _validate_posns(self):
|
|
72
|
+
# (For testing/assertions) - Confirm every term in positions also in postings
|
|
73
|
+
if self.posns is None:
|
|
74
|
+
return
|
|
75
|
+
for term in self.posns:
|
|
76
|
+
if term not in self.postings:
|
|
77
|
+
raise ValueError(f"Term {term} in positions but not in postings. ")
|
|
78
|
+
|
|
79
|
+
def termfreq(self, token):
|
|
80
|
+
return self.postings[token]
|
|
81
|
+
|
|
82
|
+
def terms(self):
|
|
83
|
+
return self.postings.items()
|
|
84
|
+
|
|
85
|
+
def positions(self, term=None):
|
|
86
|
+
if self.posns is None:
|
|
87
|
+
return {}
|
|
88
|
+
if term is None:
|
|
89
|
+
posns = self.posns.items()
|
|
90
|
+
else:
|
|
91
|
+
posns = self.posns[term]
|
|
92
|
+
return posns
|
|
93
|
+
|
|
94
|
+
def raw_positions(self, term_dict, term=None):
|
|
95
|
+
if self.posns is None:
|
|
96
|
+
return {}
|
|
97
|
+
if term is None:
|
|
98
|
+
posns = [(term_dict.get_term_id(term), posns) for term, posns in self.posns.items()]
|
|
99
|
+
else:
|
|
100
|
+
posns = [(term_dict.get_term_id(term), self.posns[term])]
|
|
101
|
+
return posns
|
|
102
|
+
|
|
103
|
+
def tf_to_dense(self, term_dict):
|
|
104
|
+
"""Convert to a dense vector of term frequencies."""
|
|
105
|
+
dense = np.zeros(len(term_dict))
|
|
106
|
+
for term, freq in self.terms():
|
|
107
|
+
dense[term_dict.get_term_id(term)] = freq
|
|
108
|
+
return dense
|
|
109
|
+
|
|
110
|
+
def __len__(self):
|
|
111
|
+
return len(self.postings)
|
|
112
|
+
|
|
113
|
+
def __repr__(self):
|
|
114
|
+
posting_keys = set(self.postings.keys())
|
|
115
|
+
rval = f"Terms({posting_keys})"
|
|
116
|
+
return rval
|
|
117
|
+
|
|
118
|
+
def __str__(self):
|
|
119
|
+
return repr(self)
|
|
120
|
+
|
|
121
|
+
def __eq__(self, other):
|
|
122
|
+
# Flip to the other implementation if we're comparing to a SearchArray
|
|
123
|
+
# to get a boolean array back
|
|
124
|
+
if isinstance(other, SearchArray):
|
|
125
|
+
return other == self
|
|
126
|
+
same_postings = isinstance(other, Terms) and self.postings == other.postings
|
|
127
|
+
if same_postings and self.doc_len == other.doc_len:
|
|
128
|
+
return True
|
|
129
|
+
|
|
130
|
+
def __lt__(self, other):
|
|
131
|
+
# return isinstance(other, Terms) and hash(self) < hash(other)
|
|
132
|
+
keys_both = set(self.postings.keys()).union(set(other.postings.keys()))
|
|
133
|
+
# Sort lexically
|
|
134
|
+
keys_both = sorted(keys_both)
|
|
135
|
+
|
|
136
|
+
# Iterate as if these are two vectors of the same large dimensional vector sparse
|
|
137
|
+
for key in keys_both:
|
|
138
|
+
lhs_val = 0
|
|
139
|
+
rhs_val = 0
|
|
140
|
+
try:
|
|
141
|
+
lhs_val = self.postings[key]
|
|
142
|
+
except KeyError:
|
|
143
|
+
pass
|
|
144
|
+
|
|
145
|
+
try:
|
|
146
|
+
rhs_val = other.postings[key]
|
|
147
|
+
except KeyError:
|
|
148
|
+
pass
|
|
149
|
+
|
|
150
|
+
if lhs_val < rhs_val:
|
|
151
|
+
return True
|
|
152
|
+
elif lhs_val > rhs_val:
|
|
153
|
+
return False
|
|
154
|
+
else:
|
|
155
|
+
continue
|
|
156
|
+
return False
|
|
157
|
+
|
|
158
|
+
def __le__(self, other):
|
|
159
|
+
return self < other or self == other
|
|
160
|
+
|
|
161
|
+
def __gt__(self, other):
|
|
162
|
+
return not (self < other) and self != other
|
|
163
|
+
|
|
164
|
+
def __hash__(self):
|
|
165
|
+
return hash(json.dumps(self.postings, sort_keys=True))
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class TermsDtype(ExtensionDtype):
|
|
169
|
+
"""Pandas dtype for terms."""
|
|
170
|
+
|
|
171
|
+
name = 'tokenized_text'
|
|
172
|
+
type = Terms
|
|
173
|
+
kind = 'O'
|
|
174
|
+
|
|
175
|
+
@classmethod
|
|
176
|
+
def construct_from_string(cls, string):
|
|
177
|
+
if not isinstance(string, str):
|
|
178
|
+
raise TypeError(
|
|
179
|
+
"'construct_from_string' expects a string, got {}".format(type(string))
|
|
180
|
+
)
|
|
181
|
+
elif string == cls.name:
|
|
182
|
+
return cls()
|
|
183
|
+
else:
|
|
184
|
+
raise TypeError(
|
|
185
|
+
"Cannot construct a '{}' from '{}'".format(cls.__name__, string)
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
@classmethod
|
|
189
|
+
def construct_array_type(cls):
|
|
190
|
+
return SearchArray
|
|
191
|
+
|
|
192
|
+
def __repr__(self):
|
|
193
|
+
return 'TermsDtype()'
|
|
194
|
+
|
|
195
|
+
@property
|
|
196
|
+
def na_value(self):
|
|
197
|
+
return Terms({})
|
|
198
|
+
|
|
199
|
+
def valid_value(self, value):
|
|
200
|
+
return isinstance(value, dict) or pd.isna(value) or isinstance(value, Terms)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
register_extension_dtype(TermsDtype)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def ws_tokenizer(string):
|
|
207
|
+
if pd.isna(string):
|
|
208
|
+
return []
|
|
209
|
+
if not isinstance(string, str):
|
|
210
|
+
raise ValueError("Expected a string")
|
|
211
|
+
return string.split()
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _row_to_postings_row(doc_id, row, doc_len, term_dict, posns: PosnBitArray):
|
|
215
|
+
tfs = {}
|
|
216
|
+
labeled_posns = {}
|
|
217
|
+
for term_idx in row.cols:
|
|
218
|
+
term = term_dict.get_term(term_idx)
|
|
219
|
+
tfs[term] = 1
|
|
220
|
+
enc_term_posns = posns.doc_encoded_posns(term_idx, doc_id=doc_id)
|
|
221
|
+
labeled_posns[term] = enc_term_posns
|
|
222
|
+
|
|
223
|
+
result = Terms(tfs, posns=labeled_posns,
|
|
224
|
+
doc_len=doc_len, encoded=True)
|
|
225
|
+
return result
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
class SearchArray(ExtensionArray):
|
|
229
|
+
"""An array of tokenized text (Terms).
|
|
230
|
+
|
|
231
|
+
Use the class method `index` to create a SearchArray from a list of strings and tokenizer
|
|
232
|
+
|
|
233
|
+
"""
|
|
234
|
+
|
|
235
|
+
dtype = TermsDtype()
|
|
236
|
+
|
|
237
|
+
def __init__(self, postings, tokenizer=ws_tokenizer, avoid_copies=True):
|
|
238
|
+
# Check dtype, raise TypeError
|
|
239
|
+
if not is_list_like(postings):
|
|
240
|
+
raise TypeError("Expected list-like object, got {}".format(type(postings)))
|
|
241
|
+
|
|
242
|
+
self.avoid_copies = avoid_copies
|
|
243
|
+
self.tokenizer = tokenizer
|
|
244
|
+
self.term_mat, self.posns, \
|
|
245
|
+
self.term_dict, self.avg_doc_length, \
|
|
246
|
+
self.doc_lens = build_index_from_terms_list(postings, Terms)
|
|
247
|
+
self.corpus_size = len(self.doc_lens)
|
|
248
|
+
|
|
249
|
+
@classmethod
|
|
250
|
+
def index(cls, array: Iterable,
|
|
251
|
+
tokenizer=ws_tokenizer,
|
|
252
|
+
truncate=False,
|
|
253
|
+
batch_size=100000,
|
|
254
|
+
avoid_copies=True,
|
|
255
|
+
workers=4,
|
|
256
|
+
cache_gt_than=25,
|
|
257
|
+
data_dir: Optional[str] = None,
|
|
258
|
+
autowarm=True) -> 'SearchArray':
|
|
259
|
+
"""Index an array of strings into SearchArray using tokenizer.
|
|
260
|
+
|
|
261
|
+
Parameters
|
|
262
|
+
----------
|
|
263
|
+
array : Iterable
|
|
264
|
+
A list of strings to index
|
|
265
|
+
tokenizer : Callable
|
|
266
|
+
A function that takes a string and returns an iterable of tokens
|
|
267
|
+
truncate : bool
|
|
268
|
+
If true, truncate the tokenized strings to a maximum length (2^18-1)
|
|
269
|
+
batch_size : int
|
|
270
|
+
For tuning memory usage, the number of strings to encode at once.
|
|
271
|
+
(which will be merged into one index at the end). Higher the more memory
|
|
272
|
+
will be used.
|
|
273
|
+
avoid_copies : bool
|
|
274
|
+
If true, avoid copying the term dictionary and positions during normal
|
|
275
|
+
pandas copy operations
|
|
276
|
+
autowarm : bool
|
|
277
|
+
If true, precompute docfreq / term freqs for most common terms
|
|
278
|
+
after indexing
|
|
279
|
+
"""
|
|
280
|
+
if not is_list_like(array):
|
|
281
|
+
raise TypeError("Expected list-like object, got {}".format(type(array)))
|
|
282
|
+
|
|
283
|
+
term_mat, posns, term_dict, avg_doc_length, doc_lens =\
|
|
284
|
+
build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
|
|
285
|
+
truncate=truncate,
|
|
286
|
+
data_dir=data_dir,
|
|
287
|
+
cache_gt_than=cache_gt_than,
|
|
288
|
+
workers=workers)
|
|
289
|
+
|
|
290
|
+
if autowarm:
|
|
291
|
+
posns.warm()
|
|
292
|
+
|
|
293
|
+
postings = cls([], tokenizer=tokenizer, avoid_copies=avoid_copies)
|
|
294
|
+
postings.term_mat = term_mat
|
|
295
|
+
postings.posns = posns
|
|
296
|
+
postings.term_dict = term_dict
|
|
297
|
+
postings.avg_doc_length = avg_doc_length
|
|
298
|
+
postings.doc_lens = doc_lens
|
|
299
|
+
postings.corpus_size = len(doc_lens)
|
|
300
|
+
return postings
|
|
301
|
+
|
|
302
|
+
def warm(self):
|
|
303
|
+
self.posns.warm()
|
|
304
|
+
|
|
305
|
+
@classmethod
|
|
306
|
+
def _from_sequence(cls, scalars, dtype=None, copy=False):
|
|
307
|
+
"""Construct a new SearchArray from a sequence of scalars (PostingRow or convertible into)."""
|
|
308
|
+
if dtype is not None:
|
|
309
|
+
if not isinstance(dtype, TermsDtype):
|
|
310
|
+
return scalars
|
|
311
|
+
if isinstance(scalars, np.ndarray) and scalars.dtype == TermsDtype():
|
|
312
|
+
return cls(scalars)
|
|
313
|
+
# String types
|
|
314
|
+
elif isinstance(scalars, np.ndarray) and scalars.dtype.kind in 'US':
|
|
315
|
+
return cls(scalars)
|
|
316
|
+
# Other objects
|
|
317
|
+
elif isinstance(scalars, np.ndarray) and scalars.dtype != object:
|
|
318
|
+
return scalars
|
|
319
|
+
return cls(scalars)
|
|
320
|
+
|
|
321
|
+
def memory_usage(self, deep=False):
|
|
322
|
+
"""Return memory usage of this array in bytes."""
|
|
323
|
+
return self.nbytes
|
|
324
|
+
|
|
325
|
+
@property
|
|
326
|
+
def nbytes(self):
|
|
327
|
+
return self.term_mat.nbytes + self.posns.nbytes + self.doc_lens.nbytes + self.term_dict.nbytes
|
|
328
|
+
|
|
329
|
+
def __getitem__(self, key):
|
|
330
|
+
key = pd.api.indexers.check_array_indexer(self, key)
|
|
331
|
+
# Want to take rows of term freqs
|
|
332
|
+
if isinstance(key, numbers.Integral):
|
|
333
|
+
try:
|
|
334
|
+
rows = self.term_mat[key]
|
|
335
|
+
doc_len = self.doc_lens[key]
|
|
336
|
+
doc_id = key
|
|
337
|
+
if doc_id < 0:
|
|
338
|
+
doc_id += len(self)
|
|
339
|
+
return _row_to_postings_row(doc_id, rows[0], doc_len,
|
|
340
|
+
self.term_dict, self.posns)
|
|
341
|
+
except IndexError:
|
|
342
|
+
raise IndexError("index out of bounds")
|
|
343
|
+
else:
|
|
344
|
+
# Construct a sliced view of this array
|
|
345
|
+
sliced_tfs = self.term_mat.slice(key)
|
|
346
|
+
if not self.avoid_copies:
|
|
347
|
+
sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
|
|
348
|
+
else:
|
|
349
|
+
sliced_posns = self.posns.filter(sliced_tfs.rows)
|
|
350
|
+
|
|
351
|
+
arr = SearchArray([], tokenizer=self.tokenizer)
|
|
352
|
+
arr.term_mat = sliced_tfs
|
|
353
|
+
arr.doc_lens = self.doc_lens[key]
|
|
354
|
+
arr.posns = sliced_posns
|
|
355
|
+
arr.term_dict = self.term_dict
|
|
356
|
+
arr.avg_doc_length = self.avg_doc_length
|
|
357
|
+
arr.corpus_size = self.corpus_size
|
|
358
|
+
return arr
|
|
359
|
+
|
|
360
|
+
def __setitem__(self, key, value):
|
|
361
|
+
"""Set an item in the array."""
|
|
362
|
+
key = pd.api.indexers.check_array_indexer(self, key)
|
|
363
|
+
if isinstance(value, pd.Series):
|
|
364
|
+
value = value.values
|
|
365
|
+
if isinstance(value, pd.DataFrame):
|
|
366
|
+
value = value.values.flatten()
|
|
367
|
+
if isinstance(value, SearchArray):
|
|
368
|
+
value = value.to_numpy()
|
|
369
|
+
if isinstance(value, list):
|
|
370
|
+
value = np.asarray(value, dtype=object)
|
|
371
|
+
|
|
372
|
+
if not isinstance(value, np.ndarray) and not self.dtype.valid_value(value):
|
|
373
|
+
raise ValueError(f"Cannot set non-object array to SearchArray -- you passed type:{type(value)} -- {value}")
|
|
374
|
+
|
|
375
|
+
# Cant set a single value to an array
|
|
376
|
+
if isinstance(key, numbers.Integral) and isinstance(value, np.ndarray):
|
|
377
|
+
raise ValueError("Cannot set a single value to an array")
|
|
378
|
+
|
|
379
|
+
try:
|
|
380
|
+
is_encoded = False
|
|
381
|
+
posns = None
|
|
382
|
+
term_mat = np.asarray([])
|
|
383
|
+
doc_lens = np.asarray([], dtype=np.float32)
|
|
384
|
+
if isinstance(value, float):
|
|
385
|
+
term_mat = np.asarray([value])
|
|
386
|
+
doc_lens = np.asarray([0])
|
|
387
|
+
elif isinstance(value, Terms):
|
|
388
|
+
term_mat = np.asarray([value.tf_to_dense(self.term_dict)])
|
|
389
|
+
doc_lens = np.asarray([value.doc_len], dtype=np.float32)
|
|
390
|
+
is_encoded = value.encoded
|
|
391
|
+
posns = [value.raw_positions(self.term_dict)]
|
|
392
|
+
elif isinstance(value, np.ndarray):
|
|
393
|
+
term_mat = np.asarray([x.tf_to_dense(self.term_dict) for x in value])
|
|
394
|
+
doc_lens = np.asarray([np.float32(x.doc_len) for x in value], dtype=np.float32)
|
|
395
|
+
is_encoded = value[0].encoded if len(value) > 0 else False
|
|
396
|
+
posns = [x.raw_positions(self.term_dict) for x in value]
|
|
397
|
+
np.nan_to_num(term_mat, copy=False, nan=0)
|
|
398
|
+
self.term_mat[key] = term_mat
|
|
399
|
+
self.doc_lens[key] = doc_lens
|
|
400
|
+
|
|
401
|
+
if posns is not None:
|
|
402
|
+
self.posns.insert(key, posns, is_encoded)
|
|
403
|
+
|
|
404
|
+
# Assume we have a positions for each term, doc pair. We can just update it.
|
|
405
|
+
# Otherwise we would have added new terms
|
|
406
|
+
except TermMissingError:
|
|
407
|
+
self._add_new_terms(key, value)
|
|
408
|
+
|
|
409
|
+
def _add_new_terms(self, key, value):
|
|
410
|
+
msg = """Adding new terms! This might not be good if you tokenized this new text
|
|
411
|
+
with a different tokenizer.
|
|
412
|
+
|
|
413
|
+
Also. This is slow."""
|
|
414
|
+
warnings.warn(msg)
|
|
415
|
+
|
|
416
|
+
scan_value = value
|
|
417
|
+
if isinstance(value, Terms):
|
|
418
|
+
scan_value = np.asarray([value])
|
|
419
|
+
for row in scan_value:
|
|
420
|
+
for term in row.terms():
|
|
421
|
+
self.term_dict.add_term(term[0])
|
|
422
|
+
|
|
423
|
+
self.term_mat.resize((self.term_mat.shape[0], len(self.term_dict)))
|
|
424
|
+
# Ensure posns_lookup has at least max self.posns
|
|
425
|
+
self[key] = value
|
|
426
|
+
|
|
427
|
+
def value_counts(
|
|
428
|
+
self,
|
|
429
|
+
dropna: bool = True,
|
|
430
|
+
):
|
|
431
|
+
if dropna:
|
|
432
|
+
counts = Counter(self[:])
|
|
433
|
+
counts.pop(Terms({}), None)
|
|
434
|
+
else:
|
|
435
|
+
counts = Counter(self[:])
|
|
436
|
+
return pd.Series(counts)
|
|
437
|
+
|
|
438
|
+
def __len__(self):
|
|
439
|
+
len_rval = len(self.term_mat.rows)
|
|
440
|
+
return len_rval
|
|
441
|
+
|
|
442
|
+
def __ne__(self, other):
|
|
443
|
+
if isinstance(other, pd.DataFrame) or isinstance(other, pd.Series) or isinstance(other, pd.Index):
|
|
444
|
+
return NotImplemented
|
|
445
|
+
|
|
446
|
+
return ~(self == other)
|
|
447
|
+
|
|
448
|
+
def __eq__(self, other):
|
|
449
|
+
"""Return a boolean numpy array indicating elementwise equality."""
|
|
450
|
+
# When other is a dataframe or series, not implemented
|
|
451
|
+
if isinstance(other, pd.DataFrame) or isinstance(other, pd.Series) or isinstance(other, pd.Index):
|
|
452
|
+
return NotImplemented
|
|
453
|
+
|
|
454
|
+
# When other is an ExtensionArray
|
|
455
|
+
if isinstance(other, SearchArray):
|
|
456
|
+
if len(self) != len(other):
|
|
457
|
+
return False
|
|
458
|
+
elif len(other) == 0:
|
|
459
|
+
return np.array([], dtype=bool)
|
|
460
|
+
else:
|
|
461
|
+
# Compatible term dicts, and same term freqs
|
|
462
|
+
# (not looking at positions, maybe we should?)
|
|
463
|
+
if self.term_dict.compatible(other.term_dict):
|
|
464
|
+
return (self.term_mat == other.term_mat) & (self.doc_lens == other.doc_lens)
|
|
465
|
+
else:
|
|
466
|
+
return np.zeros(len(self), dtype=bool)
|
|
467
|
+
# return np.array(self[:]) == np.array(other[:])
|
|
468
|
+
|
|
469
|
+
# When other is a scalar value
|
|
470
|
+
elif isinstance(other, Terms):
|
|
471
|
+
other = SearchArray([other], tokenizer=self.tokenizer)
|
|
472
|
+
warnings.warn("Comparing a scalar value to a SearchArray. This is slow.")
|
|
473
|
+
return np.array(self[:]) == np.array(other[:])
|
|
474
|
+
|
|
475
|
+
# When other is a sequence but not an ExtensionArray
|
|
476
|
+
# its an array of dicts
|
|
477
|
+
elif is_list_like(other):
|
|
478
|
+
if len(self) != len(other):
|
|
479
|
+
return False
|
|
480
|
+
elif len(other) == 0:
|
|
481
|
+
return np.array([], dtype=bool)
|
|
482
|
+
# We actually don't know how it was tokenized
|
|
483
|
+
other = SearchArray(other, tokenizer=self.tokenizer)
|
|
484
|
+
return np.array(self[:]) == np.array(other[:])
|
|
485
|
+
|
|
486
|
+
# Return False where 'other' is neither the same length nor a scalar
|
|
487
|
+
else:
|
|
488
|
+
return np.full(len(self), False)
|
|
489
|
+
|
|
490
|
+
def isna(self):
|
|
491
|
+
# Every row with all 0s
|
|
492
|
+
empties = self.doc_lens == 0
|
|
493
|
+
return empties
|
|
494
|
+
|
|
495
|
+
def unique(self):
|
|
496
|
+
"""This is a hack for colab which wants to visit every element of the array."""
|
|
497
|
+
logger.warning("Unique called on SearchArray. This is not supported.")
|
|
498
|
+
return self[:]
|
|
499
|
+
|
|
500
|
+
def __iter__(self):
|
|
501
|
+
if len(self) > 10000:
|
|
502
|
+
warning_text = """Iterating over SearchArray is very slow and not reccomended.
|
|
503
|
+
|
|
504
|
+
If you're looping a dataframe, slice out the non-SearchArray columns first."""
|
|
505
|
+
logger.warning(warning_text)
|
|
506
|
+
warnings.warn(warning_text)
|
|
507
|
+
return super().__iter__()
|
|
508
|
+
|
|
509
|
+
def take(self, indices, allow_fill=False, fill_value=None):
|
|
510
|
+
# Want to take rows of term freqs
|
|
511
|
+
row_indices = np.arange(len(self.term_mat.rows))
|
|
512
|
+
# Take within the row indices themselves
|
|
513
|
+
result_indices = take(row_indices, indices, allow_fill=allow_fill, fill_value=-1)
|
|
514
|
+
|
|
515
|
+
if allow_fill and -1 in result_indices:
|
|
516
|
+
if fill_value is None or pd.isna(fill_value):
|
|
517
|
+
fill_value = Terms({}, encoded=True)
|
|
518
|
+
|
|
519
|
+
to_fill_mask = result_indices == -1
|
|
520
|
+
# This is slow as it rebuilds all the term dictionaries
|
|
521
|
+
# on the subsequent assignment lines
|
|
522
|
+
# However, this case tends to be the exception for
|
|
523
|
+
# most dataframe operations
|
|
524
|
+
taken = SearchArray([fill_value] * len(result_indices))
|
|
525
|
+
taken[~to_fill_mask] = self[result_indices[~to_fill_mask]].copy()
|
|
526
|
+
|
|
527
|
+
return taken
|
|
528
|
+
else:
|
|
529
|
+
taken = self[result_indices].copy()
|
|
530
|
+
return taken
|
|
531
|
+
|
|
532
|
+
def copy(self):
|
|
533
|
+
postings_arr = SearchArray([], tokenizer=self.tokenizer)
|
|
534
|
+
postings_arr.doc_lens = self.doc_lens.copy()
|
|
535
|
+
postings_arr.term_mat = self.term_mat.copy()
|
|
536
|
+
postings_arr.posns = self.posns
|
|
537
|
+
postings_arr.term_dict = self.term_dict
|
|
538
|
+
postings_arr.avg_doc_length = self.avg_doc_length
|
|
539
|
+
postings_arr.corpus_size = self.corpus_size
|
|
540
|
+
|
|
541
|
+
if not self.avoid_copies:
|
|
542
|
+
postings_arr.posns = self.posns.copy()
|
|
543
|
+
postings_arr.term_dict = self.term_dict.copy()
|
|
544
|
+
return postings_arr
|
|
545
|
+
|
|
546
|
+
@classmethod
|
|
547
|
+
def _concat_same_type(cls, to_concat):
|
|
548
|
+
concatenated_data = np.concatenate([ea[:] for ea in to_concat])
|
|
549
|
+
return SearchArray(concatenated_data, tokenizer=to_concat[0].tokenizer)
|
|
550
|
+
|
|
551
|
+
@classmethod
|
|
552
|
+
def _from_factorized(cls, values, original):
|
|
553
|
+
return cls(values)
|
|
554
|
+
|
|
555
|
+
def _values_for_factorize(self):
|
|
556
|
+
"""Return an array and missing value suitable for factorization (ie grouping)."""
|
|
557
|
+
arr = np.asarray(self[:], dtype=object)
|
|
558
|
+
return arr, Terms({})
|
|
559
|
+
|
|
560
|
+
def _check_token_arg(self, token):
|
|
561
|
+
if isinstance(token, str):
|
|
562
|
+
return token
|
|
563
|
+
elif isinstance(token, list) and len(token) == 1:
|
|
564
|
+
return token[0]
|
|
565
|
+
elif isinstance(token, list):
|
|
566
|
+
return token
|
|
567
|
+
else:
|
|
568
|
+
raise TypeError("Expected a string or list of strings for phrases")
|
|
569
|
+
|
|
570
|
+
def memory_report(self, N=1000):
|
|
571
|
+
"""Return a string with memory usage information."""
|
|
572
|
+
term_mat_bytes = self.term_mat.nbytes
|
|
573
|
+
posns_bytes = self.posns.nbytes
|
|
574
|
+
term_dict_bytes = self.term_dict.nbytes
|
|
575
|
+
num_terms = len(self.term_dict)
|
|
576
|
+
top_10_terms_size = []
|
|
577
|
+
|
|
578
|
+
for i in range(N):
|
|
579
|
+
term_len = len(self.posns.encoded_term_posns[i])
|
|
580
|
+
term = self.term_dict.get_term(i)
|
|
581
|
+
term_bytes = term_len * 8
|
|
582
|
+
top_10_terms_size.append((term, term_bytes))
|
|
583
|
+
|
|
584
|
+
top_10_terms_size = sorted(top_10_terms_size, key=lambda x: x[1], reverse=True)
|
|
585
|
+
|
|
586
|
+
report = f"""
|
|
587
|
+
SearchArray Memory Report
|
|
588
|
+
-------------------------
|
|
589
|
+
Number of Terms: {num_terms}
|
|
590
|
+
-------------------------
|
|
591
|
+
Term Matrix: {bytes_to_human_readable(term_mat_bytes)}
|
|
592
|
+
Positions: {bytes_to_human_readable(posns_bytes)}
|
|
593
|
+
Term Dictionary: {bytes_to_human_readable(term_dict_bytes)}
|
|
594
|
+
|
|
595
|
+
"""
|
|
596
|
+
|
|
597
|
+
cum_sum = 0
|
|
598
|
+
for i in range(N):
|
|
599
|
+
term, term_bytes = top_10_terms_size[i]
|
|
600
|
+
cum_sum += term_bytes
|
|
601
|
+
report += f" Term {i}: {term} - {bytes_to_human_readable(term_bytes)} - Cumulative: {bytes_to_human_readable(cum_sum)}\n"
|
|
602
|
+
return report
|
|
603
|
+
|
|
604
|
+
# ***********************************************************
|
|
605
|
+
# Search API
|
|
606
|
+
# ***********************************************************
|
|
607
|
+
def termfreqs(self, token: Union[List[str], str],
|
|
608
|
+
slop: int = 0,
|
|
609
|
+
min_posn: Optional[int] = None,
|
|
610
|
+
max_posn: Optional[int] = None) -> np.ndarray:
|
|
611
|
+
token = self._check_token_arg(token)
|
|
612
|
+
if isinstance(token, list):
|
|
613
|
+
tfs = self._phrase_freq(token, slop=slop, min_posn=min_posn, max_posn=max_posn)
|
|
614
|
+
return tfs
|
|
615
|
+
|
|
616
|
+
try:
|
|
617
|
+
term_id = self.term_dict.get_term_id(token)
|
|
618
|
+
slice_of_rows = self.term_mat.rows
|
|
619
|
+
if self.term_mat.subset:
|
|
620
|
+
matches = np.zeros(len(self), dtype=np.float32)
|
|
621
|
+
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
622
|
+
doc_ids=slice_of_rows,
|
|
623
|
+
min_posn=min_posn,
|
|
624
|
+
max_posn=max_posn)
|
|
625
|
+
mask = np.isin(self.term_mat.rows, doc_ids)
|
|
626
|
+
matches[mask] = termfreqs
|
|
627
|
+
return matches
|
|
628
|
+
else:
|
|
629
|
+
doc_ids, termfreqs = self.posns.termfreqs(term_id,
|
|
630
|
+
doc_ids=None,
|
|
631
|
+
min_posn=min_posn,
|
|
632
|
+
max_posn=max_posn)
|
|
633
|
+
|
|
634
|
+
# This copy to a dense numpy array is the
|
|
635
|
+
# bottleneck for termfreqs
|
|
636
|
+
return as_dense(doc_ids, termfreqs, len(self))
|
|
637
|
+
except TermMissingError:
|
|
638
|
+
return np.zeros(len(self), dtype=np.float32)
|
|
639
|
+
|
|
640
|
+
def docfreq(self, token: str) -> int:
|
|
641
|
+
if not isinstance(token, str):
|
|
642
|
+
raise TypeError("Expected a string")
|
|
643
|
+
# Count number of rows where the term appears
|
|
644
|
+
try:
|
|
645
|
+
return self.posns.docfreq(self.term_dict.get_term_id(token))
|
|
646
|
+
except TermMissingError:
|
|
647
|
+
return 0
|
|
648
|
+
|
|
649
|
+
def doclengths(self) -> np.ndarray:
|
|
650
|
+
return self.doc_lens
|
|
651
|
+
|
|
652
|
+
def score(self, token: Union[str, List[str]],
|
|
653
|
+
similarity: Similarity = default_bm25,
|
|
654
|
+
slop: int = 0,
|
|
655
|
+
min_posn: Optional[int] = None,
|
|
656
|
+
max_posn: Optional[int] = None) -> np.ndarray:
|
|
657
|
+
"""Score each doc using a similarity function.
|
|
658
|
+
|
|
659
|
+
Parameters
|
|
660
|
+
----------
|
|
661
|
+
token : str or list of str of what to search (already tokenized)
|
|
662
|
+
similarity : How to score the documents. Default is BM25.
|
|
663
|
+
min_posn : int - minimum position of the term in the document, in multiples of 18
|
|
664
|
+
max_posn : int - maximum position of the term in the document, in multiples of 18
|
|
665
|
+
"""
|
|
666
|
+
# Get term freqs per token
|
|
667
|
+
token = self._check_token_arg(token)
|
|
668
|
+
|
|
669
|
+
# For expensive toknes, we compute doc freq first, so we
|
|
670
|
+
# cache them in the DF cache, to let TF cache know it should be cached
|
|
671
|
+
tokens_l = [token] if isinstance(token, str) else token
|
|
672
|
+
all_dfs = np.asarray([self.docfreq(token) for token in tokens_l])
|
|
673
|
+
|
|
674
|
+
tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn,
|
|
675
|
+
slop=slop)
|
|
676
|
+
token = self._check_token_arg(token)
|
|
677
|
+
doc_lens = self.doclengths()
|
|
678
|
+
|
|
679
|
+
scores = similarity(tfs, all_dfs, doc_lens, self.avg_doc_length, self.corpus_size)
|
|
680
|
+
return scores
|
|
681
|
+
|
|
682
|
+
def positions(self, token: str, key=None) -> List[np.ndarray]:
|
|
683
|
+
"""Return a list of lists of positions of the given term."""
|
|
684
|
+
term_id = self.term_dict.get_term_id(token)
|
|
685
|
+
key = self.term_mat.rows[key] if key is not None else self.term_mat.rows
|
|
686
|
+
posns = self.posns.positions(term_id, doc_ids=key)
|
|
687
|
+
return posns
|
|
688
|
+
|
|
689
|
+
def _phrase_freq(self, tokens: List[str],
|
|
690
|
+
slop=0,
|
|
691
|
+
min_posn: Optional[int] = None,
|
|
692
|
+
max_posn: Optional[int] = None) -> np.ndarray:
|
|
693
|
+
if slop > 0:
|
|
694
|
+
logger.warning("!! Slop is experimental and may be slow, crash, or inaccurate etc")
|
|
695
|
+
try:
|
|
696
|
+
# Decide how/if we need to filter doc ids
|
|
697
|
+
term_ids = [self.term_dict.get_term_id(token) for token in tokens]
|
|
698
|
+
phrase_freqs = self.posns.phrase_freqs(term_ids,
|
|
699
|
+
slop=slop,
|
|
700
|
+
min_posn=min_posn,
|
|
701
|
+
max_posn=max_posn)
|
|
702
|
+
if self.term_mat.subset:
|
|
703
|
+
return phrase_freqs[self.term_mat.rows]
|
|
704
|
+
return phrase_freqs
|
|
705
|
+
except TermMissingError:
|
|
706
|
+
if self.term_mat.subset:
|
|
707
|
+
return np.zeros(len(self), dtype=np.float32)
|
|
708
|
+
return self.posns.empty_buffer()
|