searcharray 0.0.73__cp313-cp313-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. searcharray/__init__.py +2 -0
  2. searcharray/bm25/__init__.py +1 -0
  3. searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
  4. searcharray/bm25/bm25.pyi +11 -0
  5. searcharray/bm25/bm25.pyx +41 -0
  6. searcharray/indexing.py +342 -0
  7. searcharray/phrase/__init__.py +0 -0
  8. searcharray/phrase/bigram_freqs.py +307 -0
  9. searcharray/phrase/memmap_arrays.py +208 -0
  10. searcharray/phrase/middle_out.py +553 -0
  11. searcharray/phrase/spans.py +187 -0
  12. searcharray/postings.py +708 -0
  13. searcharray/roaringish/__init__.py +7 -0
  14. searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
  15. searcharray/roaringish/intersect.pyi +25 -0
  16. searcharray/roaringish/intersect.pyx +390 -0
  17. searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
  18. searcharray/roaringish/merge.pyi +16 -0
  19. searcharray/roaringish/merge.pyx +232 -0
  20. searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
  21. searcharray/roaringish/popcount.pyi +36 -0
  22. searcharray/roaringish/popcount.pyx +278 -0
  23. searcharray/roaringish/roaringish.py +298 -0
  24. searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
  25. searcharray/roaringish/roaringish_ops.pyx +98 -0
  26. searcharray/roaringish/scatter_assign.h +31 -0
  27. searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
  28. searcharray/roaringish/search.pyi +25 -0
  29. searcharray/roaringish/search.pyx +140 -0
  30. searcharray/roaringish/snp_ops.pxd +13 -0
  31. searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
  32. searcharray/roaringish/spans.pyx +330 -0
  33. searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
  34. searcharray/roaringish/unique.pyi +6 -0
  35. searcharray/roaringish/unique.pyx +145 -0
  36. searcharray/similarity.py +92 -0
  37. searcharray/solr.py +355 -0
  38. searcharray/term_dict.py +59 -0
  39. searcharray/utils/__init__.py +0 -0
  40. searcharray/utils/bitcount.py +34 -0
  41. searcharray/utils/mat_set.py +170 -0
  42. searcharray/utils/row_viewable_matrix.py +91 -0
  43. searcharray/utils/sort.py +45 -0
  44. searcharray-0.0.73.dist-info/METADATA +245 -0
  45. searcharray-0.0.73.dist-info/RECORD +48 -0
  46. searcharray-0.0.73.dist-info/WHEEL +5 -0
  47. searcharray-0.0.73.dist-info/entry_points.txt +2 -0
  48. searcharray-0.0.73.dist-info/top_level.txt +1 -0
@@ -0,0 +1,708 @@
1
+ """Tokenized, searchable text as a pandas dtype."""
2
+ import pandas as pd
3
+ import numbers
4
+ from pandas.api.extensions import ExtensionDtype, ExtensionArray, register_extension_dtype
5
+ from pandas.api.types import is_list_like
6
+ from pandas.api.extensions import take
7
+ import json
8
+ from collections import Counter
9
+ import warnings
10
+ import logging
11
+ from typing import List, Union, Optional, Iterable
12
+
13
+
14
+ import numpy as np
15
+ from searcharray.phrase.middle_out import PosnBitArray
16
+ from searcharray.similarity import Similarity, default_bm25
17
+ from searcharray.indexing import build_index_from_tokenizer, build_index_from_terms_list
18
+ from searcharray.term_dict import TermMissingError
19
+ from searcharray.roaringish.roaringish_ops import as_dense
20
+
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+ # When running in pytest
25
+ import sys # noqa
26
+ handler = logging.StreamHandler(sys.stdout)
27
+ handler.setLevel(logging.ERROR)
28
+ formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
29
+ handler.setFormatter(formatter)
30
+ logger.addHandler(handler)
31
+ logger.setLevel(logging.ERROR)
32
+
33
+
34
+ def bytes_to_human_readable(num_bytes):
35
+ """
36
+ Convert a number of bytes into a human-readable format (KB, MB, GB, etc.)
37
+
38
+ :param num_bytes: Number of bytes to be converted
39
+ :type num_bytes: int or float
40
+ :return: Human-readable string
41
+ :rtype: str
42
+ """
43
+ # Define the suffixes for byte conversion
44
+ suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
45
+
46
+ if num_bytes == 0:
47
+ return "0B"
48
+
49
+ i = 0
50
+ while num_bytes >= 1024 and i < len(suffixes) - 1:
51
+ num_bytes /= 1024.
52
+ i += 1
53
+
54
+ return f"{num_bytes:.2f} {suffixes[i]}"
55
+
56
+
57
+ class Terms:
58
+ """An indexed search doc - a single bag of tokenized words and positions."""
59
+
60
+ def __init__(self,
61
+ postings,
62
+ doc_len: int = 0,
63
+ posns: Optional[dict] = None,
64
+ encoded=False):
65
+ self.postings = postings
66
+ self.posns = None
67
+ self.encoded = encoded
68
+ self.doc_len = doc_len
69
+ self.posns = posns
70
+
71
+ def _validate_posns(self):
72
+ # (For testing/assertions) - Confirm every term in positions also in postings
73
+ if self.posns is None:
74
+ return
75
+ for term in self.posns:
76
+ if term not in self.postings:
77
+ raise ValueError(f"Term {term} in positions but not in postings. ")
78
+
79
+ def termfreq(self, token):
80
+ return self.postings[token]
81
+
82
+ def terms(self):
83
+ return self.postings.items()
84
+
85
+ def positions(self, term=None):
86
+ if self.posns is None:
87
+ return {}
88
+ if term is None:
89
+ posns = self.posns.items()
90
+ else:
91
+ posns = self.posns[term]
92
+ return posns
93
+
94
+ def raw_positions(self, term_dict, term=None):
95
+ if self.posns is None:
96
+ return {}
97
+ if term is None:
98
+ posns = [(term_dict.get_term_id(term), posns) for term, posns in self.posns.items()]
99
+ else:
100
+ posns = [(term_dict.get_term_id(term), self.posns[term])]
101
+ return posns
102
+
103
+ def tf_to_dense(self, term_dict):
104
+ """Convert to a dense vector of term frequencies."""
105
+ dense = np.zeros(len(term_dict))
106
+ for term, freq in self.terms():
107
+ dense[term_dict.get_term_id(term)] = freq
108
+ return dense
109
+
110
+ def __len__(self):
111
+ return len(self.postings)
112
+
113
+ def __repr__(self):
114
+ posting_keys = set(self.postings.keys())
115
+ rval = f"Terms({posting_keys})"
116
+ return rval
117
+
118
+ def __str__(self):
119
+ return repr(self)
120
+
121
+ def __eq__(self, other):
122
+ # Flip to the other implementation if we're comparing to a SearchArray
123
+ # to get a boolean array back
124
+ if isinstance(other, SearchArray):
125
+ return other == self
126
+ same_postings = isinstance(other, Terms) and self.postings == other.postings
127
+ if same_postings and self.doc_len == other.doc_len:
128
+ return True
129
+
130
+ def __lt__(self, other):
131
+ # return isinstance(other, Terms) and hash(self) < hash(other)
132
+ keys_both = set(self.postings.keys()).union(set(other.postings.keys()))
133
+ # Sort lexically
134
+ keys_both = sorted(keys_both)
135
+
136
+ # Iterate as if these are two vectors of the same large dimensional vector sparse
137
+ for key in keys_both:
138
+ lhs_val = 0
139
+ rhs_val = 0
140
+ try:
141
+ lhs_val = self.postings[key]
142
+ except KeyError:
143
+ pass
144
+
145
+ try:
146
+ rhs_val = other.postings[key]
147
+ except KeyError:
148
+ pass
149
+
150
+ if lhs_val < rhs_val:
151
+ return True
152
+ elif lhs_val > rhs_val:
153
+ return False
154
+ else:
155
+ continue
156
+ return False
157
+
158
+ def __le__(self, other):
159
+ return self < other or self == other
160
+
161
+ def __gt__(self, other):
162
+ return not (self < other) and self != other
163
+
164
+ def __hash__(self):
165
+ return hash(json.dumps(self.postings, sort_keys=True))
166
+
167
+
168
+ class TermsDtype(ExtensionDtype):
169
+ """Pandas dtype for terms."""
170
+
171
+ name = 'tokenized_text'
172
+ type = Terms
173
+ kind = 'O'
174
+
175
+ @classmethod
176
+ def construct_from_string(cls, string):
177
+ if not isinstance(string, str):
178
+ raise TypeError(
179
+ "'construct_from_string' expects a string, got {}".format(type(string))
180
+ )
181
+ elif string == cls.name:
182
+ return cls()
183
+ else:
184
+ raise TypeError(
185
+ "Cannot construct a '{}' from '{}'".format(cls.__name__, string)
186
+ )
187
+
188
+ @classmethod
189
+ def construct_array_type(cls):
190
+ return SearchArray
191
+
192
+ def __repr__(self):
193
+ return 'TermsDtype()'
194
+
195
+ @property
196
+ def na_value(self):
197
+ return Terms({})
198
+
199
+ def valid_value(self, value):
200
+ return isinstance(value, dict) or pd.isna(value) or isinstance(value, Terms)
201
+
202
+
203
+ register_extension_dtype(TermsDtype)
204
+
205
+
206
+ def ws_tokenizer(string):
207
+ if pd.isna(string):
208
+ return []
209
+ if not isinstance(string, str):
210
+ raise ValueError("Expected a string")
211
+ return string.split()
212
+
213
+
214
+ def _row_to_postings_row(doc_id, row, doc_len, term_dict, posns: PosnBitArray):
215
+ tfs = {}
216
+ labeled_posns = {}
217
+ for term_idx in row.cols:
218
+ term = term_dict.get_term(term_idx)
219
+ tfs[term] = 1
220
+ enc_term_posns = posns.doc_encoded_posns(term_idx, doc_id=doc_id)
221
+ labeled_posns[term] = enc_term_posns
222
+
223
+ result = Terms(tfs, posns=labeled_posns,
224
+ doc_len=doc_len, encoded=True)
225
+ return result
226
+
227
+
228
+ class SearchArray(ExtensionArray):
229
+ """An array of tokenized text (Terms).
230
+
231
+ Use the class method `index` to create a SearchArray from a list of strings and tokenizer
232
+
233
+ """
234
+
235
+ dtype = TermsDtype()
236
+
237
+ def __init__(self, postings, tokenizer=ws_tokenizer, avoid_copies=True):
238
+ # Check dtype, raise TypeError
239
+ if not is_list_like(postings):
240
+ raise TypeError("Expected list-like object, got {}".format(type(postings)))
241
+
242
+ self.avoid_copies = avoid_copies
243
+ self.tokenizer = tokenizer
244
+ self.term_mat, self.posns, \
245
+ self.term_dict, self.avg_doc_length, \
246
+ self.doc_lens = build_index_from_terms_list(postings, Terms)
247
+ self.corpus_size = len(self.doc_lens)
248
+
249
+ @classmethod
250
+ def index(cls, array: Iterable,
251
+ tokenizer=ws_tokenizer,
252
+ truncate=False,
253
+ batch_size=100000,
254
+ avoid_copies=True,
255
+ workers=4,
256
+ cache_gt_than=25,
257
+ data_dir: Optional[str] = None,
258
+ autowarm=True) -> 'SearchArray':
259
+ """Index an array of strings into SearchArray using tokenizer.
260
+
261
+ Parameters
262
+ ----------
263
+ array : Iterable
264
+ A list of strings to index
265
+ tokenizer : Callable
266
+ A function that takes a string and returns an iterable of tokens
267
+ truncate : bool
268
+ If true, truncate the tokenized strings to a maximum length (2^18-1)
269
+ batch_size : int
270
+ For tuning memory usage, the number of strings to encode at once.
271
+ (which will be merged into one index at the end). Higher the more memory
272
+ will be used.
273
+ avoid_copies : bool
274
+ If true, avoid copying the term dictionary and positions during normal
275
+ pandas copy operations
276
+ autowarm : bool
277
+ If true, precompute docfreq / term freqs for most common terms
278
+ after indexing
279
+ """
280
+ if not is_list_like(array):
281
+ raise TypeError("Expected list-like object, got {}".format(type(array)))
282
+
283
+ term_mat, posns, term_dict, avg_doc_length, doc_lens =\
284
+ build_index_from_tokenizer(array, tokenizer, batch_size=batch_size,
285
+ truncate=truncate,
286
+ data_dir=data_dir,
287
+ cache_gt_than=cache_gt_than,
288
+ workers=workers)
289
+
290
+ if autowarm:
291
+ posns.warm()
292
+
293
+ postings = cls([], tokenizer=tokenizer, avoid_copies=avoid_copies)
294
+ postings.term_mat = term_mat
295
+ postings.posns = posns
296
+ postings.term_dict = term_dict
297
+ postings.avg_doc_length = avg_doc_length
298
+ postings.doc_lens = doc_lens
299
+ postings.corpus_size = len(doc_lens)
300
+ return postings
301
+
302
+ def warm(self):
303
+ self.posns.warm()
304
+
305
+ @classmethod
306
+ def _from_sequence(cls, scalars, dtype=None, copy=False):
307
+ """Construct a new SearchArray from a sequence of scalars (PostingRow or convertible into)."""
308
+ if dtype is not None:
309
+ if not isinstance(dtype, TermsDtype):
310
+ return scalars
311
+ if isinstance(scalars, np.ndarray) and scalars.dtype == TermsDtype():
312
+ return cls(scalars)
313
+ # String types
314
+ elif isinstance(scalars, np.ndarray) and scalars.dtype.kind in 'US':
315
+ return cls(scalars)
316
+ # Other objects
317
+ elif isinstance(scalars, np.ndarray) and scalars.dtype != object:
318
+ return scalars
319
+ return cls(scalars)
320
+
321
+ def memory_usage(self, deep=False):
322
+ """Return memory usage of this array in bytes."""
323
+ return self.nbytes
324
+
325
+ @property
326
+ def nbytes(self):
327
+ return self.term_mat.nbytes + self.posns.nbytes + self.doc_lens.nbytes + self.term_dict.nbytes
328
+
329
+ def __getitem__(self, key):
330
+ key = pd.api.indexers.check_array_indexer(self, key)
331
+ # Want to take rows of term freqs
332
+ if isinstance(key, numbers.Integral):
333
+ try:
334
+ rows = self.term_mat[key]
335
+ doc_len = self.doc_lens[key]
336
+ doc_id = key
337
+ if doc_id < 0:
338
+ doc_id += len(self)
339
+ return _row_to_postings_row(doc_id, rows[0], doc_len,
340
+ self.term_dict, self.posns)
341
+ except IndexError:
342
+ raise IndexError("index out of bounds")
343
+ else:
344
+ # Construct a sliced view of this array
345
+ sliced_tfs = self.term_mat.slice(key)
346
+ if not self.avoid_copies:
347
+ sliced_posns = self.posns.slice(sliced_tfs.rows) if not self.avoid_copies else self.posns
348
+ else:
349
+ sliced_posns = self.posns.filter(sliced_tfs.rows)
350
+
351
+ arr = SearchArray([], tokenizer=self.tokenizer)
352
+ arr.term_mat = sliced_tfs
353
+ arr.doc_lens = self.doc_lens[key]
354
+ arr.posns = sliced_posns
355
+ arr.term_dict = self.term_dict
356
+ arr.avg_doc_length = self.avg_doc_length
357
+ arr.corpus_size = self.corpus_size
358
+ return arr
359
+
360
+ def __setitem__(self, key, value):
361
+ """Set an item in the array."""
362
+ key = pd.api.indexers.check_array_indexer(self, key)
363
+ if isinstance(value, pd.Series):
364
+ value = value.values
365
+ if isinstance(value, pd.DataFrame):
366
+ value = value.values.flatten()
367
+ if isinstance(value, SearchArray):
368
+ value = value.to_numpy()
369
+ if isinstance(value, list):
370
+ value = np.asarray(value, dtype=object)
371
+
372
+ if not isinstance(value, np.ndarray) and not self.dtype.valid_value(value):
373
+ raise ValueError(f"Cannot set non-object array to SearchArray -- you passed type:{type(value)} -- {value}")
374
+
375
+ # Cant set a single value to an array
376
+ if isinstance(key, numbers.Integral) and isinstance(value, np.ndarray):
377
+ raise ValueError("Cannot set a single value to an array")
378
+
379
+ try:
380
+ is_encoded = False
381
+ posns = None
382
+ term_mat = np.asarray([])
383
+ doc_lens = np.asarray([], dtype=np.float32)
384
+ if isinstance(value, float):
385
+ term_mat = np.asarray([value])
386
+ doc_lens = np.asarray([0])
387
+ elif isinstance(value, Terms):
388
+ term_mat = np.asarray([value.tf_to_dense(self.term_dict)])
389
+ doc_lens = np.asarray([value.doc_len], dtype=np.float32)
390
+ is_encoded = value.encoded
391
+ posns = [value.raw_positions(self.term_dict)]
392
+ elif isinstance(value, np.ndarray):
393
+ term_mat = np.asarray([x.tf_to_dense(self.term_dict) for x in value])
394
+ doc_lens = np.asarray([np.float32(x.doc_len) for x in value], dtype=np.float32)
395
+ is_encoded = value[0].encoded if len(value) > 0 else False
396
+ posns = [x.raw_positions(self.term_dict) for x in value]
397
+ np.nan_to_num(term_mat, copy=False, nan=0)
398
+ self.term_mat[key] = term_mat
399
+ self.doc_lens[key] = doc_lens
400
+
401
+ if posns is not None:
402
+ self.posns.insert(key, posns, is_encoded)
403
+
404
+ # Assume we have a positions for each term, doc pair. We can just update it.
405
+ # Otherwise we would have added new terms
406
+ except TermMissingError:
407
+ self._add_new_terms(key, value)
408
+
409
+ def _add_new_terms(self, key, value):
410
+ msg = """Adding new terms! This might not be good if you tokenized this new text
411
+ with a different tokenizer.
412
+
413
+ Also. This is slow."""
414
+ warnings.warn(msg)
415
+
416
+ scan_value = value
417
+ if isinstance(value, Terms):
418
+ scan_value = np.asarray([value])
419
+ for row in scan_value:
420
+ for term in row.terms():
421
+ self.term_dict.add_term(term[0])
422
+
423
+ self.term_mat.resize((self.term_mat.shape[0], len(self.term_dict)))
424
+ # Ensure posns_lookup has at least max self.posns
425
+ self[key] = value
426
+
427
+ def value_counts(
428
+ self,
429
+ dropna: bool = True,
430
+ ):
431
+ if dropna:
432
+ counts = Counter(self[:])
433
+ counts.pop(Terms({}), None)
434
+ else:
435
+ counts = Counter(self[:])
436
+ return pd.Series(counts)
437
+
438
+ def __len__(self):
439
+ len_rval = len(self.term_mat.rows)
440
+ return len_rval
441
+
442
+ def __ne__(self, other):
443
+ if isinstance(other, pd.DataFrame) or isinstance(other, pd.Series) or isinstance(other, pd.Index):
444
+ return NotImplemented
445
+
446
+ return ~(self == other)
447
+
448
+ def __eq__(self, other):
449
+ """Return a boolean numpy array indicating elementwise equality."""
450
+ # When other is a dataframe or series, not implemented
451
+ if isinstance(other, pd.DataFrame) or isinstance(other, pd.Series) or isinstance(other, pd.Index):
452
+ return NotImplemented
453
+
454
+ # When other is an ExtensionArray
455
+ if isinstance(other, SearchArray):
456
+ if len(self) != len(other):
457
+ return False
458
+ elif len(other) == 0:
459
+ return np.array([], dtype=bool)
460
+ else:
461
+ # Compatible term dicts, and same term freqs
462
+ # (not looking at positions, maybe we should?)
463
+ if self.term_dict.compatible(other.term_dict):
464
+ return (self.term_mat == other.term_mat) & (self.doc_lens == other.doc_lens)
465
+ else:
466
+ return np.zeros(len(self), dtype=bool)
467
+ # return np.array(self[:]) == np.array(other[:])
468
+
469
+ # When other is a scalar value
470
+ elif isinstance(other, Terms):
471
+ other = SearchArray([other], tokenizer=self.tokenizer)
472
+ warnings.warn("Comparing a scalar value to a SearchArray. This is slow.")
473
+ return np.array(self[:]) == np.array(other[:])
474
+
475
+ # When other is a sequence but not an ExtensionArray
476
+ # its an array of dicts
477
+ elif is_list_like(other):
478
+ if len(self) != len(other):
479
+ return False
480
+ elif len(other) == 0:
481
+ return np.array([], dtype=bool)
482
+ # We actually don't know how it was tokenized
483
+ other = SearchArray(other, tokenizer=self.tokenizer)
484
+ return np.array(self[:]) == np.array(other[:])
485
+
486
+ # Return False where 'other' is neither the same length nor a scalar
487
+ else:
488
+ return np.full(len(self), False)
489
+
490
+ def isna(self):
491
+ # Every row with all 0s
492
+ empties = self.doc_lens == 0
493
+ return empties
494
+
495
+ def unique(self):
496
+ """This is a hack for colab which wants to visit every element of the array."""
497
+ logger.warning("Unique called on SearchArray. This is not supported.")
498
+ return self[:]
499
+
500
+ def __iter__(self):
501
+ if len(self) > 10000:
502
+ warning_text = """Iterating over SearchArray is very slow and not reccomended.
503
+
504
+ If you're looping a dataframe, slice out the non-SearchArray columns first."""
505
+ logger.warning(warning_text)
506
+ warnings.warn(warning_text)
507
+ return super().__iter__()
508
+
509
+ def take(self, indices, allow_fill=False, fill_value=None):
510
+ # Want to take rows of term freqs
511
+ row_indices = np.arange(len(self.term_mat.rows))
512
+ # Take within the row indices themselves
513
+ result_indices = take(row_indices, indices, allow_fill=allow_fill, fill_value=-1)
514
+
515
+ if allow_fill and -1 in result_indices:
516
+ if fill_value is None or pd.isna(fill_value):
517
+ fill_value = Terms({}, encoded=True)
518
+
519
+ to_fill_mask = result_indices == -1
520
+ # This is slow as it rebuilds all the term dictionaries
521
+ # on the subsequent assignment lines
522
+ # However, this case tends to be the exception for
523
+ # most dataframe operations
524
+ taken = SearchArray([fill_value] * len(result_indices))
525
+ taken[~to_fill_mask] = self[result_indices[~to_fill_mask]].copy()
526
+
527
+ return taken
528
+ else:
529
+ taken = self[result_indices].copy()
530
+ return taken
531
+
532
+ def copy(self):
533
+ postings_arr = SearchArray([], tokenizer=self.tokenizer)
534
+ postings_arr.doc_lens = self.doc_lens.copy()
535
+ postings_arr.term_mat = self.term_mat.copy()
536
+ postings_arr.posns = self.posns
537
+ postings_arr.term_dict = self.term_dict
538
+ postings_arr.avg_doc_length = self.avg_doc_length
539
+ postings_arr.corpus_size = self.corpus_size
540
+
541
+ if not self.avoid_copies:
542
+ postings_arr.posns = self.posns.copy()
543
+ postings_arr.term_dict = self.term_dict.copy()
544
+ return postings_arr
545
+
546
+ @classmethod
547
+ def _concat_same_type(cls, to_concat):
548
+ concatenated_data = np.concatenate([ea[:] for ea in to_concat])
549
+ return SearchArray(concatenated_data, tokenizer=to_concat[0].tokenizer)
550
+
551
+ @classmethod
552
+ def _from_factorized(cls, values, original):
553
+ return cls(values)
554
+
555
+ def _values_for_factorize(self):
556
+ """Return an array and missing value suitable for factorization (ie grouping)."""
557
+ arr = np.asarray(self[:], dtype=object)
558
+ return arr, Terms({})
559
+
560
+ def _check_token_arg(self, token):
561
+ if isinstance(token, str):
562
+ return token
563
+ elif isinstance(token, list) and len(token) == 1:
564
+ return token[0]
565
+ elif isinstance(token, list):
566
+ return token
567
+ else:
568
+ raise TypeError("Expected a string or list of strings for phrases")
569
+
570
+ def memory_report(self, N=1000):
571
+ """Return a string with memory usage information."""
572
+ term_mat_bytes = self.term_mat.nbytes
573
+ posns_bytes = self.posns.nbytes
574
+ term_dict_bytes = self.term_dict.nbytes
575
+ num_terms = len(self.term_dict)
576
+ top_10_terms_size = []
577
+
578
+ for i in range(N):
579
+ term_len = len(self.posns.encoded_term_posns[i])
580
+ term = self.term_dict.get_term(i)
581
+ term_bytes = term_len * 8
582
+ top_10_terms_size.append((term, term_bytes))
583
+
584
+ top_10_terms_size = sorted(top_10_terms_size, key=lambda x: x[1], reverse=True)
585
+
586
+ report = f"""
587
+ SearchArray Memory Report
588
+ -------------------------
589
+ Number of Terms: {num_terms}
590
+ -------------------------
591
+ Term Matrix: {bytes_to_human_readable(term_mat_bytes)}
592
+ Positions: {bytes_to_human_readable(posns_bytes)}
593
+ Term Dictionary: {bytes_to_human_readable(term_dict_bytes)}
594
+
595
+ """
596
+
597
+ cum_sum = 0
598
+ for i in range(N):
599
+ term, term_bytes = top_10_terms_size[i]
600
+ cum_sum += term_bytes
601
+ report += f" Term {i}: {term} - {bytes_to_human_readable(term_bytes)} - Cumulative: {bytes_to_human_readable(cum_sum)}\n"
602
+ return report
603
+
604
+ # ***********************************************************
605
+ # Search API
606
+ # ***********************************************************
607
+ def termfreqs(self, token: Union[List[str], str],
608
+ slop: int = 0,
609
+ min_posn: Optional[int] = None,
610
+ max_posn: Optional[int] = None) -> np.ndarray:
611
+ token = self._check_token_arg(token)
612
+ if isinstance(token, list):
613
+ tfs = self._phrase_freq(token, slop=slop, min_posn=min_posn, max_posn=max_posn)
614
+ return tfs
615
+
616
+ try:
617
+ term_id = self.term_dict.get_term_id(token)
618
+ slice_of_rows = self.term_mat.rows
619
+ if self.term_mat.subset:
620
+ matches = np.zeros(len(self), dtype=np.float32)
621
+ doc_ids, termfreqs = self.posns.termfreqs(term_id,
622
+ doc_ids=slice_of_rows,
623
+ min_posn=min_posn,
624
+ max_posn=max_posn)
625
+ mask = np.isin(self.term_mat.rows, doc_ids)
626
+ matches[mask] = termfreqs
627
+ return matches
628
+ else:
629
+ doc_ids, termfreqs = self.posns.termfreqs(term_id,
630
+ doc_ids=None,
631
+ min_posn=min_posn,
632
+ max_posn=max_posn)
633
+
634
+ # This copy to a dense numpy array is the
635
+ # bottleneck for termfreqs
636
+ return as_dense(doc_ids, termfreqs, len(self))
637
+ except TermMissingError:
638
+ return np.zeros(len(self), dtype=np.float32)
639
+
640
+ def docfreq(self, token: str) -> int:
641
+ if not isinstance(token, str):
642
+ raise TypeError("Expected a string")
643
+ # Count number of rows where the term appears
644
+ try:
645
+ return self.posns.docfreq(self.term_dict.get_term_id(token))
646
+ except TermMissingError:
647
+ return 0
648
+
649
+ def doclengths(self) -> np.ndarray:
650
+ return self.doc_lens
651
+
652
+ def score(self, token: Union[str, List[str]],
653
+ similarity: Similarity = default_bm25,
654
+ slop: int = 0,
655
+ min_posn: Optional[int] = None,
656
+ max_posn: Optional[int] = None) -> np.ndarray:
657
+ """Score each doc using a similarity function.
658
+
659
+ Parameters
660
+ ----------
661
+ token : str or list of str of what to search (already tokenized)
662
+ similarity : How to score the documents. Default is BM25.
663
+ min_posn : int - minimum position of the term in the document, in multiples of 18
664
+ max_posn : int - maximum position of the term in the document, in multiples of 18
665
+ """
666
+ # Get term freqs per token
667
+ token = self._check_token_arg(token)
668
+
669
+ # For expensive toknes, we compute doc freq first, so we
670
+ # cache them in the DF cache, to let TF cache know it should be cached
671
+ tokens_l = [token] if isinstance(token, str) else token
672
+ all_dfs = np.asarray([self.docfreq(token) for token in tokens_l])
673
+
674
+ tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn,
675
+ slop=slop)
676
+ token = self._check_token_arg(token)
677
+ doc_lens = self.doclengths()
678
+
679
+ scores = similarity(tfs, all_dfs, doc_lens, self.avg_doc_length, self.corpus_size)
680
+ return scores
681
+
682
+ def positions(self, token: str, key=None) -> List[np.ndarray]:
683
+ """Return a list of lists of positions of the given term."""
684
+ term_id = self.term_dict.get_term_id(token)
685
+ key = self.term_mat.rows[key] if key is not None else self.term_mat.rows
686
+ posns = self.posns.positions(term_id, doc_ids=key)
687
+ return posns
688
+
689
+ def _phrase_freq(self, tokens: List[str],
690
+ slop=0,
691
+ min_posn: Optional[int] = None,
692
+ max_posn: Optional[int] = None) -> np.ndarray:
693
+ if slop > 0:
694
+ logger.warning("!! Slop is experimental and may be slow, crash, or inaccurate etc")
695
+ try:
696
+ # Decide how/if we need to filter doc ids
697
+ term_ids = [self.term_dict.get_term_id(token) for token in tokens]
698
+ phrase_freqs = self.posns.phrase_freqs(term_ids,
699
+ slop=slop,
700
+ min_posn=min_posn,
701
+ max_posn=max_posn)
702
+ if self.term_mat.subset:
703
+ return phrase_freqs[self.term_mat.rows]
704
+ return phrase_freqs
705
+ except TermMissingError:
706
+ if self.term_mat.subset:
707
+ return np.zeros(len(self), dtype=np.float32)
708
+ return self.posns.empty_buffer()