searcharray 0.0.73__cp311-cp311-win_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. searcharray/__init__.py +2 -0
  2. searcharray/bm25/__init__.py +1 -0
  3. searcharray/bm25/bm25.cp311-win_arm64.pyd +0 -0
  4. searcharray/bm25/bm25.pyi +11 -0
  5. searcharray/bm25/bm25.pyx +41 -0
  6. searcharray/indexing.py +342 -0
  7. searcharray/phrase/__init__.py +0 -0
  8. searcharray/phrase/bigram_freqs.py +307 -0
  9. searcharray/phrase/memmap_arrays.py +208 -0
  10. searcharray/phrase/middle_out.py +553 -0
  11. searcharray/phrase/spans.py +187 -0
  12. searcharray/postings.py +708 -0
  13. searcharray/roaringish/__init__.py +7 -0
  14. searcharray/roaringish/intersect.cp311-win_arm64.pyd +0 -0
  15. searcharray/roaringish/intersect.pyi +25 -0
  16. searcharray/roaringish/intersect.pyx +390 -0
  17. searcharray/roaringish/merge.cp311-win_arm64.pyd +0 -0
  18. searcharray/roaringish/merge.pyi +16 -0
  19. searcharray/roaringish/merge.pyx +232 -0
  20. searcharray/roaringish/popcount.cp311-win_arm64.pyd +0 -0
  21. searcharray/roaringish/popcount.pyi +36 -0
  22. searcharray/roaringish/popcount.pyx +278 -0
  23. searcharray/roaringish/roaringish.py +298 -0
  24. searcharray/roaringish/roaringish_ops.cp311-win_arm64.pyd +0 -0
  25. searcharray/roaringish/roaringish_ops.pyx +98 -0
  26. searcharray/roaringish/scatter_assign.h +31 -0
  27. searcharray/roaringish/search.cp311-win_arm64.pyd +0 -0
  28. searcharray/roaringish/search.pyi +25 -0
  29. searcharray/roaringish/search.pyx +140 -0
  30. searcharray/roaringish/snp_ops.pxd +13 -0
  31. searcharray/roaringish/spans.cp311-win_arm64.pyd +0 -0
  32. searcharray/roaringish/spans.pyx +330 -0
  33. searcharray/roaringish/unique.cp311-win_arm64.pyd +0 -0
  34. searcharray/roaringish/unique.pyi +6 -0
  35. searcharray/roaringish/unique.pyx +145 -0
  36. searcharray/similarity.py +92 -0
  37. searcharray/solr.py +355 -0
  38. searcharray/term_dict.py +59 -0
  39. searcharray/utils/__init__.py +0 -0
  40. searcharray/utils/bitcount.py +34 -0
  41. searcharray/utils/mat_set.py +170 -0
  42. searcharray/utils/row_viewable_matrix.py +91 -0
  43. searcharray/utils/sort.py +45 -0
  44. searcharray-0.0.73.dist-info/METADATA +245 -0
  45. searcharray-0.0.73.dist-info/RECORD +48 -0
  46. searcharray-0.0.73.dist-info/WHEEL +5 -0
  47. searcharray-0.0.73.dist-info/entry_points.txt +2 -0
  48. searcharray-0.0.73.dist-info/top_level.txt +1 -0
@@ -0,0 +1,2 @@
1
+ from searcharray.postings import SearchArray, Terms, TermsDtype # noqa: E401, F401
2
+ from searcharray.utils.sort import SetOfResults # noqa: E401, F401
@@ -0,0 +1 @@
1
+ from searcharray.bm25.bm25 import bm25_score # noqa
@@ -0,0 +1,11 @@
1
+ import numpy as np
2
+ from numpy.typing import NDArray
3
+
4
+
5
+ def bm25_score(term_freqs: NDArray[np.float32],
6
+ doc_lens: NDArray[np.float32],
7
+ avg_doc_lens: float,
8
+ idf: float,
9
+ b: float,
10
+ k1: float):
11
+ ...
@@ -0,0 +1,41 @@
1
+ # cython: boundscheck=False
2
+ # cython: wraparound=False
3
+ # cython: initializedcheck=False
4
+ # cython: cdivision=True
5
+ # cython: nonecheck=False
6
+ # cython: language_level=3
7
+ cimport numpy as np
8
+ import numpy as np
9
+
10
+
11
+ cdef void _bm25_score(float* term_freqs,
12
+ float* doc_lens,
13
+ float idf,
14
+ float avg_doc_lens,
15
+ float k1,
16
+ float b,
17
+ long length) nogil:
18
+ """Modify termfreqs in place changing to BM25 score."""
19
+ cdef float one_minus_b = 1 - b
20
+ for _ in range(length):
21
+ term_freqs[0] = (
22
+ term_freqs[0] / (term_freqs[0] + (k1 * (one_minus_b + (b * (doc_lens[0] / avg_doc_lens)))))
23
+ ) * idf
24
+ term_freqs += 1
25
+ doc_lens += 1
26
+
27
+
28
+ def bm25_score(np.ndarray[np.float32_t, ndim=1] term_freqs,
29
+ np.ndarray[np.float32_t, ndim=1] doc_lens,
30
+ float avg_doc_lens,
31
+ float idf,
32
+ float k1,
33
+ float b):
34
+ cdef long length = term_freqs.shape[0]
35
+ _bm25_score(&term_freqs[0],
36
+ &doc_lens[0],
37
+ idf,
38
+ avg_doc_lens,
39
+ k1,
40
+ b,
41
+ length)
@@ -0,0 +1,342 @@
1
+ import numpy as np
2
+ from numpy.typing import NDArray
3
+ import math
4
+ import os
5
+ import sys
6
+ from typing import Iterable, List, Optional
7
+ from concurrent.futures import ThreadPoolExecutor, as_completed
8
+ from itertools import islice
9
+ from searcharray.phrase.middle_out import MAX_POSN, PosnBitArrayFromFlatBuilder, PosnBitArrayBuilder, PosnBitArrayAlreadyEncBuilder
10
+ from searcharray.term_dict import TermDict
11
+ from searcharray.utils.mat_set import SparseMatSetBuilder
12
+ from searcharray.utils.row_viewable_matrix import RowViewableMatrix
13
+
14
+ import logging
15
+ logger = logging.getLogger(__name__)
16
+ handler = logging.StreamHandler(sys.stdout)
17
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
18
+ logger.setLevel(logging.INFO)
19
+ handler.setFormatter(formatter)
20
+ logger.addHandler(handler)
21
+
22
+
23
+ # Set to stdout for debugging
24
+ # logging.basicConfig(level=logging.DEBUG)
25
+
26
+
27
+ def home_directory():
28
+ return os.path.expanduser("~")
29
+
30
+
31
+ def searcharray_home():
32
+ searcharray_dir = os.path.join(home_directory(), ".searcharray")
33
+ # Ensure exists
34
+ os.makedirs(searcharray_dir, exist_ok=True)
35
+ return searcharray_dir
36
+
37
+
38
+ def _compute_doc_lens(posns: np.ndarray, doc_ids: np.ndarray, num_docs: int) -> NDArray[np.float32]:
39
+ """Given an array of positions, compute the length of each document."""
40
+ doc_lens = np.zeros(num_docs, dtype=np.float32)
41
+
42
+ # Find were we ave posns for each doc
43
+ non_empty_doc_lens = -np.diff(posns).astype(np.float32) + 1
44
+
45
+ non_empty_idxs = np.argwhere(non_empty_doc_lens > 0).flatten()
46
+ non_empty_doc_ids = doc_ids[non_empty_idxs]
47
+ non_empty_doc_lens = non_empty_doc_lens[non_empty_idxs]
48
+ doc_lens[non_empty_doc_ids] = non_empty_doc_lens
49
+ if len(doc_ids) > 0 and doc_ids[-1] not in non_empty_doc_ids:
50
+ doc_lens[doc_ids[-1]] = np.float32(posns[-1] + 1)
51
+ return doc_lens
52
+
53
+
54
+ def convert_size(size_bytes):
55
+ if size_bytes == 0:
56
+ return "0B"
57
+ size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
58
+ i = int(math.floor(math.log(size_bytes, 1024)))
59
+ p = math.pow(1024, i)
60
+ s = round(size_bytes / p, 2)
61
+ return "%s %s" % (s, size_name[i])
62
+
63
+
64
+ def _gather_tokens(array, tokenizer,
65
+ term_dict, term_doc, start_doc_id=0, trunc_posn=None):
66
+ all_terms = []
67
+ all_docs = []
68
+ all_posns = []
69
+
70
+ logger.info(f"Tokenizing {len(array)} documents")
71
+
72
+ for idx, doc in enumerate(array):
73
+ doc_id = start_doc_id + idx
74
+
75
+ # Speedup - use predefined vocabulary
76
+ terms = np.asarray([term_dict.add_term(token)
77
+ for token in tokenizer(doc)], dtype=np.uint32)[:trunc_posn]
78
+ doc_ids = np.full(len(terms), doc_id, dtype=np.uint32)[:trunc_posn]
79
+ all_terms.append(terms)
80
+ all_docs.append(doc_ids)
81
+
82
+ all_posns.append(np.arange(len(terms)))
83
+
84
+ term_doc.append(np.unique(terms))
85
+
86
+ if idx % 10000 == 0 and idx > 0:
87
+ logger.info(f"Tokenized {doc_id} ({100.0 * (idx / len(array))}%)")
88
+
89
+ # Flatten each
90
+ all_terms = np.concatenate(all_terms)
91
+ all_docs = np.concatenate(all_docs)
92
+ all_posns = np.concatenate(all_posns)
93
+
94
+ logger.info("Tokenization -- vstacking")
95
+ terms_w_posns = np.vstack([all_terms, all_docs, all_posns])
96
+ # del all_terms, all_docs, all_posns
97
+ # gc.collect()
98
+ logger.info("Tokenization -- DONE")
99
+ return terms_w_posns, term_doc
100
+
101
+
102
+ def _lex_sort(terms_w_posns):
103
+ """Sort terms, then doc_id, then posn."""
104
+ # Because docs / posns already sorted, we can just sort on terms
105
+ # Equivelant to np.lexsort(terms_w_posns[[::-1], :])
106
+ return np.argsort(terms_w_posns[0, :], kind='stable')
107
+
108
+
109
+ def _invert_docs_terms(terms_w_posns):
110
+ """Sort terms, then doc_id, then posn."""
111
+ # An in place sort could be faster if we could figure this out,
112
+ # an np.sort(arr) is about 1/3 the time of np.argsort(arr)
113
+ # possibly more with just arr.sort()
114
+ lexsort = _lex_sort(terms_w_posns)
115
+ return terms_w_posns[:, lexsort]
116
+
117
+
118
+ def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate=False):
119
+ logger.info(f"{batch_beg} Batch Start tokenization")
120
+ trunc_posn = None
121
+ if truncate:
122
+ trunc_posn = MAX_POSN
123
+
124
+ term_doc = SparseMatSetBuilder()
125
+ terms_w_posns, term_doc = _gather_tokens(array, tokenizer,
126
+ term_dict, term_doc, start_doc_id=batch_beg,
127
+ trunc_posn=trunc_posn)
128
+
129
+ # Use posns to compute doc lens
130
+ doc_lens = _compute_doc_lens(posns=terms_w_posns[2, :],
131
+ doc_ids=(terms_w_posns[1, :] - batch_beg),
132
+ num_docs=len(array))
133
+ logger.info("Inverting docs->terms")
134
+ terms_w_posns = _invert_docs_terms(terms_w_posns)
135
+
136
+ # Encode posns to bit array
137
+ logger.info("Encoding positions to bit array")
138
+ posns = PosnBitArrayFromFlatBuilder(terms_w_posns, max_doc_id=(batch_beg + len(array) - 1))
139
+ bit_posns = posns.build()
140
+
141
+ if np.any(doc_lens > MAX_POSN):
142
+ raise ValueError(f"Document length exceeds maximum of {MAX_POSN}")
143
+
144
+ logger.info("Batch tokenization complete")
145
+ return batch_beg, term_doc, bit_posns, doc_lens
146
+
147
+
148
+ def batch_iterator(iterator, batch_size):
149
+ """
150
+
151
+ :param iterator: The iterator to slice into batches.
152
+ :param batch_size: The size of each batch.
153
+ """
154
+ batch_beg = 0
155
+ it = iter(iterator)
156
+ while True:
157
+ batch = list(islice(it, batch_size))
158
+ if not batch:
159
+ break
160
+ yield batch_beg, batch
161
+ batch_beg += batch_size
162
+
163
+
164
+ def _process_batches(term_doc, batch_size,
165
+ futures,
166
+ last_batch_beg_processed,
167
+ bit_posns=None,
168
+ doc_lens=None,
169
+ truncate=False):
170
+ batch_results = [None] * len(futures)
171
+ batch_beg = 0
172
+ for future in as_completed(futures):
173
+ try:
174
+ batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = future.result()
175
+ idx = (batch_beg - last_batch_beg_processed) // batch_size
176
+ assert batch_results[idx] is None
177
+ batch_results[idx] \
178
+ = (batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens)
179
+ except ValueError as e:
180
+ logger.error(e)
181
+ logger.error(f"Batch {batch_beg} failed to tokenize")
182
+ raise e
183
+
184
+ logger.info(f"(main thread) Processing {len(batch_results)} batch results")
185
+ for result in batch_results:
186
+ assert result is not None
187
+ batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = result
188
+ term_doc.concat(batch_term_doc)
189
+ if bit_posns is None:
190
+ bit_posns = batch_bit_posns
191
+ else:
192
+ logger.info("(main thread) Concatenating bit positions")
193
+ bit_posns.concat(batch_bit_posns)
194
+ logger.info("(main thread) Concatenated bit positions... Done")
195
+
196
+ doc_lens.append(batch_doc_lens)
197
+ return bit_posns
198
+
199
+
200
+ def build_index_no_workers(array: Iterable, tokenizer, batch_size=10000,
201
+ data_dir: Optional[str] = None,
202
+ cache_gt_than=25,
203
+ truncate=False):
204
+ term_dict = TermDict()
205
+ term_doc = SparseMatSetBuilder()
206
+ doc_lens: List[np.ndarray] = []
207
+ bit_posns = None
208
+
209
+ logger.info("Indexing begins w/ NO workers")
210
+ for batch_beg, batch in batch_iterator(array, batch_size):
211
+ batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = _tokenize_batch(batch, tokenizer, term_dict, batch_size, batch_beg, truncate=truncate)
212
+ term_doc.concat(batch_term_doc)
213
+ if bit_posns is None:
214
+ bit_posns = batch_bit_posns
215
+ else:
216
+ bit_posns.concat(batch_bit_posns)
217
+ doc_lens.append(batch_doc_lens)
218
+
219
+ doc_lens = np.concatenate(doc_lens)
220
+
221
+ avg_doc_length = np.mean(doc_lens)
222
+
223
+ term_doc_built = RowViewableMatrix(term_doc.build())
224
+ logger.info("Indexing from tokenization complete")
225
+ assert bit_posns is not None
226
+ # if data_dir is None:
227
+ # data_dir = searcharray_home()
228
+ if data_dir is not None:
229
+ logger.info(f"Memmapping bit positions to {data_dir}")
230
+ bit_posns.memmap(data_dir)
231
+ bit_posns.cache_gt_than = cache_gt_than
232
+ return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
233
+
234
+
235
+ def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
236
+ data_dir: Optional[str] = None,
237
+ truncate=False, workers=4,
238
+ cache_gt_than=25):
239
+ """Build index directly from tokenizing docs (array of string)."""
240
+ if workers == 1:
241
+ return build_index_no_workers(array, tokenizer, batch_size=batch_size,
242
+ cache_gt_than=cache_gt_than,
243
+ data_dir=data_dir, truncate=truncate)
244
+ term_dict = TermDict()
245
+ term_doc = SparseMatSetBuilder()
246
+ doc_lens: List[np.ndarray] = []
247
+ bit_posns = None
248
+
249
+ logger.info(f"Indexing begins w/ {workers} workers")
250
+ futures = []
251
+ last_batch_beg_processed = 0
252
+
253
+ with ThreadPoolExecutor(max_workers=workers) as executor:
254
+
255
+ for batch_beg, batch in batch_iterator(array, batch_size):
256
+ futures.append(executor.submit(_tokenize_batch,
257
+ batch,
258
+ tokenizer,
259
+ term_dict,
260
+ batch_size,
261
+ batch_beg,
262
+ truncate=truncate))
263
+
264
+ if len(futures) >= workers:
265
+ logger.info(f"Collecting {len(futures)} futures")
266
+ bit_posns = _process_batches(term_doc, batch_size,
267
+ futures,
268
+ last_batch_beg_processed,
269
+ bit_posns, doc_lens, truncate=truncate)
270
+ logger.info(f"{batch_beg} Batch Complete")
271
+ logger.info(f"Roaringish NBytes -- {convert_size(bit_posns.nbytes)}")
272
+ logger.info(f"Term Dict Size -- {len(term_dict)}")
273
+
274
+ last_batch_beg_processed += len(futures) * batch_size
275
+ futures = []
276
+ if len(futures) > 0:
277
+ bit_posns = _process_batches(term_doc, batch_size,
278
+ futures,
279
+ last_batch_beg_processed,
280
+ bit_posns, doc_lens, truncate=truncate)
281
+
282
+ doc_lens = np.concatenate(doc_lens)
283
+
284
+ avg_doc_length = np.mean(doc_lens)
285
+
286
+ term_doc_built = RowViewableMatrix(term_doc.build())
287
+ logger.info("Indexing from tokenization complete")
288
+ assert bit_posns is not None
289
+ # if data_dir is None:
290
+ # data_dir = searcharray_home()
291
+ if data_dir is not None:
292
+ logger.info(f"Memmapping bit positions to {data_dir}")
293
+ bit_posns.memmap(data_dir)
294
+ bit_posns.cache_gt_than = cache_gt_than
295
+ return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens, dtype=np.float32)
296
+
297
+
298
+ def build_index_from_terms_list(postings, Terms):
299
+ """Bulid an index from postings that are already tokenized and point at their term frequencies/posns."""
300
+ term_dict = TermDict()
301
+ term_doc = SparseMatSetBuilder()
302
+ doc_lens = []
303
+ avg_doc_length = 0
304
+ num_postings = 0
305
+ posns = PosnBitArrayBuilder()
306
+ posns_enc = PosnBitArrayAlreadyEncBuilder()
307
+
308
+ # COPY 1
309
+ # Consume generator (tokenized postings) into list
310
+ # its faster this way?
311
+ postings = list(postings)
312
+
313
+ # COPY 2
314
+ for doc_id, tokenized in enumerate(postings):
315
+ if isinstance(tokenized, dict):
316
+ tokenized = Terms(tokenized, doc_len=len(tokenized))
317
+ elif not isinstance(tokenized, Terms):
318
+ raise TypeError("Expected a Terms or a dict")
319
+
320
+ if tokenized.encoded:
321
+ posns = posns_enc
322
+
323
+ doc_lens.append(tokenized.doc_len)
324
+ avg_doc_length += doc_lens[-1]
325
+ terms = []
326
+ for token, term_freq in tokenized.terms():
327
+ term_id = term_dict.add_term(token)
328
+ terms.append(term_id)
329
+ positions = tokenized.positions(token)
330
+ if positions is not None:
331
+ posns.add_posns(doc_id, term_id, positions)
332
+
333
+ term_doc.append(terms)
334
+
335
+ posns.ensure_capacity(doc_id)
336
+ num_postings += 1
337
+
338
+ if num_postings > 0:
339
+ avg_doc_length /= num_postings
340
+
341
+ bit_posns = posns.build()
342
+ return RowViewableMatrix(term_doc.build()), bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
File without changes