searcharray 0.0.73__cp311-cp311-win_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- searcharray/__init__.py +2 -0
- searcharray/bm25/__init__.py +1 -0
- searcharray/bm25/bm25.cp311-win_arm64.pyd +0 -0
- searcharray/bm25/bm25.pyi +11 -0
- searcharray/bm25/bm25.pyx +41 -0
- searcharray/indexing.py +342 -0
- searcharray/phrase/__init__.py +0 -0
- searcharray/phrase/bigram_freqs.py +307 -0
- searcharray/phrase/memmap_arrays.py +208 -0
- searcharray/phrase/middle_out.py +553 -0
- searcharray/phrase/spans.py +187 -0
- searcharray/postings.py +708 -0
- searcharray/roaringish/__init__.py +7 -0
- searcharray/roaringish/intersect.cp311-win_arm64.pyd +0 -0
- searcharray/roaringish/intersect.pyi +25 -0
- searcharray/roaringish/intersect.pyx +390 -0
- searcharray/roaringish/merge.cp311-win_arm64.pyd +0 -0
- searcharray/roaringish/merge.pyi +16 -0
- searcharray/roaringish/merge.pyx +232 -0
- searcharray/roaringish/popcount.cp311-win_arm64.pyd +0 -0
- searcharray/roaringish/popcount.pyi +36 -0
- searcharray/roaringish/popcount.pyx +278 -0
- searcharray/roaringish/roaringish.py +298 -0
- searcharray/roaringish/roaringish_ops.cp311-win_arm64.pyd +0 -0
- searcharray/roaringish/roaringish_ops.pyx +98 -0
- searcharray/roaringish/scatter_assign.h +31 -0
- searcharray/roaringish/search.cp311-win_arm64.pyd +0 -0
- searcharray/roaringish/search.pyi +25 -0
- searcharray/roaringish/search.pyx +140 -0
- searcharray/roaringish/snp_ops.pxd +13 -0
- searcharray/roaringish/spans.cp311-win_arm64.pyd +0 -0
- searcharray/roaringish/spans.pyx +330 -0
- searcharray/roaringish/unique.cp311-win_arm64.pyd +0 -0
- searcharray/roaringish/unique.pyi +6 -0
- searcharray/roaringish/unique.pyx +145 -0
- searcharray/similarity.py +92 -0
- searcharray/solr.py +355 -0
- searcharray/term_dict.py +59 -0
- searcharray/utils/__init__.py +0 -0
- searcharray/utils/bitcount.py +34 -0
- searcharray/utils/mat_set.py +170 -0
- searcharray/utils/row_viewable_matrix.py +91 -0
- searcharray/utils/sort.py +45 -0
- searcharray-0.0.73.dist-info/METADATA +245 -0
- searcharray-0.0.73.dist-info/RECORD +48 -0
- searcharray-0.0.73.dist-info/WHEEL +5 -0
- searcharray-0.0.73.dist-info/entry_points.txt +2 -0
- searcharray-0.0.73.dist-info/top_level.txt +1 -0
searcharray/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from searcharray.bm25.bm25 import bm25_score # noqa
|
|
Binary file
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# cython: boundscheck=False
|
|
2
|
+
# cython: wraparound=False
|
|
3
|
+
# cython: initializedcheck=False
|
|
4
|
+
# cython: cdivision=True
|
|
5
|
+
# cython: nonecheck=False
|
|
6
|
+
# cython: language_level=3
|
|
7
|
+
cimport numpy as np
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
cdef void _bm25_score(float* term_freqs,
|
|
12
|
+
float* doc_lens,
|
|
13
|
+
float idf,
|
|
14
|
+
float avg_doc_lens,
|
|
15
|
+
float k1,
|
|
16
|
+
float b,
|
|
17
|
+
long length) nogil:
|
|
18
|
+
"""Modify termfreqs in place changing to BM25 score."""
|
|
19
|
+
cdef float one_minus_b = 1 - b
|
|
20
|
+
for _ in range(length):
|
|
21
|
+
term_freqs[0] = (
|
|
22
|
+
term_freqs[0] / (term_freqs[0] + (k1 * (one_minus_b + (b * (doc_lens[0] / avg_doc_lens)))))
|
|
23
|
+
) * idf
|
|
24
|
+
term_freqs += 1
|
|
25
|
+
doc_lens += 1
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def bm25_score(np.ndarray[np.float32_t, ndim=1] term_freqs,
|
|
29
|
+
np.ndarray[np.float32_t, ndim=1] doc_lens,
|
|
30
|
+
float avg_doc_lens,
|
|
31
|
+
float idf,
|
|
32
|
+
float k1,
|
|
33
|
+
float b):
|
|
34
|
+
cdef long length = term_freqs.shape[0]
|
|
35
|
+
_bm25_score(&term_freqs[0],
|
|
36
|
+
&doc_lens[0],
|
|
37
|
+
idf,
|
|
38
|
+
avg_doc_lens,
|
|
39
|
+
k1,
|
|
40
|
+
b,
|
|
41
|
+
length)
|
searcharray/indexing.py
ADDED
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from numpy.typing import NDArray
|
|
3
|
+
import math
|
|
4
|
+
import os
|
|
5
|
+
import sys
|
|
6
|
+
from typing import Iterable, List, Optional
|
|
7
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
8
|
+
from itertools import islice
|
|
9
|
+
from searcharray.phrase.middle_out import MAX_POSN, PosnBitArrayFromFlatBuilder, PosnBitArrayBuilder, PosnBitArrayAlreadyEncBuilder
|
|
10
|
+
from searcharray.term_dict import TermDict
|
|
11
|
+
from searcharray.utils.mat_set import SparseMatSetBuilder
|
|
12
|
+
from searcharray.utils.row_viewable_matrix import RowViewableMatrix
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
17
|
+
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
18
|
+
logger.setLevel(logging.INFO)
|
|
19
|
+
handler.setFormatter(formatter)
|
|
20
|
+
logger.addHandler(handler)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# Set to stdout for debugging
|
|
24
|
+
# logging.basicConfig(level=logging.DEBUG)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def home_directory():
|
|
28
|
+
return os.path.expanduser("~")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def searcharray_home():
|
|
32
|
+
searcharray_dir = os.path.join(home_directory(), ".searcharray")
|
|
33
|
+
# Ensure exists
|
|
34
|
+
os.makedirs(searcharray_dir, exist_ok=True)
|
|
35
|
+
return searcharray_dir
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _compute_doc_lens(posns: np.ndarray, doc_ids: np.ndarray, num_docs: int) -> NDArray[np.float32]:
|
|
39
|
+
"""Given an array of positions, compute the length of each document."""
|
|
40
|
+
doc_lens = np.zeros(num_docs, dtype=np.float32)
|
|
41
|
+
|
|
42
|
+
# Find were we ave posns for each doc
|
|
43
|
+
non_empty_doc_lens = -np.diff(posns).astype(np.float32) + 1
|
|
44
|
+
|
|
45
|
+
non_empty_idxs = np.argwhere(non_empty_doc_lens > 0).flatten()
|
|
46
|
+
non_empty_doc_ids = doc_ids[non_empty_idxs]
|
|
47
|
+
non_empty_doc_lens = non_empty_doc_lens[non_empty_idxs]
|
|
48
|
+
doc_lens[non_empty_doc_ids] = non_empty_doc_lens
|
|
49
|
+
if len(doc_ids) > 0 and doc_ids[-1] not in non_empty_doc_ids:
|
|
50
|
+
doc_lens[doc_ids[-1]] = np.float32(posns[-1] + 1)
|
|
51
|
+
return doc_lens
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def convert_size(size_bytes):
|
|
55
|
+
if size_bytes == 0:
|
|
56
|
+
return "0B"
|
|
57
|
+
size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
|
|
58
|
+
i = int(math.floor(math.log(size_bytes, 1024)))
|
|
59
|
+
p = math.pow(1024, i)
|
|
60
|
+
s = round(size_bytes / p, 2)
|
|
61
|
+
return "%s %s" % (s, size_name[i])
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _gather_tokens(array, tokenizer,
|
|
65
|
+
term_dict, term_doc, start_doc_id=0, trunc_posn=None):
|
|
66
|
+
all_terms = []
|
|
67
|
+
all_docs = []
|
|
68
|
+
all_posns = []
|
|
69
|
+
|
|
70
|
+
logger.info(f"Tokenizing {len(array)} documents")
|
|
71
|
+
|
|
72
|
+
for idx, doc in enumerate(array):
|
|
73
|
+
doc_id = start_doc_id + idx
|
|
74
|
+
|
|
75
|
+
# Speedup - use predefined vocabulary
|
|
76
|
+
terms = np.asarray([term_dict.add_term(token)
|
|
77
|
+
for token in tokenizer(doc)], dtype=np.uint32)[:trunc_posn]
|
|
78
|
+
doc_ids = np.full(len(terms), doc_id, dtype=np.uint32)[:trunc_posn]
|
|
79
|
+
all_terms.append(terms)
|
|
80
|
+
all_docs.append(doc_ids)
|
|
81
|
+
|
|
82
|
+
all_posns.append(np.arange(len(terms)))
|
|
83
|
+
|
|
84
|
+
term_doc.append(np.unique(terms))
|
|
85
|
+
|
|
86
|
+
if idx % 10000 == 0 and idx > 0:
|
|
87
|
+
logger.info(f"Tokenized {doc_id} ({100.0 * (idx / len(array))}%)")
|
|
88
|
+
|
|
89
|
+
# Flatten each
|
|
90
|
+
all_terms = np.concatenate(all_terms)
|
|
91
|
+
all_docs = np.concatenate(all_docs)
|
|
92
|
+
all_posns = np.concatenate(all_posns)
|
|
93
|
+
|
|
94
|
+
logger.info("Tokenization -- vstacking")
|
|
95
|
+
terms_w_posns = np.vstack([all_terms, all_docs, all_posns])
|
|
96
|
+
# del all_terms, all_docs, all_posns
|
|
97
|
+
# gc.collect()
|
|
98
|
+
logger.info("Tokenization -- DONE")
|
|
99
|
+
return terms_w_posns, term_doc
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _lex_sort(terms_w_posns):
|
|
103
|
+
"""Sort terms, then doc_id, then posn."""
|
|
104
|
+
# Because docs / posns already sorted, we can just sort on terms
|
|
105
|
+
# Equivelant to np.lexsort(terms_w_posns[[::-1], :])
|
|
106
|
+
return np.argsort(terms_w_posns[0, :], kind='stable')
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _invert_docs_terms(terms_w_posns):
|
|
110
|
+
"""Sort terms, then doc_id, then posn."""
|
|
111
|
+
# An in place sort could be faster if we could figure this out,
|
|
112
|
+
# an np.sort(arr) is about 1/3 the time of np.argsort(arr)
|
|
113
|
+
# possibly more with just arr.sort()
|
|
114
|
+
lexsort = _lex_sort(terms_w_posns)
|
|
115
|
+
return terms_w_posns[:, lexsort]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _tokenize_batch(array, tokenizer, term_dict, batch_size, batch_beg, truncate=False):
|
|
119
|
+
logger.info(f"{batch_beg} Batch Start tokenization")
|
|
120
|
+
trunc_posn = None
|
|
121
|
+
if truncate:
|
|
122
|
+
trunc_posn = MAX_POSN
|
|
123
|
+
|
|
124
|
+
term_doc = SparseMatSetBuilder()
|
|
125
|
+
terms_w_posns, term_doc = _gather_tokens(array, tokenizer,
|
|
126
|
+
term_dict, term_doc, start_doc_id=batch_beg,
|
|
127
|
+
trunc_posn=trunc_posn)
|
|
128
|
+
|
|
129
|
+
# Use posns to compute doc lens
|
|
130
|
+
doc_lens = _compute_doc_lens(posns=terms_w_posns[2, :],
|
|
131
|
+
doc_ids=(terms_w_posns[1, :] - batch_beg),
|
|
132
|
+
num_docs=len(array))
|
|
133
|
+
logger.info("Inverting docs->terms")
|
|
134
|
+
terms_w_posns = _invert_docs_terms(terms_w_posns)
|
|
135
|
+
|
|
136
|
+
# Encode posns to bit array
|
|
137
|
+
logger.info("Encoding positions to bit array")
|
|
138
|
+
posns = PosnBitArrayFromFlatBuilder(terms_w_posns, max_doc_id=(batch_beg + len(array) - 1))
|
|
139
|
+
bit_posns = posns.build()
|
|
140
|
+
|
|
141
|
+
if np.any(doc_lens > MAX_POSN):
|
|
142
|
+
raise ValueError(f"Document length exceeds maximum of {MAX_POSN}")
|
|
143
|
+
|
|
144
|
+
logger.info("Batch tokenization complete")
|
|
145
|
+
return batch_beg, term_doc, bit_posns, doc_lens
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def batch_iterator(iterator, batch_size):
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
:param iterator: The iterator to slice into batches.
|
|
152
|
+
:param batch_size: The size of each batch.
|
|
153
|
+
"""
|
|
154
|
+
batch_beg = 0
|
|
155
|
+
it = iter(iterator)
|
|
156
|
+
while True:
|
|
157
|
+
batch = list(islice(it, batch_size))
|
|
158
|
+
if not batch:
|
|
159
|
+
break
|
|
160
|
+
yield batch_beg, batch
|
|
161
|
+
batch_beg += batch_size
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _process_batches(term_doc, batch_size,
|
|
165
|
+
futures,
|
|
166
|
+
last_batch_beg_processed,
|
|
167
|
+
bit_posns=None,
|
|
168
|
+
doc_lens=None,
|
|
169
|
+
truncate=False):
|
|
170
|
+
batch_results = [None] * len(futures)
|
|
171
|
+
batch_beg = 0
|
|
172
|
+
for future in as_completed(futures):
|
|
173
|
+
try:
|
|
174
|
+
batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = future.result()
|
|
175
|
+
idx = (batch_beg - last_batch_beg_processed) // batch_size
|
|
176
|
+
assert batch_results[idx] is None
|
|
177
|
+
batch_results[idx] \
|
|
178
|
+
= (batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens)
|
|
179
|
+
except ValueError as e:
|
|
180
|
+
logger.error(e)
|
|
181
|
+
logger.error(f"Batch {batch_beg} failed to tokenize")
|
|
182
|
+
raise e
|
|
183
|
+
|
|
184
|
+
logger.info(f"(main thread) Processing {len(batch_results)} batch results")
|
|
185
|
+
for result in batch_results:
|
|
186
|
+
assert result is not None
|
|
187
|
+
batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = result
|
|
188
|
+
term_doc.concat(batch_term_doc)
|
|
189
|
+
if bit_posns is None:
|
|
190
|
+
bit_posns = batch_bit_posns
|
|
191
|
+
else:
|
|
192
|
+
logger.info("(main thread) Concatenating bit positions")
|
|
193
|
+
bit_posns.concat(batch_bit_posns)
|
|
194
|
+
logger.info("(main thread) Concatenated bit positions... Done")
|
|
195
|
+
|
|
196
|
+
doc_lens.append(batch_doc_lens)
|
|
197
|
+
return bit_posns
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def build_index_no_workers(array: Iterable, tokenizer, batch_size=10000,
|
|
201
|
+
data_dir: Optional[str] = None,
|
|
202
|
+
cache_gt_than=25,
|
|
203
|
+
truncate=False):
|
|
204
|
+
term_dict = TermDict()
|
|
205
|
+
term_doc = SparseMatSetBuilder()
|
|
206
|
+
doc_lens: List[np.ndarray] = []
|
|
207
|
+
bit_posns = None
|
|
208
|
+
|
|
209
|
+
logger.info("Indexing begins w/ NO workers")
|
|
210
|
+
for batch_beg, batch in batch_iterator(array, batch_size):
|
|
211
|
+
batch_beg, batch_term_doc, batch_bit_posns, batch_doc_lens = _tokenize_batch(batch, tokenizer, term_dict, batch_size, batch_beg, truncate=truncate)
|
|
212
|
+
term_doc.concat(batch_term_doc)
|
|
213
|
+
if bit_posns is None:
|
|
214
|
+
bit_posns = batch_bit_posns
|
|
215
|
+
else:
|
|
216
|
+
bit_posns.concat(batch_bit_posns)
|
|
217
|
+
doc_lens.append(batch_doc_lens)
|
|
218
|
+
|
|
219
|
+
doc_lens = np.concatenate(doc_lens)
|
|
220
|
+
|
|
221
|
+
avg_doc_length = np.mean(doc_lens)
|
|
222
|
+
|
|
223
|
+
term_doc_built = RowViewableMatrix(term_doc.build())
|
|
224
|
+
logger.info("Indexing from tokenization complete")
|
|
225
|
+
assert bit_posns is not None
|
|
226
|
+
# if data_dir is None:
|
|
227
|
+
# data_dir = searcharray_home()
|
|
228
|
+
if data_dir is not None:
|
|
229
|
+
logger.info(f"Memmapping bit positions to {data_dir}")
|
|
230
|
+
bit_posns.memmap(data_dir)
|
|
231
|
+
bit_posns.cache_gt_than = cache_gt_than
|
|
232
|
+
return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def build_index_from_tokenizer(array: Iterable, tokenizer, batch_size=10000,
|
|
236
|
+
data_dir: Optional[str] = None,
|
|
237
|
+
truncate=False, workers=4,
|
|
238
|
+
cache_gt_than=25):
|
|
239
|
+
"""Build index directly from tokenizing docs (array of string)."""
|
|
240
|
+
if workers == 1:
|
|
241
|
+
return build_index_no_workers(array, tokenizer, batch_size=batch_size,
|
|
242
|
+
cache_gt_than=cache_gt_than,
|
|
243
|
+
data_dir=data_dir, truncate=truncate)
|
|
244
|
+
term_dict = TermDict()
|
|
245
|
+
term_doc = SparseMatSetBuilder()
|
|
246
|
+
doc_lens: List[np.ndarray] = []
|
|
247
|
+
bit_posns = None
|
|
248
|
+
|
|
249
|
+
logger.info(f"Indexing begins w/ {workers} workers")
|
|
250
|
+
futures = []
|
|
251
|
+
last_batch_beg_processed = 0
|
|
252
|
+
|
|
253
|
+
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
254
|
+
|
|
255
|
+
for batch_beg, batch in batch_iterator(array, batch_size):
|
|
256
|
+
futures.append(executor.submit(_tokenize_batch,
|
|
257
|
+
batch,
|
|
258
|
+
tokenizer,
|
|
259
|
+
term_dict,
|
|
260
|
+
batch_size,
|
|
261
|
+
batch_beg,
|
|
262
|
+
truncate=truncate))
|
|
263
|
+
|
|
264
|
+
if len(futures) >= workers:
|
|
265
|
+
logger.info(f"Collecting {len(futures)} futures")
|
|
266
|
+
bit_posns = _process_batches(term_doc, batch_size,
|
|
267
|
+
futures,
|
|
268
|
+
last_batch_beg_processed,
|
|
269
|
+
bit_posns, doc_lens, truncate=truncate)
|
|
270
|
+
logger.info(f"{batch_beg} Batch Complete")
|
|
271
|
+
logger.info(f"Roaringish NBytes -- {convert_size(bit_posns.nbytes)}")
|
|
272
|
+
logger.info(f"Term Dict Size -- {len(term_dict)}")
|
|
273
|
+
|
|
274
|
+
last_batch_beg_processed += len(futures) * batch_size
|
|
275
|
+
futures = []
|
|
276
|
+
if len(futures) > 0:
|
|
277
|
+
bit_posns = _process_batches(term_doc, batch_size,
|
|
278
|
+
futures,
|
|
279
|
+
last_batch_beg_processed,
|
|
280
|
+
bit_posns, doc_lens, truncate=truncate)
|
|
281
|
+
|
|
282
|
+
doc_lens = np.concatenate(doc_lens)
|
|
283
|
+
|
|
284
|
+
avg_doc_length = np.mean(doc_lens)
|
|
285
|
+
|
|
286
|
+
term_doc_built = RowViewableMatrix(term_doc.build())
|
|
287
|
+
logger.info("Indexing from tokenization complete")
|
|
288
|
+
assert bit_posns is not None
|
|
289
|
+
# if data_dir is None:
|
|
290
|
+
# data_dir = searcharray_home()
|
|
291
|
+
if data_dir is not None:
|
|
292
|
+
logger.info(f"Memmapping bit positions to {data_dir}")
|
|
293
|
+
bit_posns.memmap(data_dir)
|
|
294
|
+
bit_posns.cache_gt_than = cache_gt_than
|
|
295
|
+
return term_doc_built, bit_posns, term_dict, avg_doc_length, np.array(doc_lens, dtype=np.float32)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def build_index_from_terms_list(postings, Terms):
|
|
299
|
+
"""Bulid an index from postings that are already tokenized and point at their term frequencies/posns."""
|
|
300
|
+
term_dict = TermDict()
|
|
301
|
+
term_doc = SparseMatSetBuilder()
|
|
302
|
+
doc_lens = []
|
|
303
|
+
avg_doc_length = 0
|
|
304
|
+
num_postings = 0
|
|
305
|
+
posns = PosnBitArrayBuilder()
|
|
306
|
+
posns_enc = PosnBitArrayAlreadyEncBuilder()
|
|
307
|
+
|
|
308
|
+
# COPY 1
|
|
309
|
+
# Consume generator (tokenized postings) into list
|
|
310
|
+
# its faster this way?
|
|
311
|
+
postings = list(postings)
|
|
312
|
+
|
|
313
|
+
# COPY 2
|
|
314
|
+
for doc_id, tokenized in enumerate(postings):
|
|
315
|
+
if isinstance(tokenized, dict):
|
|
316
|
+
tokenized = Terms(tokenized, doc_len=len(tokenized))
|
|
317
|
+
elif not isinstance(tokenized, Terms):
|
|
318
|
+
raise TypeError("Expected a Terms or a dict")
|
|
319
|
+
|
|
320
|
+
if tokenized.encoded:
|
|
321
|
+
posns = posns_enc
|
|
322
|
+
|
|
323
|
+
doc_lens.append(tokenized.doc_len)
|
|
324
|
+
avg_doc_length += doc_lens[-1]
|
|
325
|
+
terms = []
|
|
326
|
+
for token, term_freq in tokenized.terms():
|
|
327
|
+
term_id = term_dict.add_term(token)
|
|
328
|
+
terms.append(term_id)
|
|
329
|
+
positions = tokenized.positions(token)
|
|
330
|
+
if positions is not None:
|
|
331
|
+
posns.add_posns(doc_id, term_id, positions)
|
|
332
|
+
|
|
333
|
+
term_doc.append(terms)
|
|
334
|
+
|
|
335
|
+
posns.ensure_capacity(doc_id)
|
|
336
|
+
num_postings += 1
|
|
337
|
+
|
|
338
|
+
if num_postings > 0:
|
|
339
|
+
avg_doc_length /= num_postings
|
|
340
|
+
|
|
341
|
+
bit_posns = posns.build()
|
|
342
|
+
return RowViewableMatrix(term_doc.build()), bit_posns, term_dict, avg_doc_length, np.array(doc_lens)
|
|
File without changes
|