searcharray 0.0.73__cp313-cp313-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- searcharray/__init__.py +2 -0
- searcharray/bm25/__init__.py +1 -0
- searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
- searcharray/bm25/bm25.pyi +11 -0
- searcharray/bm25/bm25.pyx +41 -0
- searcharray/indexing.py +342 -0
- searcharray/phrase/__init__.py +0 -0
- searcharray/phrase/bigram_freqs.py +307 -0
- searcharray/phrase/memmap_arrays.py +208 -0
- searcharray/phrase/middle_out.py +553 -0
- searcharray/phrase/spans.py +187 -0
- searcharray/postings.py +708 -0
- searcharray/roaringish/__init__.py +7 -0
- searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/intersect.pyi +25 -0
- searcharray/roaringish/intersect.pyx +390 -0
- searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/merge.pyi +16 -0
- searcharray/roaringish/merge.pyx +232 -0
- searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/popcount.pyi +36 -0
- searcharray/roaringish/popcount.pyx +278 -0
- searcharray/roaringish/roaringish.py +298 -0
- searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/roaringish_ops.pyx +98 -0
- searcharray/roaringish/scatter_assign.h +31 -0
- searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/search.pyi +25 -0
- searcharray/roaringish/search.pyx +140 -0
- searcharray/roaringish/snp_ops.pxd +13 -0
- searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/spans.pyx +330 -0
- searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/unique.pyi +6 -0
- searcharray/roaringish/unique.pyx +145 -0
- searcharray/similarity.py +92 -0
- searcharray/solr.py +355 -0
- searcharray/term_dict.py +59 -0
- searcharray/utils/__init__.py +0 -0
- searcharray/utils/bitcount.py +34 -0
- searcharray/utils/mat_set.py +170 -0
- searcharray/utils/row_viewable_matrix.py +91 -0
- searcharray/utils/sort.py +45 -0
- searcharray-0.0.73.dist-info/METADATA +245 -0
- searcharray-0.0.73.dist-info/RECORD +48 -0
- searcharray-0.0.73.dist-info/WHEEL +5 -0
- searcharray-0.0.73.dist-info/entry_points.txt +2 -0
- searcharray-0.0.73.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
"""Given two roaringish encoded arrays, count bigram matches between them."""
|
|
2
|
+
import numpy as np
|
|
3
|
+
from typing import Tuple, Optional
|
|
4
|
+
from searcharray.roaringish import RoaringishEncoder
|
|
5
|
+
import logging
|
|
6
|
+
from enum import Enum
|
|
7
|
+
|
|
8
|
+
from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at, key_sum_over, sort_merge_counts
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
# When running in pytest
|
|
14
|
+
import sys # noqa
|
|
15
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
16
|
+
handler.setLevel(logging.ERROR)
|
|
17
|
+
formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
|
|
18
|
+
handler.setFormatter(formatter)
|
|
19
|
+
logger.addHandler(handler)
|
|
20
|
+
logger.setLevel(logging.ERROR)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
encoder = RoaringishEncoder()
|
|
24
|
+
|
|
25
|
+
# To not constantly type coerce
|
|
26
|
+
_64 = np.uint64(64)
|
|
27
|
+
_2 = np.uint64(2)
|
|
28
|
+
_1 = np.uint64(1)
|
|
29
|
+
_0 = np.uint64(0)
|
|
30
|
+
_neg1 = np.int64(-1)
|
|
31
|
+
_upper_bit = _1 << (encoder.payload_lsb_bits - _1)
|
|
32
|
+
|
|
33
|
+
MAX_POSN = encoder.max_payload
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Continuation(Enum):
|
|
37
|
+
"""Enum for continuation of bigram search."""
|
|
38
|
+
LHS = 0
|
|
39
|
+
RHS = 1
|
|
40
|
+
BOTH = 2
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _any_payload_empty(enc: np.ndarray):
|
|
44
|
+
"""Check if the payload has any empty of enc."""
|
|
45
|
+
return np.any((enc & encoder.payload_lsb_mask) == 0)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _adj_to_phrase_freq(overlap: np.ndarray, adjacents: np.ndarray) -> np.ndarray:
|
|
49
|
+
"""Adjust phrase freqs for adjacent matches."""
|
|
50
|
+
# ?? 1 1 1 0 1
|
|
51
|
+
# we need to treat the 2nd consecutive 1 as 'not a match'
|
|
52
|
+
# and also update 'term' to not include it
|
|
53
|
+
# Search for foo foo
|
|
54
|
+
# foo foo -> phrase freq 1
|
|
55
|
+
# foo foo foo -> 2 adjs, phrase freqs = 1
|
|
56
|
+
# foo foo foo foo -> 3 adjs, phrase freqs = 2
|
|
57
|
+
# [foo foo] [foo foo] foo -> 4 adjs, phrase freqs = 2
|
|
58
|
+
# [foo foo] [foo foo] mal [foo foo] -> 4 adjs, phrase freqs = 3
|
|
59
|
+
consecutive_ones = encoder.payload_lsb(overlap & (overlap << _1))
|
|
60
|
+
consecutive_ones = popcount64(consecutive_ones)
|
|
61
|
+
adjacents -= -np.floor_divide(consecutive_ones, -2, dtype=np.int64)
|
|
62
|
+
return adjacents
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
|
|
66
|
+
lhs_doc_ids: np.ndarray,
|
|
67
|
+
cont: Continuation = Continuation.RHS) -> Tuple[Tuple[np.ndarray, np.ndarray],
|
|
68
|
+
Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
69
|
+
"""Count bigram matches when lhs / rhs are same term.
|
|
70
|
+
|
|
71
|
+
Despite being the same term, its possible lhs_int != rhs_int due to lhs corresponding
|
|
72
|
+
to a past match of identical term."""
|
|
73
|
+
# Shift RHS to left by 1
|
|
74
|
+
# 3 2 1 0 3 2 1 0
|
|
75
|
+
# ?? foo foo foo -> foo foo foo ??
|
|
76
|
+
rhs_shift = rhs_int << _1
|
|
77
|
+
# Now we have unshifted and shifted to count adjacents:
|
|
78
|
+
# foo foo foo ?? & ?? foo foo foo
|
|
79
|
+
# -> ?? foo foo ?? or two times term is adjacent
|
|
80
|
+
# Count these bits...
|
|
81
|
+
overlap = lhs_int & rhs_shift
|
|
82
|
+
adj_count = encoder.payload_lsb(overlap)
|
|
83
|
+
adjacents = popcount64(adj_count).view(np.int64)
|
|
84
|
+
|
|
85
|
+
adjusted = _adj_to_phrase_freq(overlap, adjacents).astype(np.uint64)
|
|
86
|
+
lhs_doc_ids, phrase_freqs = key_sum_over(lhs_doc_ids, adjusted)
|
|
87
|
+
# Continue with ?? ?? foo foo
|
|
88
|
+
# term_int without lsbs
|
|
89
|
+
term_int_msbs = lhs_int & ~encoder.payload_lsb_mask
|
|
90
|
+
rhs_cont = None
|
|
91
|
+
lhs_cont = None
|
|
92
|
+
# rhs_cont = term_int_msbs | encoder.payload_lsb(rhs_shift)
|
|
93
|
+
rhs_cont = encoder.payload_lsb(rhs_shift & rhs_int) | term_int_msbs
|
|
94
|
+
lhs_cont = term_int_msbs | encoder.payload_lsb(lhs_int & (lhs_int >> _1))
|
|
95
|
+
|
|
96
|
+
if cont not in [Continuation.RHS, Continuation.BOTH]:
|
|
97
|
+
rhs_cont = None
|
|
98
|
+
if cont not in [Continuation.LHS, Continuation.BOTH]:
|
|
99
|
+
lhs_cont = None
|
|
100
|
+
|
|
101
|
+
return (lhs_doc_ids, phrase_freqs), (lhs_cont, rhs_cont)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _inner_bigram_freqs(lhs: np.ndarray,
|
|
105
|
+
rhs: np.ndarray,
|
|
106
|
+
lhs_int: np.ndarray,
|
|
107
|
+
rhs_int: np.ndarray,
|
|
108
|
+
cont: Continuation = Continuation.RHS) -> Tuple[Tuple[np.ndarray, np.ndarray],
|
|
109
|
+
Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
110
|
+
"""Count bigram matches between two encoded arrays, within a 64 bit word with same MSBs.
|
|
111
|
+
|
|
112
|
+
Parameters:
|
|
113
|
+
-----------
|
|
114
|
+
lhs: roaringish encoded posn array of left term
|
|
115
|
+
rhs: roaringish encoded posn array of right term
|
|
116
|
+
lhs_int: np.ndarray, intersected lhs
|
|
117
|
+
rhs_int: np.ndarray, intersected rhs
|
|
118
|
+
cont_rhs: bool, whether to continue matching on the rhs or lhs
|
|
119
|
+
|
|
120
|
+
Returns:
|
|
121
|
+
--------
|
|
122
|
+
doc_ids, overlap: the doc ids and overlap bits of the bigram matches (to be collected)
|
|
123
|
+
cont_next: tuple the next (lhs or rhs) array to continue matching
|
|
124
|
+
|
|
125
|
+
"""
|
|
126
|
+
# lhs_int, rhs_int = encoder.intersect(lhs, rhs)
|
|
127
|
+
lhs_doc_ids = encoder.keys(lhs_int)
|
|
128
|
+
if len(lhs_int) != len(rhs_int):
|
|
129
|
+
raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
|
|
130
|
+
# No intersections, no inner bigram matches
|
|
131
|
+
if len(lhs_int) == 0 and cont == Continuation.RHS:
|
|
132
|
+
return (np.array([], dtype=np.uint64), np.array([])), (None, rhs_int)
|
|
133
|
+
if len(lhs_int) == 0 and cont == Continuation.LHS:
|
|
134
|
+
return (np.array([], dtype=np.uint64), np.array([])), (lhs_int, None)
|
|
135
|
+
if len(lhs_int) == 0 and cont == Continuation.BOTH:
|
|
136
|
+
return (np.array([], dtype=np.uint64), np.array([])), (lhs_int, rhs_int)
|
|
137
|
+
|
|
138
|
+
# For perf we don't check all values, but if overlapping posns allowed in future, maybe we should
|
|
139
|
+
same_term = (len(lhs_int) == len(rhs_int) and np.all(lhs_int == rhs_int))
|
|
140
|
+
if same_term:
|
|
141
|
+
return _inner_bigram_same_term(lhs_int, rhs_int, lhs_doc_ids, cont)
|
|
142
|
+
|
|
143
|
+
rhs_next = None
|
|
144
|
+
lhs_next = None
|
|
145
|
+
overlap_bits = (lhs_int & encoder.payload_lsb_mask) & ((rhs_int & encoder.payload_lsb_mask) >> _1)
|
|
146
|
+
|
|
147
|
+
if cont in [Continuation.RHS, Continuation.BOTH]:
|
|
148
|
+
rhs_next = (overlap_bits << _1) & encoder.payload_lsb_mask
|
|
149
|
+
rhs_next |= (rhs_int & (encoder.key_mask | encoder.payload_msb_mask))
|
|
150
|
+
if cont in [Continuation.LHS, Continuation.BOTH]:
|
|
151
|
+
lhs_next = overlap_bits.copy()
|
|
152
|
+
lhs_next |= (lhs_int & (encoder.key_mask | encoder.payload_msb_mask))
|
|
153
|
+
|
|
154
|
+
lhs_doc_ids, counts = popcount_reduce_at(lhs_doc_ids, overlap_bits)
|
|
155
|
+
return (lhs_doc_ids, counts), (lhs_next, rhs_next)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
|
|
159
|
+
lhs_adj: np.ndarray, rhs_adj: np.ndarray,
|
|
160
|
+
cont: Continuation = Continuation.RHS) -> Tuple[Tuple[np.ndarray, np.ndarray],
|
|
161
|
+
Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
162
|
+
"""Count bigram matches between two encoded arrays where they occur in adjacent 64 bit words.
|
|
163
|
+
|
|
164
|
+
Returns:
|
|
165
|
+
--------
|
|
166
|
+
ids 1 id per adjacent for that doc
|
|
167
|
+
Tuple[lhs_next, rhs_next]: the next lhs/rhs array to continue matching
|
|
168
|
+
|
|
169
|
+
"""
|
|
170
|
+
lhs_doc_ids = encoder.keys(lhs_adj)
|
|
171
|
+
# lhs lsb set and rhs lsb's most significant bit set
|
|
172
|
+
matches = ((lhs_adj & _upper_bit) != 0) & ((rhs_adj & _1) != 0)
|
|
173
|
+
unique, counts = np.unique(lhs_doc_ids[matches], return_counts=True)
|
|
174
|
+
# phrase_freqs[unique] += counts
|
|
175
|
+
# Set lsb to 0 where no match, lsb to 1 where match
|
|
176
|
+
rhs_next = None if cont == Continuation.LHS else np.asarray([], dtype=np.uint64)
|
|
177
|
+
lhs_next = None if cont == Continuation.RHS else np.asarray([], dtype=np.uint64)
|
|
178
|
+
if np.any(matches):
|
|
179
|
+
if cont in [Continuation.RHS, Continuation.BOTH]:
|
|
180
|
+
rhs_next = rhs_adj[matches]
|
|
181
|
+
assert rhs_next is not None
|
|
182
|
+
rhs_next = encoder.header(rhs_next) | _1
|
|
183
|
+
if cont in [Continuation.LHS, Continuation.BOTH]:
|
|
184
|
+
lhs_next = lhs_adj[matches]
|
|
185
|
+
assert lhs_next is not None
|
|
186
|
+
lhs_next = encoder.header(lhs_next) | _upper_bit
|
|
187
|
+
rhs_next = None
|
|
188
|
+
return (unique, counts), (lhs_next, rhs_next)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _set_adjbit_at_header(next_inner: np.ndarray, next_adj: np.ndarray,
|
|
192
|
+
cont: Continuation = Continuation.RHS) -> np.ndarray:
|
|
193
|
+
"""Merge two encoded arrays on their headers."""
|
|
194
|
+
if len(next_inner) == 0:
|
|
195
|
+
return next_adj
|
|
196
|
+
if len(next_adj) == 0:
|
|
197
|
+
return next_inner
|
|
198
|
+
|
|
199
|
+
same_header_inner, same_header_adj = intersect(next_inner, next_adj,
|
|
200
|
+
mask=encoder.header_mask)
|
|
201
|
+
# Set _1 on intersection
|
|
202
|
+
ignore_mask = np.ones(len(next_adj), dtype=bool)
|
|
203
|
+
ignore_mask[same_header_adj] = False
|
|
204
|
+
if len(same_header_inner) > 0 and cont == Continuation.RHS:
|
|
205
|
+
next_inner[same_header_inner] |= _1
|
|
206
|
+
next_adj = next_adj[ignore_mask]
|
|
207
|
+
if len(same_header_adj) > 0 and cont == Continuation.LHS:
|
|
208
|
+
next_inner[same_header_inner] |= _upper_bit
|
|
209
|
+
next_adj = next_adj[ignore_mask]
|
|
210
|
+
return merge(next_inner, next_adj)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def bigram_freqs(lhs: np.ndarray,
|
|
214
|
+
rhs: np.ndarray,
|
|
215
|
+
cont: Continuation = Continuation.RHS) -> Tuple[Tuple[np.ndarray, np.ndarray],
|
|
216
|
+
Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
|
|
217
|
+
"""Count bigram matches between two roaringish encoded posn arrays.
|
|
218
|
+
Also return connection on right hand of the bigram
|
|
219
|
+
|
|
220
|
+
Parameters:
|
|
221
|
+
-----------
|
|
222
|
+
lhs: roaringish encoded posn array of left term
|
|
223
|
+
rhs: roaringish encoded posn array of right term
|
|
224
|
+
phrase_freqs: np.ndarray, preallocated phrase freqs for output
|
|
225
|
+
cont_rhs: bool, whether to continue matching on the rhs or lhs
|
|
226
|
+
|
|
227
|
+
Returns:
|
|
228
|
+
--------
|
|
229
|
+
phrase_freqs: number of matches per doc (updated input)
|
|
230
|
+
rhs_next: encoded array of end of each bigram
|
|
231
|
+
(use as lhs on subsequent calls to continue the phrase)
|
|
232
|
+
|
|
233
|
+
"""
|
|
234
|
+
# Example zooming into just one 64 bit word (this of course is repeated over
|
|
235
|
+
# the entire array, for all docs and posn msbs)
|
|
236
|
+
#
|
|
237
|
+
# ***************************************************************************
|
|
238
|
+
# "Inner" matches:
|
|
239
|
+
# As roaringish arrays put posns on a 64 bit word, we first search for
|
|
240
|
+
# simpler matches within 64 bit words from same docs with same posn header.
|
|
241
|
+
# We do this by intersecting on the header (doc_id and posn_msb), then checking
|
|
242
|
+
# the following pattern in the posn bits:
|
|
243
|
+
#
|
|
244
|
+
# doc_id posns_msb posns_bits
|
|
245
|
+
# lhs_term: 1234 101 0010001010
|
|
246
|
+
# rhs_term: 1234 101 0101000100
|
|
247
|
+
# ^^ ^^ <- inner matches, terms next to each other
|
|
248
|
+
#
|
|
249
|
+
lhs_int, rhs_int, lhs_adj, rhs_adj = encoder.intersect_candidates(lhs, rhs)
|
|
250
|
+
|
|
251
|
+
(ids, counts), (lhs_next_inner, rhs_next_inner)\
|
|
252
|
+
= _inner_bigram_freqs(lhs, rhs,
|
|
253
|
+
lhs_int, rhs_int,
|
|
254
|
+
cont)
|
|
255
|
+
# ***************************************************************************
|
|
256
|
+
# "Adjacent" matches:
|
|
257
|
+
# Of course we also need to check for matches that span two 64 bit words,
|
|
258
|
+
# thats what adjacent matches do. First finding where header meets header + 1,
|
|
259
|
+
# then checking the bit pattern shown below:
|
|
260
|
+
#
|
|
261
|
+
# doc_id posns_msb posns_bits
|
|
262
|
+
# lhs_term: 1234 101 1010000000
|
|
263
|
+
# rhs_term: 1234 111 0001000101
|
|
264
|
+
# ^ ^ <- adjacent matches, terms next to each other
|
|
265
|
+
# but note the posn_msb of lhs is
|
|
266
|
+
# 1 less than the posn_msb of rhs
|
|
267
|
+
# so we detect these differently
|
|
268
|
+
(adj_doc_ids, adj_counts), (lhs_next_adj, rhs_next_adj) \
|
|
269
|
+
= _adjacent_bigram_freqs(lhs, rhs,
|
|
270
|
+
lhs_adj, rhs_adj,
|
|
271
|
+
cont)
|
|
272
|
+
|
|
273
|
+
doc_ids1, counts1 = sort_merge_counts(ids, counts.astype(np.float32),
|
|
274
|
+
adj_doc_ids, adj_counts.astype(np.float32))
|
|
275
|
+
|
|
276
|
+
# ***************************************************************************
|
|
277
|
+
# rhs_next is where the bigram ends on the rhs side, we can use this
|
|
278
|
+
# to continue the phrase matching
|
|
279
|
+
#
|
|
280
|
+
# Combining the examples above:
|
|
281
|
+
# rhs_next_inner: 1234 101 0001000100
|
|
282
|
+
# ^ ^ <- where we had a match (inner)
|
|
283
|
+
#
|
|
284
|
+
# Merged with
|
|
285
|
+
# rhs_next_adj: 1234 111 0000000001
|
|
286
|
+
# ^ <- where we had a match (adjacent)
|
|
287
|
+
#
|
|
288
|
+
# So we merge these... in this case, we have a two-value array
|
|
289
|
+
#
|
|
290
|
+
# rhs_next: 1234,101,0001000100|1234,111,0000000001
|
|
291
|
+
#
|
|
292
|
+
#
|
|
293
|
+
# Now rhs_next can be the LHS of the next call to bigram_freqs
|
|
294
|
+
# to continue the phrase
|
|
295
|
+
#
|
|
296
|
+
rhs_next = None
|
|
297
|
+
lhs_next = None
|
|
298
|
+
if cont in [Continuation.RHS, Continuation.BOTH]:
|
|
299
|
+
assert rhs_next_inner is not None
|
|
300
|
+
assert rhs_next_adj is not None
|
|
301
|
+
rhs_next = _set_adjbit_at_header(rhs_next_inner, rhs_next_adj, Continuation.RHS)
|
|
302
|
+
if cont in [Continuation.LHS, Continuation.BOTH]:
|
|
303
|
+
assert lhs_next_inner is not None
|
|
304
|
+
assert lhs_next_adj is not None
|
|
305
|
+
lhs_next = _set_adjbit_at_header(lhs_next_inner, lhs_next_adj, Continuation.LHS)
|
|
306
|
+
|
|
307
|
+
return (doc_ids1, counts1), (lhs_next, rhs_next)
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import os
|
|
3
|
+
import reprlib
|
|
4
|
+
from typing import Optional, List, Dict
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def create_filename(data_dir: str):
|
|
8
|
+
"""Create a filename for the memory-mapped array"""
|
|
9
|
+
# Ls path and get the number of files
|
|
10
|
+
files = os.listdir(data_dir)
|
|
11
|
+
num_files = len(files)
|
|
12
|
+
return os.path.join(data_dir, f'{num_files}')
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ArrayDict:
|
|
16
|
+
"""An array of ints to np.array as contiguous memory"""
|
|
17
|
+
|
|
18
|
+
def __init__(self):
|
|
19
|
+
self.dtype = np.uint64
|
|
20
|
+
self.data = np.ndarray(0, dtype=self.dtype)
|
|
21
|
+
self.metadata = {}
|
|
22
|
+
|
|
23
|
+
def __repr__(self):
|
|
24
|
+
repr_obj = reprlib.Repr()
|
|
25
|
+
return f'ArrayDict({repr_obj.repr(self.metadata)})'
|
|
26
|
+
|
|
27
|
+
@staticmethod
|
|
28
|
+
def from_arrays(arrays: List[np.ndarray]):
|
|
29
|
+
arr = ArrayDict()
|
|
30
|
+
metadata = {}
|
|
31
|
+
total_len = 0
|
|
32
|
+
for i, array in enumerate(arrays):
|
|
33
|
+
metadata[i] = {'offset': total_len, 'length': array.size}
|
|
34
|
+
total_len += array.size
|
|
35
|
+
data = np.concatenate(arrays)
|
|
36
|
+
arr.data = data
|
|
37
|
+
arr.metadata = metadata
|
|
38
|
+
return arr
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def from_array_with_boundaries(data: np.ndarray,
|
|
42
|
+
ids: np.ndarray,
|
|
43
|
+
boundaries: np.ndarray):
|
|
44
|
+
metadata = {}
|
|
45
|
+
for idx, (beg, end) in enumerate(zip(boundaries[:-1], boundaries[1:])):
|
|
46
|
+
offset = int(beg)
|
|
47
|
+
length = int(end - beg)
|
|
48
|
+
actual_id = ids[idx]
|
|
49
|
+
metadata[int(actual_id)] = {'offset': offset, 'length': length}
|
|
50
|
+
arr = ArrayDict()
|
|
51
|
+
arr.data = data
|
|
52
|
+
arr.metadata = metadata
|
|
53
|
+
return arr
|
|
54
|
+
|
|
55
|
+
@staticmethod
|
|
56
|
+
def concat(lhs: 'ArrayDict', rhs: 'ArrayDict', sort=True):
|
|
57
|
+
arr = ArrayDict()
|
|
58
|
+
metadata: Dict[int, Dict[str, int]] = {}
|
|
59
|
+
lst_of_arrays: List[np.ndarray] = []
|
|
60
|
+
|
|
61
|
+
curr_offset = 0
|
|
62
|
+
last_offset = 0
|
|
63
|
+
|
|
64
|
+
fetched_keys = set()
|
|
65
|
+
for key, value in lhs.items():
|
|
66
|
+
fetched_keys.add(key)
|
|
67
|
+
all_for_key = [value]
|
|
68
|
+
curr_offset += value.size
|
|
69
|
+
if key in rhs.metadata:
|
|
70
|
+
rhs_value = rhs[key]
|
|
71
|
+
all_for_key.append(rhs_value)
|
|
72
|
+
curr_offset += rhs_value.size
|
|
73
|
+
metadata[key] = {'offset': last_offset, 'length': curr_offset - last_offset}
|
|
74
|
+
last_offset = curr_offset
|
|
75
|
+
lst_of_arrays.append(np.concatenate(all_for_key))
|
|
76
|
+
|
|
77
|
+
for key, value in rhs.items():
|
|
78
|
+
if key not in fetched_keys:
|
|
79
|
+
curr_offset += value.size
|
|
80
|
+
lst_of_arrays.append(value)
|
|
81
|
+
metadata[key] = {'offset': last_offset, 'length': curr_offset - last_offset}
|
|
82
|
+
last_offset = curr_offset
|
|
83
|
+
|
|
84
|
+
arr.metadata = metadata
|
|
85
|
+
for curr_arr in lst_of_arrays:
|
|
86
|
+
if sort:
|
|
87
|
+
curr_arr.sort()
|
|
88
|
+
|
|
89
|
+
if len(lst_of_arrays) > 0:
|
|
90
|
+
arr.data = np.concatenate(lst_of_arrays)
|
|
91
|
+
return arr
|
|
92
|
+
|
|
93
|
+
def __getitem__(self, key):
|
|
94
|
+
key = int(key)
|
|
95
|
+
if key in self.metadata:
|
|
96
|
+
offset = self.metadata[key]['offset']
|
|
97
|
+
length = self.metadata[key]['length']
|
|
98
|
+
return self.data[offset:offset + length]
|
|
99
|
+
else:
|
|
100
|
+
raise KeyError(f'Key {key} not found.')
|
|
101
|
+
|
|
102
|
+
def __setitem__(self, key, value):
|
|
103
|
+
key = int(key)
|
|
104
|
+
if value.dtype != self.dtype:
|
|
105
|
+
raise ValueError(f'Value must be of type {self.dtype}')
|
|
106
|
+
offset = self.data.size
|
|
107
|
+
length = value.size
|
|
108
|
+
self.metadata[key] = {'offset': offset, 'length': length}
|
|
109
|
+
self.data = np.append(self.data, value)
|
|
110
|
+
|
|
111
|
+
def __delitem__(self, key):
|
|
112
|
+
key = int(key)
|
|
113
|
+
if key in self.metadata:
|
|
114
|
+
del self.metadata[key]
|
|
115
|
+
else:
|
|
116
|
+
raise KeyError(f'Key {key} not found.')
|
|
117
|
+
|
|
118
|
+
def __len__(self):
|
|
119
|
+
return len(self.metadata)
|
|
120
|
+
|
|
121
|
+
def __iter__(self):
|
|
122
|
+
yield from self.metadata
|
|
123
|
+
|
|
124
|
+
def __contains__(self, key):
|
|
125
|
+
return key in self.metadata
|
|
126
|
+
|
|
127
|
+
def items(self):
|
|
128
|
+
for key, value in self.metadata.items():
|
|
129
|
+
yield key, self.data[value['offset']:value['offset'] + value['length']]
|
|
130
|
+
|
|
131
|
+
def keys(self):
|
|
132
|
+
return self.metadata.keys()
|
|
133
|
+
|
|
134
|
+
def compact(self):
|
|
135
|
+
"""Compact the data array"""
|
|
136
|
+
new_data = np.ndarray(sum([value['length'] for value in self.metadata.values()]), dtype=self.dtype)
|
|
137
|
+
offset = 0
|
|
138
|
+
for value in self.metadata.values():
|
|
139
|
+
new_data[offset:offset + value['length']] = self.data[value['offset']:value['offset'] + value['length']]
|
|
140
|
+
value['offset'] = offset
|
|
141
|
+
offset += value['length']
|
|
142
|
+
self.data = new_data
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class MemoryMappedArrays:
|
|
146
|
+
def __init__(self,
|
|
147
|
+
data_dir: str,
|
|
148
|
+
arrays: Optional[ArrayDict] = None):
|
|
149
|
+
root_filename = create_filename(data_dir)
|
|
150
|
+
self.filename = root_filename + '.dat'
|
|
151
|
+
self.fp = None
|
|
152
|
+
if arrays is not None:
|
|
153
|
+
self.arrays = arrays
|
|
154
|
+
self._initialize_file()
|
|
155
|
+
else:
|
|
156
|
+
self._load_arrays()
|
|
157
|
+
|
|
158
|
+
def _initialize_file(self):
|
|
159
|
+
"""Memmap underlying array in ArrayDict."""
|
|
160
|
+
with open(self.filename, 'wb') as f:
|
|
161
|
+
self.arrays.data.tofile(f)
|
|
162
|
+
|
|
163
|
+
def _load_arrays(self):
|
|
164
|
+
"""Memmap underlying array in ArrayDict from self.filename."""
|
|
165
|
+
self.arrays.data = np.memmap(self.filename, dtype=self.arrays.dtype, mode='r+')
|
|
166
|
+
|
|
167
|
+
def __getitem__(self, key):
|
|
168
|
+
# Use metadatat to get the offset and length of the array
|
|
169
|
+
return self.arrays[key]
|
|
170
|
+
|
|
171
|
+
def __setitem__(self, key, value):
|
|
172
|
+
"""Append to file and update metadata. Not fast."""
|
|
173
|
+
self.arrays[key] = value
|
|
174
|
+
self._save_metadata()
|
|
175
|
+
self.fp = np.memmap(self.filename, dtype=self.dtype, mode='r+', shape=(self.fp.size + value.size,))
|
|
176
|
+
|
|
177
|
+
def __delitem__(self, key):
|
|
178
|
+
length = self.arrays[key].size
|
|
179
|
+
del self.arrays[key]
|
|
180
|
+
self.fp = np.memmap(self.filename, dtype=self.dtype, mode='r+', shape=(self.fp.size - length,))
|
|
181
|
+
|
|
182
|
+
def __len__(self):
|
|
183
|
+
return len(self.arrays)
|
|
184
|
+
|
|
185
|
+
def __iter__(self):
|
|
186
|
+
yield from self.arrays
|
|
187
|
+
|
|
188
|
+
def __contains__(self, key):
|
|
189
|
+
return key in self.arrays
|
|
190
|
+
|
|
191
|
+
def items(self):
|
|
192
|
+
yield from self.arrays.items()
|
|
193
|
+
|
|
194
|
+
def keys(self):
|
|
195
|
+
return self.arrays.keys()
|
|
196
|
+
|
|
197
|
+
def __getstate__(self):
|
|
198
|
+
"""Return arrays without the numpy array."""
|
|
199
|
+
# Create arrays without data
|
|
200
|
+
arrays = ArrayDict()
|
|
201
|
+
arrays.metadata = self.arrays.metadata
|
|
202
|
+
return {'arrays': arrays,
|
|
203
|
+
'filename': self.filename}
|
|
204
|
+
|
|
205
|
+
def __setstate__(self, state):
|
|
206
|
+
self.filename = state['filename']
|
|
207
|
+
self.arrays = state['arrays']
|
|
208
|
+
self._load_arrays()
|