searcharray 0.0.73__cp313-cp313-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. searcharray/__init__.py +2 -0
  2. searcharray/bm25/__init__.py +1 -0
  3. searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
  4. searcharray/bm25/bm25.pyi +11 -0
  5. searcharray/bm25/bm25.pyx +41 -0
  6. searcharray/indexing.py +342 -0
  7. searcharray/phrase/__init__.py +0 -0
  8. searcharray/phrase/bigram_freqs.py +307 -0
  9. searcharray/phrase/memmap_arrays.py +208 -0
  10. searcharray/phrase/middle_out.py +553 -0
  11. searcharray/phrase/spans.py +187 -0
  12. searcharray/postings.py +708 -0
  13. searcharray/roaringish/__init__.py +7 -0
  14. searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
  15. searcharray/roaringish/intersect.pyi +25 -0
  16. searcharray/roaringish/intersect.pyx +390 -0
  17. searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
  18. searcharray/roaringish/merge.pyi +16 -0
  19. searcharray/roaringish/merge.pyx +232 -0
  20. searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
  21. searcharray/roaringish/popcount.pyi +36 -0
  22. searcharray/roaringish/popcount.pyx +278 -0
  23. searcharray/roaringish/roaringish.py +298 -0
  24. searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
  25. searcharray/roaringish/roaringish_ops.pyx +98 -0
  26. searcharray/roaringish/scatter_assign.h +31 -0
  27. searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
  28. searcharray/roaringish/search.pyi +25 -0
  29. searcharray/roaringish/search.pyx +140 -0
  30. searcharray/roaringish/snp_ops.pxd +13 -0
  31. searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
  32. searcharray/roaringish/spans.pyx +330 -0
  33. searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
  34. searcharray/roaringish/unique.pyi +6 -0
  35. searcharray/roaringish/unique.pyx +145 -0
  36. searcharray/similarity.py +92 -0
  37. searcharray/solr.py +355 -0
  38. searcharray/term_dict.py +59 -0
  39. searcharray/utils/__init__.py +0 -0
  40. searcharray/utils/bitcount.py +34 -0
  41. searcharray/utils/mat_set.py +170 -0
  42. searcharray/utils/row_viewable_matrix.py +91 -0
  43. searcharray/utils/sort.py +45 -0
  44. searcharray-0.0.73.dist-info/METADATA +245 -0
  45. searcharray-0.0.73.dist-info/RECORD +48 -0
  46. searcharray-0.0.73.dist-info/WHEEL +5 -0
  47. searcharray-0.0.73.dist-info/entry_points.txt +2 -0
  48. searcharray-0.0.73.dist-info/top_level.txt +1 -0
@@ -0,0 +1,307 @@
1
+ """Given two roaringish encoded arrays, count bigram matches between them."""
2
+ import numpy as np
3
+ from typing import Tuple, Optional
4
+ from searcharray.roaringish import RoaringishEncoder
5
+ import logging
6
+ from enum import Enum
7
+
8
+ from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at, key_sum_over, sort_merge_counts
9
+
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+ # When running in pytest
14
+ import sys # noqa
15
+ handler = logging.StreamHandler(sys.stdout)
16
+ handler.setLevel(logging.ERROR)
17
+ formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
18
+ handler.setFormatter(formatter)
19
+ logger.addHandler(handler)
20
+ logger.setLevel(logging.ERROR)
21
+
22
+
23
+ encoder = RoaringishEncoder()
24
+
25
+ # To not constantly type coerce
26
+ _64 = np.uint64(64)
27
+ _2 = np.uint64(2)
28
+ _1 = np.uint64(1)
29
+ _0 = np.uint64(0)
30
+ _neg1 = np.int64(-1)
31
+ _upper_bit = _1 << (encoder.payload_lsb_bits - _1)
32
+
33
+ MAX_POSN = encoder.max_payload
34
+
35
+
36
+ class Continuation(Enum):
37
+ """Enum for continuation of bigram search."""
38
+ LHS = 0
39
+ RHS = 1
40
+ BOTH = 2
41
+
42
+
43
+ def _any_payload_empty(enc: np.ndarray):
44
+ """Check if the payload has any empty of enc."""
45
+ return np.any((enc & encoder.payload_lsb_mask) == 0)
46
+
47
+
48
+ def _adj_to_phrase_freq(overlap: np.ndarray, adjacents: np.ndarray) -> np.ndarray:
49
+ """Adjust phrase freqs for adjacent matches."""
50
+ # ?? 1 1 1 0 1
51
+ # we need to treat the 2nd consecutive 1 as 'not a match'
52
+ # and also update 'term' to not include it
53
+ # Search for foo foo
54
+ # foo foo -> phrase freq 1
55
+ # foo foo foo -> 2 adjs, phrase freqs = 1
56
+ # foo foo foo foo -> 3 adjs, phrase freqs = 2
57
+ # [foo foo] [foo foo] foo -> 4 adjs, phrase freqs = 2
58
+ # [foo foo] [foo foo] mal [foo foo] -> 4 adjs, phrase freqs = 3
59
+ consecutive_ones = encoder.payload_lsb(overlap & (overlap << _1))
60
+ consecutive_ones = popcount64(consecutive_ones)
61
+ adjacents -= -np.floor_divide(consecutive_ones, -2, dtype=np.int64)
62
+ return adjacents
63
+
64
+
65
+ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
66
+ lhs_doc_ids: np.ndarray,
67
+ cont: Continuation = Continuation.RHS) -> Tuple[Tuple[np.ndarray, np.ndarray],
68
+ Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
69
+ """Count bigram matches when lhs / rhs are same term.
70
+
71
+ Despite being the same term, its possible lhs_int != rhs_int due to lhs corresponding
72
+ to a past match of identical term."""
73
+ # Shift RHS to left by 1
74
+ # 3 2 1 0 3 2 1 0
75
+ # ?? foo foo foo -> foo foo foo ??
76
+ rhs_shift = rhs_int << _1
77
+ # Now we have unshifted and shifted to count adjacents:
78
+ # foo foo foo ?? & ?? foo foo foo
79
+ # -> ?? foo foo ?? or two times term is adjacent
80
+ # Count these bits...
81
+ overlap = lhs_int & rhs_shift
82
+ adj_count = encoder.payload_lsb(overlap)
83
+ adjacents = popcount64(adj_count).view(np.int64)
84
+
85
+ adjusted = _adj_to_phrase_freq(overlap, adjacents).astype(np.uint64)
86
+ lhs_doc_ids, phrase_freqs = key_sum_over(lhs_doc_ids, adjusted)
87
+ # Continue with ?? ?? foo foo
88
+ # term_int without lsbs
89
+ term_int_msbs = lhs_int & ~encoder.payload_lsb_mask
90
+ rhs_cont = None
91
+ lhs_cont = None
92
+ # rhs_cont = term_int_msbs | encoder.payload_lsb(rhs_shift)
93
+ rhs_cont = encoder.payload_lsb(rhs_shift & rhs_int) | term_int_msbs
94
+ lhs_cont = term_int_msbs | encoder.payload_lsb(lhs_int & (lhs_int >> _1))
95
+
96
+ if cont not in [Continuation.RHS, Continuation.BOTH]:
97
+ rhs_cont = None
98
+ if cont not in [Continuation.LHS, Continuation.BOTH]:
99
+ lhs_cont = None
100
+
101
+ return (lhs_doc_ids, phrase_freqs), (lhs_cont, rhs_cont)
102
+
103
+
104
+ def _inner_bigram_freqs(lhs: np.ndarray,
105
+ rhs: np.ndarray,
106
+ lhs_int: np.ndarray,
107
+ rhs_int: np.ndarray,
108
+ cont: Continuation = Continuation.RHS) -> Tuple[Tuple[np.ndarray, np.ndarray],
109
+ Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
110
+ """Count bigram matches between two encoded arrays, within a 64 bit word with same MSBs.
111
+
112
+ Parameters:
113
+ -----------
114
+ lhs: roaringish encoded posn array of left term
115
+ rhs: roaringish encoded posn array of right term
116
+ lhs_int: np.ndarray, intersected lhs
117
+ rhs_int: np.ndarray, intersected rhs
118
+ cont_rhs: bool, whether to continue matching on the rhs or lhs
119
+
120
+ Returns:
121
+ --------
122
+ doc_ids, overlap: the doc ids and overlap bits of the bigram matches (to be collected)
123
+ cont_next: tuple the next (lhs or rhs) array to continue matching
124
+
125
+ """
126
+ # lhs_int, rhs_int = encoder.intersect(lhs, rhs)
127
+ lhs_doc_ids = encoder.keys(lhs_int)
128
+ if len(lhs_int) != len(rhs_int):
129
+ raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
130
+ # No intersections, no inner bigram matches
131
+ if len(lhs_int) == 0 and cont == Continuation.RHS:
132
+ return (np.array([], dtype=np.uint64), np.array([])), (None, rhs_int)
133
+ if len(lhs_int) == 0 and cont == Continuation.LHS:
134
+ return (np.array([], dtype=np.uint64), np.array([])), (lhs_int, None)
135
+ if len(lhs_int) == 0 and cont == Continuation.BOTH:
136
+ return (np.array([], dtype=np.uint64), np.array([])), (lhs_int, rhs_int)
137
+
138
+ # For perf we don't check all values, but if overlapping posns allowed in future, maybe we should
139
+ same_term = (len(lhs_int) == len(rhs_int) and np.all(lhs_int == rhs_int))
140
+ if same_term:
141
+ return _inner_bigram_same_term(lhs_int, rhs_int, lhs_doc_ids, cont)
142
+
143
+ rhs_next = None
144
+ lhs_next = None
145
+ overlap_bits = (lhs_int & encoder.payload_lsb_mask) & ((rhs_int & encoder.payload_lsb_mask) >> _1)
146
+
147
+ if cont in [Continuation.RHS, Continuation.BOTH]:
148
+ rhs_next = (overlap_bits << _1) & encoder.payload_lsb_mask
149
+ rhs_next |= (rhs_int & (encoder.key_mask | encoder.payload_msb_mask))
150
+ if cont in [Continuation.LHS, Continuation.BOTH]:
151
+ lhs_next = overlap_bits.copy()
152
+ lhs_next |= (lhs_int & (encoder.key_mask | encoder.payload_msb_mask))
153
+
154
+ lhs_doc_ids, counts = popcount_reduce_at(lhs_doc_ids, overlap_bits)
155
+ return (lhs_doc_ids, counts), (lhs_next, rhs_next)
156
+
157
+
158
+ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
159
+ lhs_adj: np.ndarray, rhs_adj: np.ndarray,
160
+ cont: Continuation = Continuation.RHS) -> Tuple[Tuple[np.ndarray, np.ndarray],
161
+ Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
162
+ """Count bigram matches between two encoded arrays where they occur in adjacent 64 bit words.
163
+
164
+ Returns:
165
+ --------
166
+ ids 1 id per adjacent for that doc
167
+ Tuple[lhs_next, rhs_next]: the next lhs/rhs array to continue matching
168
+
169
+ """
170
+ lhs_doc_ids = encoder.keys(lhs_adj)
171
+ # lhs lsb set and rhs lsb's most significant bit set
172
+ matches = ((lhs_adj & _upper_bit) != 0) & ((rhs_adj & _1) != 0)
173
+ unique, counts = np.unique(lhs_doc_ids[matches], return_counts=True)
174
+ # phrase_freqs[unique] += counts
175
+ # Set lsb to 0 where no match, lsb to 1 where match
176
+ rhs_next = None if cont == Continuation.LHS else np.asarray([], dtype=np.uint64)
177
+ lhs_next = None if cont == Continuation.RHS else np.asarray([], dtype=np.uint64)
178
+ if np.any(matches):
179
+ if cont in [Continuation.RHS, Continuation.BOTH]:
180
+ rhs_next = rhs_adj[matches]
181
+ assert rhs_next is not None
182
+ rhs_next = encoder.header(rhs_next) | _1
183
+ if cont in [Continuation.LHS, Continuation.BOTH]:
184
+ lhs_next = lhs_adj[matches]
185
+ assert lhs_next is not None
186
+ lhs_next = encoder.header(lhs_next) | _upper_bit
187
+ rhs_next = None
188
+ return (unique, counts), (lhs_next, rhs_next)
189
+
190
+
191
+ def _set_adjbit_at_header(next_inner: np.ndarray, next_adj: np.ndarray,
192
+ cont: Continuation = Continuation.RHS) -> np.ndarray:
193
+ """Merge two encoded arrays on their headers."""
194
+ if len(next_inner) == 0:
195
+ return next_adj
196
+ if len(next_adj) == 0:
197
+ return next_inner
198
+
199
+ same_header_inner, same_header_adj = intersect(next_inner, next_adj,
200
+ mask=encoder.header_mask)
201
+ # Set _1 on intersection
202
+ ignore_mask = np.ones(len(next_adj), dtype=bool)
203
+ ignore_mask[same_header_adj] = False
204
+ if len(same_header_inner) > 0 and cont == Continuation.RHS:
205
+ next_inner[same_header_inner] |= _1
206
+ next_adj = next_adj[ignore_mask]
207
+ if len(same_header_adj) > 0 and cont == Continuation.LHS:
208
+ next_inner[same_header_inner] |= _upper_bit
209
+ next_adj = next_adj[ignore_mask]
210
+ return merge(next_inner, next_adj)
211
+
212
+
213
+ def bigram_freqs(lhs: np.ndarray,
214
+ rhs: np.ndarray,
215
+ cont: Continuation = Continuation.RHS) -> Tuple[Tuple[np.ndarray, np.ndarray],
216
+ Tuple[Optional[np.ndarray], Optional[np.ndarray]]]:
217
+ """Count bigram matches between two roaringish encoded posn arrays.
218
+ Also return connection on right hand of the bigram
219
+
220
+ Parameters:
221
+ -----------
222
+ lhs: roaringish encoded posn array of left term
223
+ rhs: roaringish encoded posn array of right term
224
+ phrase_freqs: np.ndarray, preallocated phrase freqs for output
225
+ cont_rhs: bool, whether to continue matching on the rhs or lhs
226
+
227
+ Returns:
228
+ --------
229
+ phrase_freqs: number of matches per doc (updated input)
230
+ rhs_next: encoded array of end of each bigram
231
+ (use as lhs on subsequent calls to continue the phrase)
232
+
233
+ """
234
+ # Example zooming into just one 64 bit word (this of course is repeated over
235
+ # the entire array, for all docs and posn msbs)
236
+ #
237
+ # ***************************************************************************
238
+ # "Inner" matches:
239
+ # As roaringish arrays put posns on a 64 bit word, we first search for
240
+ # simpler matches within 64 bit words from same docs with same posn header.
241
+ # We do this by intersecting on the header (doc_id and posn_msb), then checking
242
+ # the following pattern in the posn bits:
243
+ #
244
+ # doc_id posns_msb posns_bits
245
+ # lhs_term: 1234 101 0010001010
246
+ # rhs_term: 1234 101 0101000100
247
+ # ^^ ^^ <- inner matches, terms next to each other
248
+ #
249
+ lhs_int, rhs_int, lhs_adj, rhs_adj = encoder.intersect_candidates(lhs, rhs)
250
+
251
+ (ids, counts), (lhs_next_inner, rhs_next_inner)\
252
+ = _inner_bigram_freqs(lhs, rhs,
253
+ lhs_int, rhs_int,
254
+ cont)
255
+ # ***************************************************************************
256
+ # "Adjacent" matches:
257
+ # Of course we also need to check for matches that span two 64 bit words,
258
+ # thats what adjacent matches do. First finding where header meets header + 1,
259
+ # then checking the bit pattern shown below:
260
+ #
261
+ # doc_id posns_msb posns_bits
262
+ # lhs_term: 1234 101 1010000000
263
+ # rhs_term: 1234 111 0001000101
264
+ # ^ ^ <- adjacent matches, terms next to each other
265
+ # but note the posn_msb of lhs is
266
+ # 1 less than the posn_msb of rhs
267
+ # so we detect these differently
268
+ (adj_doc_ids, adj_counts), (lhs_next_adj, rhs_next_adj) \
269
+ = _adjacent_bigram_freqs(lhs, rhs,
270
+ lhs_adj, rhs_adj,
271
+ cont)
272
+
273
+ doc_ids1, counts1 = sort_merge_counts(ids, counts.astype(np.float32),
274
+ adj_doc_ids, adj_counts.astype(np.float32))
275
+
276
+ # ***************************************************************************
277
+ # rhs_next is where the bigram ends on the rhs side, we can use this
278
+ # to continue the phrase matching
279
+ #
280
+ # Combining the examples above:
281
+ # rhs_next_inner: 1234 101 0001000100
282
+ # ^ ^ <- where we had a match (inner)
283
+ #
284
+ # Merged with
285
+ # rhs_next_adj: 1234 111 0000000001
286
+ # ^ <- where we had a match (adjacent)
287
+ #
288
+ # So we merge these... in this case, we have a two-value array
289
+ #
290
+ # rhs_next: 1234,101,0001000100|1234,111,0000000001
291
+ #
292
+ #
293
+ # Now rhs_next can be the LHS of the next call to bigram_freqs
294
+ # to continue the phrase
295
+ #
296
+ rhs_next = None
297
+ lhs_next = None
298
+ if cont in [Continuation.RHS, Continuation.BOTH]:
299
+ assert rhs_next_inner is not None
300
+ assert rhs_next_adj is not None
301
+ rhs_next = _set_adjbit_at_header(rhs_next_inner, rhs_next_adj, Continuation.RHS)
302
+ if cont in [Continuation.LHS, Continuation.BOTH]:
303
+ assert lhs_next_inner is not None
304
+ assert lhs_next_adj is not None
305
+ lhs_next = _set_adjbit_at_header(lhs_next_inner, lhs_next_adj, Continuation.LHS)
306
+
307
+ return (doc_ids1, counts1), (lhs_next, rhs_next)
@@ -0,0 +1,208 @@
1
+ import numpy as np
2
+ import os
3
+ import reprlib
4
+ from typing import Optional, List, Dict
5
+
6
+
7
+ def create_filename(data_dir: str):
8
+ """Create a filename for the memory-mapped array"""
9
+ # Ls path and get the number of files
10
+ files = os.listdir(data_dir)
11
+ num_files = len(files)
12
+ return os.path.join(data_dir, f'{num_files}')
13
+
14
+
15
+ class ArrayDict:
16
+ """An array of ints to np.array as contiguous memory"""
17
+
18
+ def __init__(self):
19
+ self.dtype = np.uint64
20
+ self.data = np.ndarray(0, dtype=self.dtype)
21
+ self.metadata = {}
22
+
23
+ def __repr__(self):
24
+ repr_obj = reprlib.Repr()
25
+ return f'ArrayDict({repr_obj.repr(self.metadata)})'
26
+
27
+ @staticmethod
28
+ def from_arrays(arrays: List[np.ndarray]):
29
+ arr = ArrayDict()
30
+ metadata = {}
31
+ total_len = 0
32
+ for i, array in enumerate(arrays):
33
+ metadata[i] = {'offset': total_len, 'length': array.size}
34
+ total_len += array.size
35
+ data = np.concatenate(arrays)
36
+ arr.data = data
37
+ arr.metadata = metadata
38
+ return arr
39
+
40
+ @staticmethod
41
+ def from_array_with_boundaries(data: np.ndarray,
42
+ ids: np.ndarray,
43
+ boundaries: np.ndarray):
44
+ metadata = {}
45
+ for idx, (beg, end) in enumerate(zip(boundaries[:-1], boundaries[1:])):
46
+ offset = int(beg)
47
+ length = int(end - beg)
48
+ actual_id = ids[idx]
49
+ metadata[int(actual_id)] = {'offset': offset, 'length': length}
50
+ arr = ArrayDict()
51
+ arr.data = data
52
+ arr.metadata = metadata
53
+ return arr
54
+
55
+ @staticmethod
56
+ def concat(lhs: 'ArrayDict', rhs: 'ArrayDict', sort=True):
57
+ arr = ArrayDict()
58
+ metadata: Dict[int, Dict[str, int]] = {}
59
+ lst_of_arrays: List[np.ndarray] = []
60
+
61
+ curr_offset = 0
62
+ last_offset = 0
63
+
64
+ fetched_keys = set()
65
+ for key, value in lhs.items():
66
+ fetched_keys.add(key)
67
+ all_for_key = [value]
68
+ curr_offset += value.size
69
+ if key in rhs.metadata:
70
+ rhs_value = rhs[key]
71
+ all_for_key.append(rhs_value)
72
+ curr_offset += rhs_value.size
73
+ metadata[key] = {'offset': last_offset, 'length': curr_offset - last_offset}
74
+ last_offset = curr_offset
75
+ lst_of_arrays.append(np.concatenate(all_for_key))
76
+
77
+ for key, value in rhs.items():
78
+ if key not in fetched_keys:
79
+ curr_offset += value.size
80
+ lst_of_arrays.append(value)
81
+ metadata[key] = {'offset': last_offset, 'length': curr_offset - last_offset}
82
+ last_offset = curr_offset
83
+
84
+ arr.metadata = metadata
85
+ for curr_arr in lst_of_arrays:
86
+ if sort:
87
+ curr_arr.sort()
88
+
89
+ if len(lst_of_arrays) > 0:
90
+ arr.data = np.concatenate(lst_of_arrays)
91
+ return arr
92
+
93
+ def __getitem__(self, key):
94
+ key = int(key)
95
+ if key in self.metadata:
96
+ offset = self.metadata[key]['offset']
97
+ length = self.metadata[key]['length']
98
+ return self.data[offset:offset + length]
99
+ else:
100
+ raise KeyError(f'Key {key} not found.')
101
+
102
+ def __setitem__(self, key, value):
103
+ key = int(key)
104
+ if value.dtype != self.dtype:
105
+ raise ValueError(f'Value must be of type {self.dtype}')
106
+ offset = self.data.size
107
+ length = value.size
108
+ self.metadata[key] = {'offset': offset, 'length': length}
109
+ self.data = np.append(self.data, value)
110
+
111
+ def __delitem__(self, key):
112
+ key = int(key)
113
+ if key in self.metadata:
114
+ del self.metadata[key]
115
+ else:
116
+ raise KeyError(f'Key {key} not found.')
117
+
118
+ def __len__(self):
119
+ return len(self.metadata)
120
+
121
+ def __iter__(self):
122
+ yield from self.metadata
123
+
124
+ def __contains__(self, key):
125
+ return key in self.metadata
126
+
127
+ def items(self):
128
+ for key, value in self.metadata.items():
129
+ yield key, self.data[value['offset']:value['offset'] + value['length']]
130
+
131
+ def keys(self):
132
+ return self.metadata.keys()
133
+
134
+ def compact(self):
135
+ """Compact the data array"""
136
+ new_data = np.ndarray(sum([value['length'] for value in self.metadata.values()]), dtype=self.dtype)
137
+ offset = 0
138
+ for value in self.metadata.values():
139
+ new_data[offset:offset + value['length']] = self.data[value['offset']:value['offset'] + value['length']]
140
+ value['offset'] = offset
141
+ offset += value['length']
142
+ self.data = new_data
143
+
144
+
145
+ class MemoryMappedArrays:
146
+ def __init__(self,
147
+ data_dir: str,
148
+ arrays: Optional[ArrayDict] = None):
149
+ root_filename = create_filename(data_dir)
150
+ self.filename = root_filename + '.dat'
151
+ self.fp = None
152
+ if arrays is not None:
153
+ self.arrays = arrays
154
+ self._initialize_file()
155
+ else:
156
+ self._load_arrays()
157
+
158
+ def _initialize_file(self):
159
+ """Memmap underlying array in ArrayDict."""
160
+ with open(self.filename, 'wb') as f:
161
+ self.arrays.data.tofile(f)
162
+
163
+ def _load_arrays(self):
164
+ """Memmap underlying array in ArrayDict from self.filename."""
165
+ self.arrays.data = np.memmap(self.filename, dtype=self.arrays.dtype, mode='r+')
166
+
167
+ def __getitem__(self, key):
168
+ # Use metadatat to get the offset and length of the array
169
+ return self.arrays[key]
170
+
171
+ def __setitem__(self, key, value):
172
+ """Append to file and update metadata. Not fast."""
173
+ self.arrays[key] = value
174
+ self._save_metadata()
175
+ self.fp = np.memmap(self.filename, dtype=self.dtype, mode='r+', shape=(self.fp.size + value.size,))
176
+
177
+ def __delitem__(self, key):
178
+ length = self.arrays[key].size
179
+ del self.arrays[key]
180
+ self.fp = np.memmap(self.filename, dtype=self.dtype, mode='r+', shape=(self.fp.size - length,))
181
+
182
+ def __len__(self):
183
+ return len(self.arrays)
184
+
185
+ def __iter__(self):
186
+ yield from self.arrays
187
+
188
+ def __contains__(self, key):
189
+ return key in self.arrays
190
+
191
+ def items(self):
192
+ yield from self.arrays.items()
193
+
194
+ def keys(self):
195
+ return self.arrays.keys()
196
+
197
+ def __getstate__(self):
198
+ """Return arrays without the numpy array."""
199
+ # Create arrays without data
200
+ arrays = ArrayDict()
201
+ arrays.metadata = self.arrays.metadata
202
+ return {'arrays': arrays,
203
+ 'filename': self.filename}
204
+
205
+ def __setstate__(self, state):
206
+ self.filename = state['filename']
207
+ self.arrays = state['arrays']
208
+ self._load_arrays()