searcharray 0.0.73__cp313-cp313-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. searcharray/__init__.py +2 -0
  2. searcharray/bm25/__init__.py +1 -0
  3. searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
  4. searcharray/bm25/bm25.pyi +11 -0
  5. searcharray/bm25/bm25.pyx +41 -0
  6. searcharray/indexing.py +342 -0
  7. searcharray/phrase/__init__.py +0 -0
  8. searcharray/phrase/bigram_freqs.py +307 -0
  9. searcharray/phrase/memmap_arrays.py +208 -0
  10. searcharray/phrase/middle_out.py +553 -0
  11. searcharray/phrase/spans.py +187 -0
  12. searcharray/postings.py +708 -0
  13. searcharray/roaringish/__init__.py +7 -0
  14. searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
  15. searcharray/roaringish/intersect.pyi +25 -0
  16. searcharray/roaringish/intersect.pyx +390 -0
  17. searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
  18. searcharray/roaringish/merge.pyi +16 -0
  19. searcharray/roaringish/merge.pyx +232 -0
  20. searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
  21. searcharray/roaringish/popcount.pyi +36 -0
  22. searcharray/roaringish/popcount.pyx +278 -0
  23. searcharray/roaringish/roaringish.py +298 -0
  24. searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
  25. searcharray/roaringish/roaringish_ops.pyx +98 -0
  26. searcharray/roaringish/scatter_assign.h +31 -0
  27. searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
  28. searcharray/roaringish/search.pyi +25 -0
  29. searcharray/roaringish/search.pyx +140 -0
  30. searcharray/roaringish/snp_ops.pxd +13 -0
  31. searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
  32. searcharray/roaringish/spans.pyx +330 -0
  33. searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
  34. searcharray/roaringish/unique.pyi +6 -0
  35. searcharray/roaringish/unique.pyx +145 -0
  36. searcharray/similarity.py +92 -0
  37. searcharray/solr.py +355 -0
  38. searcharray/term_dict.py +59 -0
  39. searcharray/utils/__init__.py +0 -0
  40. searcharray/utils/bitcount.py +34 -0
  41. searcharray/utils/mat_set.py +170 -0
  42. searcharray/utils/row_viewable_matrix.py +91 -0
  43. searcharray/utils/sort.py +45 -0
  44. searcharray-0.0.73.dist-info/METADATA +245 -0
  45. searcharray-0.0.73.dist-info/RECORD +48 -0
  46. searcharray-0.0.73.dist-info/WHEEL +5 -0
  47. searcharray-0.0.73.dist-info/entry_points.txt +2 -0
  48. searcharray-0.0.73.dist-info/top_level.txt +1 -0
@@ -0,0 +1,36 @@
1
+ import numpy as np
2
+ from numpy.typing import NDArray
3
+ from typing import Tuple
4
+
5
+
6
+ def popcount64(arr: NDArray[np.uint64]) -> NDArray[np.uint64]:
7
+ ...
8
+
9
+
10
+ def msb_mask64(arr: NDArray[np.uint64]) -> NDArray[np.uint64]:
11
+ ...
12
+
13
+
14
+ def ctz64(arr: NDArray[np.uint64]) -> NDArray[np.uint64]:
15
+ ...
16
+
17
+
18
+ def clz64(arr: NDArray[np.uint64]) -> NDArray[np.uint64]:
19
+ ...
20
+
21
+
22
+ def popcount_reduce_at(ids: NDArray[np.uint64],
23
+ payload: NDArray[np.uint64]) -> Tuple[NDArray[np.uint64], NDArray[np.float32]]:
24
+ ...
25
+
26
+
27
+ def popcount64_reduce(arr: NDArray[np.uint64],
28
+ key_shift: np.uint64,
29
+ value_mask: np.uint64) -> Tuple[NDArray[np.uint64],
30
+ NDArray[np.uint64]]:
31
+ ...
32
+
33
+
34
+ def key_sum_over(ids: NDArray[np.uint64],
35
+ counts: NDArray[np.uint64]) -> Tuple[NDArray[np.uint64], NDArray[np.float32]]:
36
+ ...
@@ -0,0 +1,278 @@
1
+ # These are modified versions of sortednp:
2
+ # https://gitlab.sauerburger.com/frank/sortednp
3
+ # cython: boundscheck=False
4
+ # cython: wraparound=False
5
+ # cython: initializedcheck=False
6
+ # cython: cdivision=True
7
+ # cython: nonecheck=False
8
+ # cython: language_level=3
9
+ cimport numpy as np
10
+ import numpy as np
11
+
12
+
13
+ # cimport snp_ops
14
+ # from snp_ops cimport _galloping_search, DTYPE_t, ALL_BITS
15
+ cimport searcharray.roaringish.snp_ops
16
+ from searcharray.roaringish.snp_ops cimport DTYPE_t
17
+
18
+ cdef extern from *:
19
+ """
20
+ #if defined(_MSC_VER)
21
+ #include <intrin.h>
22
+ #pragma intrinsic(__popcnt64)
23
+ #pragma intrinsic(_BitScanForward64)
24
+ #pragma intrinsic(_BitScanReverse64)
25
+
26
+ static int popcountll(unsigned long long x) {
27
+ return __popcnt64(x);
28
+ }
29
+
30
+ static int ctzll(unsigned long long x) {
31
+ unsigned long index;
32
+ if (x == 0) return 64;
33
+ _BitScanForward64(&index, x);
34
+ return index;
35
+ }
36
+
37
+ static int clzll(unsigned long long x) {
38
+ unsigned long index;
39
+ if (x == 0) return 64;
40
+ _BitScanReverse64(&index, x);
41
+ return 63 - index;
42
+ }
43
+
44
+ #else
45
+ #include <stddef.h>
46
+ static int popcountll(unsigned long long x) {
47
+ return __builtin_popcountll(x);
48
+ }
49
+
50
+ static int ctzll(unsigned long long x) {
51
+ return __builtin_ctzll(x);
52
+ }
53
+
54
+ static int clzll(unsigned long long x) {
55
+ return __builtin_clzll(x);
56
+ }
57
+ #endif
58
+ """
59
+
60
+ cdef extern from *:
61
+ int popcountll(unsigned long long x)
62
+ int ctzll(unsigned long long x)
63
+ int clzll(unsigned long long x)
64
+
65
+
66
+ # Include mach performance timer
67
+ # cdef extern from "mach/mach_time.h":
68
+ # uint64_t mach_absolute_time()
69
+
70
+
71
+ cdef popcount64_arr(DTYPE_t[:] arr):
72
+ cdef np.uint64_t[:] result = np.empty(arr.shape[0], dtype=np.uint64)
73
+ # cdef int i = 0
74
+ cdef DTYPE_t* result_ptr = &result[0]
75
+ cdef DTYPE_t* arr_ptr = &arr[0]
76
+
77
+ for _ in range(arr.shape[0]):
78
+ result_ptr[0] = popcountll(arr_ptr[0])
79
+ result_ptr += 1
80
+ arr_ptr += 1
81
+ return result
82
+
83
+
84
+ cdef ctz_arr(DTYPE_t[:] arr):
85
+ cdef np.uint64_t[:] result = np.empty(arr.shape[0], dtype=np.uint64)
86
+ # cdef int i = 0
87
+ cdef DTYPE_t* result_ptr = &result[0]
88
+ cdef DTYPE_t* arr_ptr = &arr[0]
89
+
90
+ for _ in range(arr.shape[0]):
91
+ result_ptr[0] = ctzll(arr_ptr[0])
92
+ result_ptr += 1
93
+ arr_ptr += 1
94
+ return result
95
+
96
+
97
+ cdef clz_arr(DTYPE_t[:] arr):
98
+ cdef np.uint64_t[:] result = np.empty(arr.shape[0], dtype=np.uint64)
99
+ # cdef int i = 0
100
+ cdef DTYPE_t* result_ptr = &result[0]
101
+ cdef DTYPE_t* arr_ptr = &arr[0]
102
+
103
+ for _ in range(arr.shape[0]):
104
+ result_ptr[0] = clzll(arr_ptr[0])
105
+ result_ptr += 1
106
+ arr_ptr += 1
107
+ return result
108
+
109
+
110
+ cdef popcount64_arr_naive(DTYPE_t[:] arr):
111
+ cdef np.uint64_t[:] result = np.empty(arr.shape[0], dtype=np.uint64)
112
+ cdef int i = 0
113
+
114
+ for i in range(arr.shape[0]):
115
+ result[i] = popcountll(arr[i])
116
+ return result
117
+
118
+
119
+ def popcount64(np.ndarray[DTYPE_t, ndim=1] arr):
120
+ """Count the number of set bits in a 64-bit integer."""
121
+ return np.array(popcount64_arr(arr))
122
+
123
+
124
+ cdef DTYPE_t _popcount_reduce_at(DTYPE_t[:] ids, DTYPE_t[:] payload,
125
+ DTYPE_t[:] merged_ids, float[:] merged_counts):
126
+ cdef DTYPE_t last_id = ids[0]
127
+ cdef DTYPE_t popcount_sum = 0
128
+ cdef DTYPE_t* merged_ids_ptr = &merged_ids[0]
129
+ cdef DTYPE_t* payload_ptr = &payload[0]
130
+ cdef DTYPE_t* ids_ptr = &ids[0]
131
+ cdef float* merged_counts_ptr = &merged_counts[0]
132
+
133
+ # We already have 0, now add new values
134
+ while ids_ptr < &ids[0] + ids.shape[0]:
135
+ if ids_ptr[0] != last_id:
136
+ merged_ids_ptr[0] = last_id
137
+ merged_counts_ptr[0] = popcount_sum
138
+ popcount_sum = 0
139
+ merged_ids_ptr += 1
140
+ merged_counts_ptr += 1
141
+ popcount_sum += popcountll(payload_ptr[0])
142
+ last_id = ids_ptr[0]
143
+ payload_ptr += 1
144
+ ids_ptr += 1
145
+ # Save final value
146
+ merged_ids_ptr[0] = last_id
147
+ merged_counts_ptr[0] = popcount_sum
148
+ return merged_ids_ptr - &merged_ids[0] + 1
149
+
150
+
151
+ def popcount_reduce_at(np.ndarray[DTYPE_t, ndim=1] ids,
152
+ np.ndarray[DTYPE_t, ndim=1] payload):
153
+ """Write the sum of popcount of the payload at the indices in ids to the output array.
154
+
155
+ Add one to each plus_one ids
156
+
157
+ """
158
+ if len(ids) != len(payload):
159
+ raise ValueError("ids and payload must have the same length")
160
+ if len(ids) == 0:
161
+ return np.array([]), np.array([])
162
+ merged_ids = np.empty(ids.shape[0], dtype=np.uint64)
163
+ merged_counts = np.empty(ids.shape[0], dtype=np.float32)
164
+ merged_len = _popcount_reduce_at(ids, payload, merged_ids, merged_counts)
165
+ return np.array(merged_ids[:merged_len]), np.array(merged_counts[:merged_len])
166
+
167
+
168
+ cdef _key_sum_over(DTYPE_t[:] ids, DTYPE_t[:] count,
169
+ DTYPE_t[:] merged_ids, float[:] merged_counts):
170
+ cdef DTYPE_t* ids_ptr = &ids[0]
171
+ cdef DTYPE_t* count_ptr = &count[0]
172
+ cdef DTYPE_t* merged_ids_ptr = &merged_ids[0]
173
+ cdef float* merged_counts_ptr = &merged_counts[0]
174
+ cdef DTYPE_t last_id = ids[0]
175
+ cdef DTYPE_t payload_sum = 0
176
+
177
+ while ids_ptr < &ids[0] + ids.shape[0]:
178
+ if ids_ptr[0] != last_id:
179
+ merged_ids_ptr[0] = last_id
180
+ merged_counts_ptr[0] = payload_sum
181
+ payload_sum = 0
182
+ merged_ids_ptr += 1
183
+ merged_counts_ptr += 1
184
+ payload_sum += count_ptr[0]
185
+ last_id = ids_ptr[0]
186
+ ids_ptr += 1
187
+ count_ptr += 1
188
+ # Save final value
189
+ merged_ids_ptr[0] = last_id
190
+ merged_counts_ptr[0] = payload_sum
191
+ return merged_ids_ptr - &merged_ids[0] + 1
192
+
193
+
194
+ def key_sum_over(np.ndarray[DTYPE_t, ndim=1] ids,
195
+ np.ndarray[DTYPE_t, ndim=1] count):
196
+ """Write the last value of the payload at the indices in ids to the output array."""
197
+ if len(ids) != len(count):
198
+ raise ValueError("ids and count must have the same length")
199
+ if len(ids) == 0:
200
+ return np.array([]), np.array([])
201
+ merged_ids = np.empty(ids.shape[0], dtype=np.uint64)
202
+ merged_counts = np.empty(ids.shape[0], dtype=np.float32)
203
+ merged_len = _key_sum_over(ids, count, merged_ids, merged_counts)
204
+ return np.array(merged_ids[:merged_len]), np.array(merged_counts[:merged_len])
205
+
206
+
207
+ # Popcount reduce key-value pair
208
+ # for words 0xKKKKKKKK...KKKKVVVV...VVVV
209
+ # Returning two parallel arrays:
210
+ # - keys - the keys
211
+ # - values - the popcount of the values with those keys
212
+ cdef _popcount64_reduce(DTYPE_t[:] arr,
213
+ DTYPE_t key_shift,
214
+ DTYPE_t value_mask):
215
+ cdef float[:] popcounts = np.zeros(arr.shape[0], dtype=np.float32)
216
+ cdef DTYPE_t[:] keys = np.empty(arr.shape[0], dtype=np.uint64)
217
+ cdef float* popcounts_ptr = &popcounts[0]
218
+ cdef DTYPE_t* keys_ptr = &keys[0]
219
+ cdef DTYPE_t* arr_ptr = &arr[0]
220
+
221
+ cdef DTYPE_t last_key = arr_ptr[0] >> key_shift
222
+ cdef DTYPE_t key = last_key
223
+ keys_ptr[0] = last_key
224
+
225
+ for _ in range(arr.shape[0]):
226
+ key = arr_ptr[0] >> key_shift
227
+ if key == last_key:
228
+ popcounts_ptr[0] += popcountll(arr_ptr[0] & value_mask)
229
+ else:
230
+ last_key = key
231
+ popcounts_ptr += 1
232
+ keys_ptr += 1
233
+ # Init next key
234
+ keys_ptr[0] = last_key
235
+ popcounts_ptr[0] = popcountll(arr_ptr[0] & value_mask)
236
+ arr_ptr += 1
237
+ return keys, popcounts, (keys_ptr - &keys[0] + 1)
238
+
239
+
240
+ # Branchless version using lookup table instead of if...
241
+ # Popcount reduce key-value pair
242
+ # for words 0xKKKKKKKK...KKKKVVVV...VVVV
243
+ # Returning two parallel arrays:
244
+ # - keys - the keys
245
+ # - values - the popcount of the values with those keys
246
+ cdef _popcount64_reduce_nobranch(DTYPE_t[:] arr,
247
+ DTYPE_t key_shift,
248
+ DTYPE_t value_mask):
249
+ cdef float[:] popcounts = np.zeros(arr.shape[0], dtype=np.float32)
250
+ cdef DTYPE_t[:] keys = np.empty(arr.shape[0], dtype=np.uint64)
251
+ cdef float* popcounts_ptr = &popcounts[0]
252
+ cdef DTYPE_t* keys_ptr = &keys[0]
253
+ cdef DTYPE_t* arr_ptr = &arr[0]
254
+
255
+ cdef DTYPE_t last_key = arr_ptr[0] >> key_shift
256
+ cdef DTYPE_t key = last_key
257
+
258
+ keys_ptr[0] = last_key
259
+
260
+ for _ in range(arr.shape[0]):
261
+ key = arr_ptr[0] >> key_shift
262
+ popcounts_ptr += (key != last_key)
263
+ keys_ptr += (key != last_key)
264
+ popcounts_ptr[0] += popcountll(arr_ptr[0] & value_mask)
265
+ keys_ptr[0] = key
266
+ last_key = key
267
+ arr_ptr += 1
268
+ return keys, popcounts, (keys_ptr - &keys[0] + 1)
269
+
270
+
271
+ def popcount64_reduce(arr,
272
+ key_shift,
273
+ value_mask):
274
+ cdef DTYPE_t[:] arr_view = arr
275
+ if len(arr_view) == 0:
276
+ return np.array([]), np.array([])
277
+ keys, popcounts, results_idx = _popcount64_reduce(arr_view, key_shift, value_mask)
278
+ return np.array(keys[:results_idx]), np.array(popcounts[:results_idx])
@@ -0,0 +1,298 @@
1
+ """Roaring-ish bit array for storing sorted integers in numpy array.
2
+
3
+ See - https://softwaredoug.com/blog/2024/01/21/search-array-phrase-algorithm
4
+ """
5
+ import numpy as np
6
+ import logging
7
+ import numbers
8
+ from typing import Optional, Tuple, List, Union
9
+
10
+ from searcharray.roaringish.search import galloping_search
11
+ from searcharray.roaringish.merge import merge
12
+ from searcharray.roaringish.unique import unique
13
+ from searcharray.roaringish.intersect import intersect, adjacent, intersect_with_adjacents
14
+ from searcharray.roaringish.popcount import popcount64_reduce
15
+ from searcharray.roaringish.roaringish_ops import payload_slice
16
+
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+ # When running in pytest
21
+ import sys # noqa
22
+ handler = logging.StreamHandler(sys.stdout)
23
+ handler.setLevel(logging.ERROR)
24
+ formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
25
+ handler.setFormatter(formatter)
26
+ logger.addHandler(handler)
27
+ logger.setLevel(logging.ERROR)
28
+
29
+
30
+ DEFAULT_KEY_MASK = np.uint64(0xFFFFFFF000000000)
31
+ DEFAULT_KEY_BITS = np.uint64(28)
32
+ DEFAULT_PAYLOAD_MSB_MASK = np.uint64(0x0000000FFFFC0000)
33
+ DEFAULT_PAYLOAD_MSB_BITS = np.uint64(18)
34
+ DEFAULT_PAYLOAD_LSB_MASK = np.uint64(0x000000000003FFFF)
35
+ DEFAULT_PAYLOAD_LSB_BITS = np.uint64(18)
36
+
37
+ # To not constantly type coerce
38
+ _64 = np.uint64(64)
39
+ _2 = np.uint64(2)
40
+ _1 = np.uint64(1)
41
+ _0 = np.uint64(0)
42
+ _neg1 = np.int64(-1)
43
+
44
+
45
+ def n_msb_mask(n: np.uint64) -> np.uint64:
46
+ """Return the n most significant bits of num."""
47
+ return np.uint64(~(np.uint64(_1 << (_64 - n))) + _1)
48
+
49
+
50
+ def sorted_unique(arr: np.ndarray) -> np.ndarray:
51
+ return unique(arr)
52
+
53
+
54
+ class RoaringishEncoder:
55
+ """An encoder for key->integer sets as a numpy array.
56
+
57
+ Each returned array represents a single term, with key as MSBS, ie:
58
+
59
+ | 32 MSBs | 16 LSBs | 16 LSBs |
60
+ key | bits msbs | payload
61
+
62
+ (different number of MSBs / payload bits can be specified)
63
+
64
+ """
65
+
66
+ def __init__(self, key_bits: np.uint64 = DEFAULT_KEY_BITS):
67
+ payload_bits = _64 - key_bits
68
+ self.payload_msb_bits = payload_bits // _2
69
+ self.payload_lsb_bits = np.uint64(payload_bits - self.payload_msb_bits)
70
+ self.key_bits = key_bits
71
+ assert self.key_bits.dtype == np.uint64
72
+ # key bits MSB of 64 bits
73
+ self.key_mask = n_msb_mask(key_bits)
74
+ self.header_bits = key_bits + self.payload_msb_bits
75
+ self.payload_msb_mask = n_msb_mask(np.uint64(self.payload_msb_bits + key_bits)) & ~self.key_mask
76
+ assert self.payload_msb_bits.dtype == np.uint64, f"MSB bits dtype was {self.payload_msb_bits.dtype}"
77
+ assert self.payload_msb_mask.dtype == np.uint64, f"MSB mask dtype was {self.payload_msb_mask.dtype}"
78
+ self.payload_lsb_mask = (_1 << self.payload_lsb_bits) - np.uint64(1)
79
+ assert self.payload_lsb_bits.dtype == np.uint64, f"LSB bits dtype was {self.payload_lsb_bits.dtype}"
80
+ assert self.payload_lsb_mask.dtype == np.uint64, f"LSB mask dtype was {self.payload_lsb_mask.dtype}"
81
+ self.header_mask = self.key_mask | self.payload_msb_mask
82
+ if key_bits == DEFAULT_KEY_BITS:
83
+ assert self.key_mask == DEFAULT_KEY_MASK
84
+ assert self.payload_msb_mask == DEFAULT_PAYLOAD_MSB_MASK
85
+ assert self.payload_lsb_mask == DEFAULT_PAYLOAD_LSB_MASK
86
+ self.max_payload = np.uint64(2**self.payload_lsb_bits - 1)
87
+
88
+ def validate_payload(self, payload: np.ndarray):
89
+ """Optional validation of payload."""
90
+ if np.any(payload > self.max_payload):
91
+ raise ValueError(f"Positions must be less than {2**self.payload_lsb_bits}")
92
+
93
+ def encode(self, payload: np.ndarray,
94
+ keys: Optional[np.ndarray] = None,
95
+ boundaries: Optional[np.ndarray] = None) -> Tuple[np.ndarray, Optional[np.ndarray]]:
96
+ """Pack a sorted array of integers into compact bit numpy array.
97
+
98
+ each returned array represents a single term, with key as MSBS, ie:
99
+
100
+ | 32 MSBs | 16 LSBs | 16 LSBs |
101
+ key | bits msbs| payload
102
+
103
+ for later easy intersection of 32+16 msbs, then checking for adjacent
104
+ positions
105
+
106
+ If boundaries are provided, then we consider multiple distinct payloads
107
+ being encoded simultaneously, and we return the boundaries of each
108
+
109
+ """
110
+ cols = np.floor_divide(payload, self.payload_lsb_bits, dtype=np.uint64) # Header of bit to use
111
+ cols <<= self.payload_msb_bits
112
+ if keys is not None:
113
+ cols |= keys.astype(np.uint64) << (_64 - self.key_bits)
114
+ values = payload % self.payload_lsb_bits # Value to encode
115
+
116
+ change_indices_one_doc = np.nonzero(np.diff(cols))[0] + 1
117
+ change_indices_one_doc = change_indices_one_doc.view(np.uint64)
118
+ change_indices_one_doc = np.concatenate([[_0], change_indices_one_doc], dtype=np.uint64)
119
+ if boundaries is not None:
120
+ change_indices = merge(change_indices_one_doc.view(np.uint64),
121
+ boundaries.view(np.uint64),
122
+ drop_duplicates=True)
123
+ new_boundaries = intersect(boundaries,
124
+ change_indices)[-1]
125
+ new_boundaries = np.concatenate([new_boundaries,
126
+ np.asarray([len(change_indices)], dtype=np.uint64)],
127
+ dtype=np.uint64)
128
+ else:
129
+ change_indices = change_indices_one_doc
130
+ new_boundaries = None
131
+
132
+ # 0 as a position, goes in bit 1,
133
+ # 1 as a position, goes in bit 2, etc
134
+ values = _1 << values
135
+ cols |= values
136
+ encoded = cols
137
+ if len(encoded) == 0:
138
+ return encoded, new_boundaries
139
+ # All this shitty numpy casting is annoying
140
+ reduced = np.bitwise_or.reduceat(encoded.view(np.int64), change_indices.view(np.int64))
141
+ reduced = reduced.view(np.uint64)
142
+ return reduced, new_boundaries
143
+
144
+ def decode(self, encoded: np.ndarray, get_keys: bool = True) -> Union[List[Tuple[np.uint64, np.ndarray]], List[np.ndarray]]:
145
+ """Decode an encoded bit array into keys / payloads."""
146
+ keys = (encoded & self.key_mask) >> (_64 - self.key_bits)
147
+ msbs = (encoded & self.payload_msb_mask) >> self.payload_msb_bits
148
+ to_concat = []
149
+ for bit in range(self.payload_lsb_bits):
150
+ mask = 1 << bit
151
+ lsbs = encoded & mask
152
+ set_lsbs = (lsbs != 0)
153
+ this_keys = keys[set_lsbs]
154
+ payload = bit + (msbs[set_lsbs] * self.payload_lsb_bits)
155
+ doc_with_posn = np.dstack([this_keys, payload])[0]
156
+ to_concat.append(doc_with_posn)
157
+
158
+ stacked = np.vstack(to_concat)
159
+ # Sort by doc_id, then posn
160
+ sorted_payload = stacked[np.lexsort((stacked[:, 1], stacked[:, 0]))]
161
+ keys, idx = np.unique(sorted_payload[:, 0], return_index=True)
162
+ grouped = np.split(sorted_payload[:, 1], idx[1:])
163
+ if get_keys:
164
+ return list(zip(keys, grouped))
165
+ else:
166
+ return grouped
167
+
168
+ def num_values_per_key(self, encoded: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
169
+ pcr = popcount64_reduce(encoded, (_64 - self.key_bits), self.payload_lsb_mask)
170
+ return pcr
171
+
172
+ def keys(self, encoded: np.ndarray) -> np.ndarray:
173
+ """Return keys from encoded."""
174
+ return encoded >> (_64 - self.key_bits)
175
+
176
+ def keys_unique(self, encoded: np.ndarray) -> np.ndarray:
177
+ """Return keys from encoded."""
178
+ rshift = _64 - self.key_bits
179
+ return unique(encoded, rshift)
180
+
181
+ def payload_msb(self, encoded: np.ndarray) -> np.ndarray:
182
+ """Return payload MSBs from encoded."""
183
+ return (encoded & self.payload_msb_mask) >> self.payload_msb_bits
184
+
185
+ def payload_lsb(self, encoded: np.ndarray) -> np.ndarray:
186
+ """Return payload LSBs from encoded."""
187
+ return encoded & self.payload_lsb_mask
188
+
189
+ def header(self, encoded: np.ndarray) -> np.ndarray:
190
+ """Return header from encoded -- all but lsb bits."""
191
+ return encoded & ~self.payload_lsb_mask
192
+
193
+ def intersect_candidates(self, lhs: np.ndarray, rhs: np.ndarray) -> Tuple[np.ndarray, np.ndarray,
194
+ np.ndarray, np.ndarray]:
195
+
196
+ lhs_idx, rhs_idx, lhs_adj, rhs_adj = intersect_with_adjacents(lhs, rhs,
197
+ mask=self.header_mask)
198
+ return lhs[lhs_idx], rhs[rhs_idx], lhs[lhs_adj], rhs[rhs_adj]
199
+
200
+ def intersect_rshift(self, lhs: np.ndarray, rhs: np.ndarray,
201
+ rshift: np.int64 = _neg1) -> Tuple[np.ndarray, np.ndarray]:
202
+ """Return the MSBs that are common to both lhs and rhs (same keys, same MSBs)
203
+
204
+ Parameters
205
+ ----------
206
+ lhs : np.ndarray of uint64 (encoded) values
207
+ rhs : np.ndarray of uint64 (encoded) values
208
+ rshift : int how much to shift rhs by to the right
209
+ """
210
+ # print(lhs, rhs, self.header_mask)
211
+ # import pdb; pdb.set_trace()
212
+ lhs_idx, rhs_idx = adjacent(lhs, rhs, mask=self.header_mask)
213
+ return lhs[lhs_idx], rhs[rhs_idx]
214
+
215
+ def intersect(self, lhs: np.ndarray, rhs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
216
+ """Return the MSBs that are common to both lhs and rhs (same keys, same MSBs) as well as post processed values
217
+
218
+ Parameters
219
+ ----------
220
+ lhs : np.ndarray of uint64 (encoded) values
221
+ rhs : np.ndarray of uint64 (encoded) values
222
+ """
223
+ # assert np.all(np.diff(rhs_shifted) >= 0), "not sorted"
224
+ lhs_idx, rhs_idx = intersect(lhs, rhs, mask=self.header_mask)
225
+ return lhs[lhs_idx], rhs[rhs_idx]
226
+
227
+ def key_partition(self,
228
+ encoded: np.ndarray,
229
+ max_key: np.uint64,
230
+ num_partitions=2) -> np.ndarray:
231
+ """Find indices into encoded that split it into num_partitions."""
232
+ # Get every 1/8, 2/8, 3/8, etc of max_key
233
+ last_partition: np.uint64 = _0
234
+ partitions = [np.uint64(0)]
235
+ for i in range(num_partitions - 1):
236
+ max_key_partition = np.uint64(max_key * (i + 1) // num_partitions)
237
+ partition_shifted = max_key_partition << (_64 - self.key_bits)
238
+ idx, is_match = galloping_search(encoded, partition_shifted, self.key_mask, start=last_partition)
239
+ last_partition = idx
240
+ partitions.append(idx)
241
+ # Append last index
242
+ partitions.append(np.uint64(len(encoded)))
243
+ return np.asarray(partitions, dtype=np.uint64)
244
+
245
+ def slice(self,
246
+ encoded: np.ndarray,
247
+ keys: Optional[np.ndarray] = None,
248
+ header: Optional[np.ndarray] = None,
249
+ max_payload: Optional[int] = None,
250
+ min_payload: Optional[int] = None) -> np.ndarray:
251
+ """Get list of encoded that have values in keys."""
252
+ # encoded_keys = encoded.view(np.uint64) >> (_64 - self.key_bits)
253
+ if header is not None:
254
+ if keys is not None:
255
+ raise ValueError("Can't specify both keys and header")
256
+ encoded_header = self.header(encoded)
257
+ idx_docs, idx_enc = intersect(header.view(np.uint64),
258
+ encoded_header.view(np.uint64),
259
+ drop_duplicates=False)
260
+ encoded = encoded[idx_enc]
261
+ if keys is not None:
262
+ encoded_keys = self.keys(encoded)
263
+ idx_docs, idx_enc = intersect(keys.view(np.uint64),
264
+ encoded_keys.view(np.uint64),
265
+ drop_duplicates=False)
266
+ encoded = encoded[idx_enc]
267
+ if max_payload is None and min_payload is None:
268
+ return encoded
269
+ else:
270
+ if min_payload is not None and min_payload % self.payload_lsb_bits != 0:
271
+ raise ValueError(f"min_payload must be a multiple of {self.payload_lsb_bits}")
272
+ if max_payload is not None and max_payload % self.payload_lsb_bits != self.payload_lsb_bits - 1:
273
+ raise ValueError(f"max_payload must be a multiple of {self.payload_lsb_bits} - 1")
274
+ min_payload = 0 if min_payload is None else min_payload
275
+ max_payload = 0xFFFFFFFFFFFFFFFF if max_payload is None else max_payload
276
+
277
+ def payl_slice():
278
+ return payload_slice(encoded, self.payload_msb_mask,
279
+ min_payload // self.payload_lsb_bits,
280
+ max_payload // self.payload_lsb_bits)
281
+
282
+ return payl_slice()
283
+
284
+
285
+ def convert_keys(keys) -> np.ndarray:
286
+ """Convert keys to range or np.ndarray of uint64."""
287
+ if isinstance(keys, numbers.Number):
288
+ return np.asarray([keys], dtype=np.uint64)
289
+ elif isinstance(keys, list):
290
+ return np.asarray(keys, dtype=np.uint64)
291
+ elif isinstance(keys, np.ndarray):
292
+ return keys.astype(np.uint64)
293
+ elif isinstance(keys, range) and len(keys) > 0:
294
+ # UNFORTUNATE COPY
295
+ return np.arange(keys[0], keys[-1] + 1, dtype=np.uint64) + keys[0]
296
+ elif isinstance(keys, range):
297
+ return np.asarray([], dtype=np.uint64)
298
+ raise ValueError(f"Unknown type for keys: {type(keys)}")