searcharray 0.0.73__cp313-cp313-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- searcharray/__init__.py +2 -0
- searcharray/bm25/__init__.py +1 -0
- searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
- searcharray/bm25/bm25.pyi +11 -0
- searcharray/bm25/bm25.pyx +41 -0
- searcharray/indexing.py +342 -0
- searcharray/phrase/__init__.py +0 -0
- searcharray/phrase/bigram_freqs.py +307 -0
- searcharray/phrase/memmap_arrays.py +208 -0
- searcharray/phrase/middle_out.py +553 -0
- searcharray/phrase/spans.py +187 -0
- searcharray/postings.py +708 -0
- searcharray/roaringish/__init__.py +7 -0
- searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/intersect.pyi +25 -0
- searcharray/roaringish/intersect.pyx +390 -0
- searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/merge.pyi +16 -0
- searcharray/roaringish/merge.pyx +232 -0
- searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/popcount.pyi +36 -0
- searcharray/roaringish/popcount.pyx +278 -0
- searcharray/roaringish/roaringish.py +298 -0
- searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/roaringish_ops.pyx +98 -0
- searcharray/roaringish/scatter_assign.h +31 -0
- searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/search.pyi +25 -0
- searcharray/roaringish/search.pyx +140 -0
- searcharray/roaringish/snp_ops.pxd +13 -0
- searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/spans.pyx +330 -0
- searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/unique.pyi +6 -0
- searcharray/roaringish/unique.pyx +145 -0
- searcharray/similarity.py +92 -0
- searcharray/solr.py +355 -0
- searcharray/term_dict.py +59 -0
- searcharray/utils/__init__.py +0 -0
- searcharray/utils/bitcount.py +34 -0
- searcharray/utils/mat_set.py +170 -0
- searcharray/utils/row_viewable_matrix.py +91 -0
- searcharray/utils/sort.py +45 -0
- searcharray-0.0.73.dist-info/METADATA +245 -0
- searcharray-0.0.73.dist-info/RECORD +48 -0
- searcharray-0.0.73.dist-info/WHEEL +5 -0
- searcharray-0.0.73.dist-info/entry_points.txt +2 -0
- searcharray-0.0.73.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from numpy.typing import NDArray
|
|
3
|
+
from typing import Tuple
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def popcount64(arr: NDArray[np.uint64]) -> NDArray[np.uint64]:
|
|
7
|
+
...
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def msb_mask64(arr: NDArray[np.uint64]) -> NDArray[np.uint64]:
|
|
11
|
+
...
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def ctz64(arr: NDArray[np.uint64]) -> NDArray[np.uint64]:
|
|
15
|
+
...
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def clz64(arr: NDArray[np.uint64]) -> NDArray[np.uint64]:
|
|
19
|
+
...
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def popcount_reduce_at(ids: NDArray[np.uint64],
|
|
23
|
+
payload: NDArray[np.uint64]) -> Tuple[NDArray[np.uint64], NDArray[np.float32]]:
|
|
24
|
+
...
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def popcount64_reduce(arr: NDArray[np.uint64],
|
|
28
|
+
key_shift: np.uint64,
|
|
29
|
+
value_mask: np.uint64) -> Tuple[NDArray[np.uint64],
|
|
30
|
+
NDArray[np.uint64]]:
|
|
31
|
+
...
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def key_sum_over(ids: NDArray[np.uint64],
|
|
35
|
+
counts: NDArray[np.uint64]) -> Tuple[NDArray[np.uint64], NDArray[np.float32]]:
|
|
36
|
+
...
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
# These are modified versions of sortednp:
|
|
2
|
+
# https://gitlab.sauerburger.com/frank/sortednp
|
|
3
|
+
# cython: boundscheck=False
|
|
4
|
+
# cython: wraparound=False
|
|
5
|
+
# cython: initializedcheck=False
|
|
6
|
+
# cython: cdivision=True
|
|
7
|
+
# cython: nonecheck=False
|
|
8
|
+
# cython: language_level=3
|
|
9
|
+
cimport numpy as np
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# cimport snp_ops
|
|
14
|
+
# from snp_ops cimport _galloping_search, DTYPE_t, ALL_BITS
|
|
15
|
+
cimport searcharray.roaringish.snp_ops
|
|
16
|
+
from searcharray.roaringish.snp_ops cimport DTYPE_t
|
|
17
|
+
|
|
18
|
+
cdef extern from *:
|
|
19
|
+
"""
|
|
20
|
+
#if defined(_MSC_VER)
|
|
21
|
+
#include <intrin.h>
|
|
22
|
+
#pragma intrinsic(__popcnt64)
|
|
23
|
+
#pragma intrinsic(_BitScanForward64)
|
|
24
|
+
#pragma intrinsic(_BitScanReverse64)
|
|
25
|
+
|
|
26
|
+
static int popcountll(unsigned long long x) {
|
|
27
|
+
return __popcnt64(x);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
static int ctzll(unsigned long long x) {
|
|
31
|
+
unsigned long index;
|
|
32
|
+
if (x == 0) return 64;
|
|
33
|
+
_BitScanForward64(&index, x);
|
|
34
|
+
return index;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
static int clzll(unsigned long long x) {
|
|
38
|
+
unsigned long index;
|
|
39
|
+
if (x == 0) return 64;
|
|
40
|
+
_BitScanReverse64(&index, x);
|
|
41
|
+
return 63 - index;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
#else
|
|
45
|
+
#include <stddef.h>
|
|
46
|
+
static int popcountll(unsigned long long x) {
|
|
47
|
+
return __builtin_popcountll(x);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
static int ctzll(unsigned long long x) {
|
|
51
|
+
return __builtin_ctzll(x);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
static int clzll(unsigned long long x) {
|
|
55
|
+
return __builtin_clzll(x);
|
|
56
|
+
}
|
|
57
|
+
#endif
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
cdef extern from *:
|
|
61
|
+
int popcountll(unsigned long long x)
|
|
62
|
+
int ctzll(unsigned long long x)
|
|
63
|
+
int clzll(unsigned long long x)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# Include mach performance timer
|
|
67
|
+
# cdef extern from "mach/mach_time.h":
|
|
68
|
+
# uint64_t mach_absolute_time()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
cdef popcount64_arr(DTYPE_t[:] arr):
|
|
72
|
+
cdef np.uint64_t[:] result = np.empty(arr.shape[0], dtype=np.uint64)
|
|
73
|
+
# cdef int i = 0
|
|
74
|
+
cdef DTYPE_t* result_ptr = &result[0]
|
|
75
|
+
cdef DTYPE_t* arr_ptr = &arr[0]
|
|
76
|
+
|
|
77
|
+
for _ in range(arr.shape[0]):
|
|
78
|
+
result_ptr[0] = popcountll(arr_ptr[0])
|
|
79
|
+
result_ptr += 1
|
|
80
|
+
arr_ptr += 1
|
|
81
|
+
return result
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
cdef ctz_arr(DTYPE_t[:] arr):
|
|
85
|
+
cdef np.uint64_t[:] result = np.empty(arr.shape[0], dtype=np.uint64)
|
|
86
|
+
# cdef int i = 0
|
|
87
|
+
cdef DTYPE_t* result_ptr = &result[0]
|
|
88
|
+
cdef DTYPE_t* arr_ptr = &arr[0]
|
|
89
|
+
|
|
90
|
+
for _ in range(arr.shape[0]):
|
|
91
|
+
result_ptr[0] = ctzll(arr_ptr[0])
|
|
92
|
+
result_ptr += 1
|
|
93
|
+
arr_ptr += 1
|
|
94
|
+
return result
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
cdef clz_arr(DTYPE_t[:] arr):
|
|
98
|
+
cdef np.uint64_t[:] result = np.empty(arr.shape[0], dtype=np.uint64)
|
|
99
|
+
# cdef int i = 0
|
|
100
|
+
cdef DTYPE_t* result_ptr = &result[0]
|
|
101
|
+
cdef DTYPE_t* arr_ptr = &arr[0]
|
|
102
|
+
|
|
103
|
+
for _ in range(arr.shape[0]):
|
|
104
|
+
result_ptr[0] = clzll(arr_ptr[0])
|
|
105
|
+
result_ptr += 1
|
|
106
|
+
arr_ptr += 1
|
|
107
|
+
return result
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
cdef popcount64_arr_naive(DTYPE_t[:] arr):
|
|
111
|
+
cdef np.uint64_t[:] result = np.empty(arr.shape[0], dtype=np.uint64)
|
|
112
|
+
cdef int i = 0
|
|
113
|
+
|
|
114
|
+
for i in range(arr.shape[0]):
|
|
115
|
+
result[i] = popcountll(arr[i])
|
|
116
|
+
return result
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def popcount64(np.ndarray[DTYPE_t, ndim=1] arr):
|
|
120
|
+
"""Count the number of set bits in a 64-bit integer."""
|
|
121
|
+
return np.array(popcount64_arr(arr))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
cdef DTYPE_t _popcount_reduce_at(DTYPE_t[:] ids, DTYPE_t[:] payload,
|
|
125
|
+
DTYPE_t[:] merged_ids, float[:] merged_counts):
|
|
126
|
+
cdef DTYPE_t last_id = ids[0]
|
|
127
|
+
cdef DTYPE_t popcount_sum = 0
|
|
128
|
+
cdef DTYPE_t* merged_ids_ptr = &merged_ids[0]
|
|
129
|
+
cdef DTYPE_t* payload_ptr = &payload[0]
|
|
130
|
+
cdef DTYPE_t* ids_ptr = &ids[0]
|
|
131
|
+
cdef float* merged_counts_ptr = &merged_counts[0]
|
|
132
|
+
|
|
133
|
+
# We already have 0, now add new values
|
|
134
|
+
while ids_ptr < &ids[0] + ids.shape[0]:
|
|
135
|
+
if ids_ptr[0] != last_id:
|
|
136
|
+
merged_ids_ptr[0] = last_id
|
|
137
|
+
merged_counts_ptr[0] = popcount_sum
|
|
138
|
+
popcount_sum = 0
|
|
139
|
+
merged_ids_ptr += 1
|
|
140
|
+
merged_counts_ptr += 1
|
|
141
|
+
popcount_sum += popcountll(payload_ptr[0])
|
|
142
|
+
last_id = ids_ptr[0]
|
|
143
|
+
payload_ptr += 1
|
|
144
|
+
ids_ptr += 1
|
|
145
|
+
# Save final value
|
|
146
|
+
merged_ids_ptr[0] = last_id
|
|
147
|
+
merged_counts_ptr[0] = popcount_sum
|
|
148
|
+
return merged_ids_ptr - &merged_ids[0] + 1
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def popcount_reduce_at(np.ndarray[DTYPE_t, ndim=1] ids,
|
|
152
|
+
np.ndarray[DTYPE_t, ndim=1] payload):
|
|
153
|
+
"""Write the sum of popcount of the payload at the indices in ids to the output array.
|
|
154
|
+
|
|
155
|
+
Add one to each plus_one ids
|
|
156
|
+
|
|
157
|
+
"""
|
|
158
|
+
if len(ids) != len(payload):
|
|
159
|
+
raise ValueError("ids and payload must have the same length")
|
|
160
|
+
if len(ids) == 0:
|
|
161
|
+
return np.array([]), np.array([])
|
|
162
|
+
merged_ids = np.empty(ids.shape[0], dtype=np.uint64)
|
|
163
|
+
merged_counts = np.empty(ids.shape[0], dtype=np.float32)
|
|
164
|
+
merged_len = _popcount_reduce_at(ids, payload, merged_ids, merged_counts)
|
|
165
|
+
return np.array(merged_ids[:merged_len]), np.array(merged_counts[:merged_len])
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
cdef _key_sum_over(DTYPE_t[:] ids, DTYPE_t[:] count,
|
|
169
|
+
DTYPE_t[:] merged_ids, float[:] merged_counts):
|
|
170
|
+
cdef DTYPE_t* ids_ptr = &ids[0]
|
|
171
|
+
cdef DTYPE_t* count_ptr = &count[0]
|
|
172
|
+
cdef DTYPE_t* merged_ids_ptr = &merged_ids[0]
|
|
173
|
+
cdef float* merged_counts_ptr = &merged_counts[0]
|
|
174
|
+
cdef DTYPE_t last_id = ids[0]
|
|
175
|
+
cdef DTYPE_t payload_sum = 0
|
|
176
|
+
|
|
177
|
+
while ids_ptr < &ids[0] + ids.shape[0]:
|
|
178
|
+
if ids_ptr[0] != last_id:
|
|
179
|
+
merged_ids_ptr[0] = last_id
|
|
180
|
+
merged_counts_ptr[0] = payload_sum
|
|
181
|
+
payload_sum = 0
|
|
182
|
+
merged_ids_ptr += 1
|
|
183
|
+
merged_counts_ptr += 1
|
|
184
|
+
payload_sum += count_ptr[0]
|
|
185
|
+
last_id = ids_ptr[0]
|
|
186
|
+
ids_ptr += 1
|
|
187
|
+
count_ptr += 1
|
|
188
|
+
# Save final value
|
|
189
|
+
merged_ids_ptr[0] = last_id
|
|
190
|
+
merged_counts_ptr[0] = payload_sum
|
|
191
|
+
return merged_ids_ptr - &merged_ids[0] + 1
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def key_sum_over(np.ndarray[DTYPE_t, ndim=1] ids,
|
|
195
|
+
np.ndarray[DTYPE_t, ndim=1] count):
|
|
196
|
+
"""Write the last value of the payload at the indices in ids to the output array."""
|
|
197
|
+
if len(ids) != len(count):
|
|
198
|
+
raise ValueError("ids and count must have the same length")
|
|
199
|
+
if len(ids) == 0:
|
|
200
|
+
return np.array([]), np.array([])
|
|
201
|
+
merged_ids = np.empty(ids.shape[0], dtype=np.uint64)
|
|
202
|
+
merged_counts = np.empty(ids.shape[0], dtype=np.float32)
|
|
203
|
+
merged_len = _key_sum_over(ids, count, merged_ids, merged_counts)
|
|
204
|
+
return np.array(merged_ids[:merged_len]), np.array(merged_counts[:merged_len])
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
# Popcount reduce key-value pair
|
|
208
|
+
# for words 0xKKKKKKKK...KKKKVVVV...VVVV
|
|
209
|
+
# Returning two parallel arrays:
|
|
210
|
+
# - keys - the keys
|
|
211
|
+
# - values - the popcount of the values with those keys
|
|
212
|
+
cdef _popcount64_reduce(DTYPE_t[:] arr,
|
|
213
|
+
DTYPE_t key_shift,
|
|
214
|
+
DTYPE_t value_mask):
|
|
215
|
+
cdef float[:] popcounts = np.zeros(arr.shape[0], dtype=np.float32)
|
|
216
|
+
cdef DTYPE_t[:] keys = np.empty(arr.shape[0], dtype=np.uint64)
|
|
217
|
+
cdef float* popcounts_ptr = &popcounts[0]
|
|
218
|
+
cdef DTYPE_t* keys_ptr = &keys[0]
|
|
219
|
+
cdef DTYPE_t* arr_ptr = &arr[0]
|
|
220
|
+
|
|
221
|
+
cdef DTYPE_t last_key = arr_ptr[0] >> key_shift
|
|
222
|
+
cdef DTYPE_t key = last_key
|
|
223
|
+
keys_ptr[0] = last_key
|
|
224
|
+
|
|
225
|
+
for _ in range(arr.shape[0]):
|
|
226
|
+
key = arr_ptr[0] >> key_shift
|
|
227
|
+
if key == last_key:
|
|
228
|
+
popcounts_ptr[0] += popcountll(arr_ptr[0] & value_mask)
|
|
229
|
+
else:
|
|
230
|
+
last_key = key
|
|
231
|
+
popcounts_ptr += 1
|
|
232
|
+
keys_ptr += 1
|
|
233
|
+
# Init next key
|
|
234
|
+
keys_ptr[0] = last_key
|
|
235
|
+
popcounts_ptr[0] = popcountll(arr_ptr[0] & value_mask)
|
|
236
|
+
arr_ptr += 1
|
|
237
|
+
return keys, popcounts, (keys_ptr - &keys[0] + 1)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
# Branchless version using lookup table instead of if...
|
|
241
|
+
# Popcount reduce key-value pair
|
|
242
|
+
# for words 0xKKKKKKKK...KKKKVVVV...VVVV
|
|
243
|
+
# Returning two parallel arrays:
|
|
244
|
+
# - keys - the keys
|
|
245
|
+
# - values - the popcount of the values with those keys
|
|
246
|
+
cdef _popcount64_reduce_nobranch(DTYPE_t[:] arr,
|
|
247
|
+
DTYPE_t key_shift,
|
|
248
|
+
DTYPE_t value_mask):
|
|
249
|
+
cdef float[:] popcounts = np.zeros(arr.shape[0], dtype=np.float32)
|
|
250
|
+
cdef DTYPE_t[:] keys = np.empty(arr.shape[0], dtype=np.uint64)
|
|
251
|
+
cdef float* popcounts_ptr = &popcounts[0]
|
|
252
|
+
cdef DTYPE_t* keys_ptr = &keys[0]
|
|
253
|
+
cdef DTYPE_t* arr_ptr = &arr[0]
|
|
254
|
+
|
|
255
|
+
cdef DTYPE_t last_key = arr_ptr[0] >> key_shift
|
|
256
|
+
cdef DTYPE_t key = last_key
|
|
257
|
+
|
|
258
|
+
keys_ptr[0] = last_key
|
|
259
|
+
|
|
260
|
+
for _ in range(arr.shape[0]):
|
|
261
|
+
key = arr_ptr[0] >> key_shift
|
|
262
|
+
popcounts_ptr += (key != last_key)
|
|
263
|
+
keys_ptr += (key != last_key)
|
|
264
|
+
popcounts_ptr[0] += popcountll(arr_ptr[0] & value_mask)
|
|
265
|
+
keys_ptr[0] = key
|
|
266
|
+
last_key = key
|
|
267
|
+
arr_ptr += 1
|
|
268
|
+
return keys, popcounts, (keys_ptr - &keys[0] + 1)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def popcount64_reduce(arr,
|
|
272
|
+
key_shift,
|
|
273
|
+
value_mask):
|
|
274
|
+
cdef DTYPE_t[:] arr_view = arr
|
|
275
|
+
if len(arr_view) == 0:
|
|
276
|
+
return np.array([]), np.array([])
|
|
277
|
+
keys, popcounts, results_idx = _popcount64_reduce(arr_view, key_shift, value_mask)
|
|
278
|
+
return np.array(keys[:results_idx]), np.array(popcounts[:results_idx])
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"""Roaring-ish bit array for storing sorted integers in numpy array.
|
|
2
|
+
|
|
3
|
+
See - https://softwaredoug.com/blog/2024/01/21/search-array-phrase-algorithm
|
|
4
|
+
"""
|
|
5
|
+
import numpy as np
|
|
6
|
+
import logging
|
|
7
|
+
import numbers
|
|
8
|
+
from typing import Optional, Tuple, List, Union
|
|
9
|
+
|
|
10
|
+
from searcharray.roaringish.search import galloping_search
|
|
11
|
+
from searcharray.roaringish.merge import merge
|
|
12
|
+
from searcharray.roaringish.unique import unique
|
|
13
|
+
from searcharray.roaringish.intersect import intersect, adjacent, intersect_with_adjacents
|
|
14
|
+
from searcharray.roaringish.popcount import popcount64_reduce
|
|
15
|
+
from searcharray.roaringish.roaringish_ops import payload_slice
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
# When running in pytest
|
|
21
|
+
import sys # noqa
|
|
22
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
23
|
+
handler.setLevel(logging.ERROR)
|
|
24
|
+
formatter = logging.Formatter("[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
|
|
25
|
+
handler.setFormatter(formatter)
|
|
26
|
+
logger.addHandler(handler)
|
|
27
|
+
logger.setLevel(logging.ERROR)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
DEFAULT_KEY_MASK = np.uint64(0xFFFFFFF000000000)
|
|
31
|
+
DEFAULT_KEY_BITS = np.uint64(28)
|
|
32
|
+
DEFAULT_PAYLOAD_MSB_MASK = np.uint64(0x0000000FFFFC0000)
|
|
33
|
+
DEFAULT_PAYLOAD_MSB_BITS = np.uint64(18)
|
|
34
|
+
DEFAULT_PAYLOAD_LSB_MASK = np.uint64(0x000000000003FFFF)
|
|
35
|
+
DEFAULT_PAYLOAD_LSB_BITS = np.uint64(18)
|
|
36
|
+
|
|
37
|
+
# To not constantly type coerce
|
|
38
|
+
_64 = np.uint64(64)
|
|
39
|
+
_2 = np.uint64(2)
|
|
40
|
+
_1 = np.uint64(1)
|
|
41
|
+
_0 = np.uint64(0)
|
|
42
|
+
_neg1 = np.int64(-1)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def n_msb_mask(n: np.uint64) -> np.uint64:
|
|
46
|
+
"""Return the n most significant bits of num."""
|
|
47
|
+
return np.uint64(~(np.uint64(_1 << (_64 - n))) + _1)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def sorted_unique(arr: np.ndarray) -> np.ndarray:
|
|
51
|
+
return unique(arr)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class RoaringishEncoder:
|
|
55
|
+
"""An encoder for key->integer sets as a numpy array.
|
|
56
|
+
|
|
57
|
+
Each returned array represents a single term, with key as MSBS, ie:
|
|
58
|
+
|
|
59
|
+
| 32 MSBs | 16 LSBs | 16 LSBs |
|
|
60
|
+
key | bits msbs | payload
|
|
61
|
+
|
|
62
|
+
(different number of MSBs / payload bits can be specified)
|
|
63
|
+
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
def __init__(self, key_bits: np.uint64 = DEFAULT_KEY_BITS):
|
|
67
|
+
payload_bits = _64 - key_bits
|
|
68
|
+
self.payload_msb_bits = payload_bits // _2
|
|
69
|
+
self.payload_lsb_bits = np.uint64(payload_bits - self.payload_msb_bits)
|
|
70
|
+
self.key_bits = key_bits
|
|
71
|
+
assert self.key_bits.dtype == np.uint64
|
|
72
|
+
# key bits MSB of 64 bits
|
|
73
|
+
self.key_mask = n_msb_mask(key_bits)
|
|
74
|
+
self.header_bits = key_bits + self.payload_msb_bits
|
|
75
|
+
self.payload_msb_mask = n_msb_mask(np.uint64(self.payload_msb_bits + key_bits)) & ~self.key_mask
|
|
76
|
+
assert self.payload_msb_bits.dtype == np.uint64, f"MSB bits dtype was {self.payload_msb_bits.dtype}"
|
|
77
|
+
assert self.payload_msb_mask.dtype == np.uint64, f"MSB mask dtype was {self.payload_msb_mask.dtype}"
|
|
78
|
+
self.payload_lsb_mask = (_1 << self.payload_lsb_bits) - np.uint64(1)
|
|
79
|
+
assert self.payload_lsb_bits.dtype == np.uint64, f"LSB bits dtype was {self.payload_lsb_bits.dtype}"
|
|
80
|
+
assert self.payload_lsb_mask.dtype == np.uint64, f"LSB mask dtype was {self.payload_lsb_mask.dtype}"
|
|
81
|
+
self.header_mask = self.key_mask | self.payload_msb_mask
|
|
82
|
+
if key_bits == DEFAULT_KEY_BITS:
|
|
83
|
+
assert self.key_mask == DEFAULT_KEY_MASK
|
|
84
|
+
assert self.payload_msb_mask == DEFAULT_PAYLOAD_MSB_MASK
|
|
85
|
+
assert self.payload_lsb_mask == DEFAULT_PAYLOAD_LSB_MASK
|
|
86
|
+
self.max_payload = np.uint64(2**self.payload_lsb_bits - 1)
|
|
87
|
+
|
|
88
|
+
def validate_payload(self, payload: np.ndarray):
|
|
89
|
+
"""Optional validation of payload."""
|
|
90
|
+
if np.any(payload > self.max_payload):
|
|
91
|
+
raise ValueError(f"Positions must be less than {2**self.payload_lsb_bits}")
|
|
92
|
+
|
|
93
|
+
def encode(self, payload: np.ndarray,
|
|
94
|
+
keys: Optional[np.ndarray] = None,
|
|
95
|
+
boundaries: Optional[np.ndarray] = None) -> Tuple[np.ndarray, Optional[np.ndarray]]:
|
|
96
|
+
"""Pack a sorted array of integers into compact bit numpy array.
|
|
97
|
+
|
|
98
|
+
each returned array represents a single term, with key as MSBS, ie:
|
|
99
|
+
|
|
100
|
+
| 32 MSBs | 16 LSBs | 16 LSBs |
|
|
101
|
+
key | bits msbs| payload
|
|
102
|
+
|
|
103
|
+
for later easy intersection of 32+16 msbs, then checking for adjacent
|
|
104
|
+
positions
|
|
105
|
+
|
|
106
|
+
If boundaries are provided, then we consider multiple distinct payloads
|
|
107
|
+
being encoded simultaneously, and we return the boundaries of each
|
|
108
|
+
|
|
109
|
+
"""
|
|
110
|
+
cols = np.floor_divide(payload, self.payload_lsb_bits, dtype=np.uint64) # Header of bit to use
|
|
111
|
+
cols <<= self.payload_msb_bits
|
|
112
|
+
if keys is not None:
|
|
113
|
+
cols |= keys.astype(np.uint64) << (_64 - self.key_bits)
|
|
114
|
+
values = payload % self.payload_lsb_bits # Value to encode
|
|
115
|
+
|
|
116
|
+
change_indices_one_doc = np.nonzero(np.diff(cols))[0] + 1
|
|
117
|
+
change_indices_one_doc = change_indices_one_doc.view(np.uint64)
|
|
118
|
+
change_indices_one_doc = np.concatenate([[_0], change_indices_one_doc], dtype=np.uint64)
|
|
119
|
+
if boundaries is not None:
|
|
120
|
+
change_indices = merge(change_indices_one_doc.view(np.uint64),
|
|
121
|
+
boundaries.view(np.uint64),
|
|
122
|
+
drop_duplicates=True)
|
|
123
|
+
new_boundaries = intersect(boundaries,
|
|
124
|
+
change_indices)[-1]
|
|
125
|
+
new_boundaries = np.concatenate([new_boundaries,
|
|
126
|
+
np.asarray([len(change_indices)], dtype=np.uint64)],
|
|
127
|
+
dtype=np.uint64)
|
|
128
|
+
else:
|
|
129
|
+
change_indices = change_indices_one_doc
|
|
130
|
+
new_boundaries = None
|
|
131
|
+
|
|
132
|
+
# 0 as a position, goes in bit 1,
|
|
133
|
+
# 1 as a position, goes in bit 2, etc
|
|
134
|
+
values = _1 << values
|
|
135
|
+
cols |= values
|
|
136
|
+
encoded = cols
|
|
137
|
+
if len(encoded) == 0:
|
|
138
|
+
return encoded, new_boundaries
|
|
139
|
+
# All this shitty numpy casting is annoying
|
|
140
|
+
reduced = np.bitwise_or.reduceat(encoded.view(np.int64), change_indices.view(np.int64))
|
|
141
|
+
reduced = reduced.view(np.uint64)
|
|
142
|
+
return reduced, new_boundaries
|
|
143
|
+
|
|
144
|
+
def decode(self, encoded: np.ndarray, get_keys: bool = True) -> Union[List[Tuple[np.uint64, np.ndarray]], List[np.ndarray]]:
|
|
145
|
+
"""Decode an encoded bit array into keys / payloads."""
|
|
146
|
+
keys = (encoded & self.key_mask) >> (_64 - self.key_bits)
|
|
147
|
+
msbs = (encoded & self.payload_msb_mask) >> self.payload_msb_bits
|
|
148
|
+
to_concat = []
|
|
149
|
+
for bit in range(self.payload_lsb_bits):
|
|
150
|
+
mask = 1 << bit
|
|
151
|
+
lsbs = encoded & mask
|
|
152
|
+
set_lsbs = (lsbs != 0)
|
|
153
|
+
this_keys = keys[set_lsbs]
|
|
154
|
+
payload = bit + (msbs[set_lsbs] * self.payload_lsb_bits)
|
|
155
|
+
doc_with_posn = np.dstack([this_keys, payload])[0]
|
|
156
|
+
to_concat.append(doc_with_posn)
|
|
157
|
+
|
|
158
|
+
stacked = np.vstack(to_concat)
|
|
159
|
+
# Sort by doc_id, then posn
|
|
160
|
+
sorted_payload = stacked[np.lexsort((stacked[:, 1], stacked[:, 0]))]
|
|
161
|
+
keys, idx = np.unique(sorted_payload[:, 0], return_index=True)
|
|
162
|
+
grouped = np.split(sorted_payload[:, 1], idx[1:])
|
|
163
|
+
if get_keys:
|
|
164
|
+
return list(zip(keys, grouped))
|
|
165
|
+
else:
|
|
166
|
+
return grouped
|
|
167
|
+
|
|
168
|
+
def num_values_per_key(self, encoded: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
|
169
|
+
pcr = popcount64_reduce(encoded, (_64 - self.key_bits), self.payload_lsb_mask)
|
|
170
|
+
return pcr
|
|
171
|
+
|
|
172
|
+
def keys(self, encoded: np.ndarray) -> np.ndarray:
|
|
173
|
+
"""Return keys from encoded."""
|
|
174
|
+
return encoded >> (_64 - self.key_bits)
|
|
175
|
+
|
|
176
|
+
def keys_unique(self, encoded: np.ndarray) -> np.ndarray:
|
|
177
|
+
"""Return keys from encoded."""
|
|
178
|
+
rshift = _64 - self.key_bits
|
|
179
|
+
return unique(encoded, rshift)
|
|
180
|
+
|
|
181
|
+
def payload_msb(self, encoded: np.ndarray) -> np.ndarray:
|
|
182
|
+
"""Return payload MSBs from encoded."""
|
|
183
|
+
return (encoded & self.payload_msb_mask) >> self.payload_msb_bits
|
|
184
|
+
|
|
185
|
+
def payload_lsb(self, encoded: np.ndarray) -> np.ndarray:
|
|
186
|
+
"""Return payload LSBs from encoded."""
|
|
187
|
+
return encoded & self.payload_lsb_mask
|
|
188
|
+
|
|
189
|
+
def header(self, encoded: np.ndarray) -> np.ndarray:
|
|
190
|
+
"""Return header from encoded -- all but lsb bits."""
|
|
191
|
+
return encoded & ~self.payload_lsb_mask
|
|
192
|
+
|
|
193
|
+
def intersect_candidates(self, lhs: np.ndarray, rhs: np.ndarray) -> Tuple[np.ndarray, np.ndarray,
|
|
194
|
+
np.ndarray, np.ndarray]:
|
|
195
|
+
|
|
196
|
+
lhs_idx, rhs_idx, lhs_adj, rhs_adj = intersect_with_adjacents(lhs, rhs,
|
|
197
|
+
mask=self.header_mask)
|
|
198
|
+
return lhs[lhs_idx], rhs[rhs_idx], lhs[lhs_adj], rhs[rhs_adj]
|
|
199
|
+
|
|
200
|
+
def intersect_rshift(self, lhs: np.ndarray, rhs: np.ndarray,
|
|
201
|
+
rshift: np.int64 = _neg1) -> Tuple[np.ndarray, np.ndarray]:
|
|
202
|
+
"""Return the MSBs that are common to both lhs and rhs (same keys, same MSBs)
|
|
203
|
+
|
|
204
|
+
Parameters
|
|
205
|
+
----------
|
|
206
|
+
lhs : np.ndarray of uint64 (encoded) values
|
|
207
|
+
rhs : np.ndarray of uint64 (encoded) values
|
|
208
|
+
rshift : int how much to shift rhs by to the right
|
|
209
|
+
"""
|
|
210
|
+
# print(lhs, rhs, self.header_mask)
|
|
211
|
+
# import pdb; pdb.set_trace()
|
|
212
|
+
lhs_idx, rhs_idx = adjacent(lhs, rhs, mask=self.header_mask)
|
|
213
|
+
return lhs[lhs_idx], rhs[rhs_idx]
|
|
214
|
+
|
|
215
|
+
def intersect(self, lhs: np.ndarray, rhs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
|
216
|
+
"""Return the MSBs that are common to both lhs and rhs (same keys, same MSBs) as well as post processed values
|
|
217
|
+
|
|
218
|
+
Parameters
|
|
219
|
+
----------
|
|
220
|
+
lhs : np.ndarray of uint64 (encoded) values
|
|
221
|
+
rhs : np.ndarray of uint64 (encoded) values
|
|
222
|
+
"""
|
|
223
|
+
# assert np.all(np.diff(rhs_shifted) >= 0), "not sorted"
|
|
224
|
+
lhs_idx, rhs_idx = intersect(lhs, rhs, mask=self.header_mask)
|
|
225
|
+
return lhs[lhs_idx], rhs[rhs_idx]
|
|
226
|
+
|
|
227
|
+
def key_partition(self,
|
|
228
|
+
encoded: np.ndarray,
|
|
229
|
+
max_key: np.uint64,
|
|
230
|
+
num_partitions=2) -> np.ndarray:
|
|
231
|
+
"""Find indices into encoded that split it into num_partitions."""
|
|
232
|
+
# Get every 1/8, 2/8, 3/8, etc of max_key
|
|
233
|
+
last_partition: np.uint64 = _0
|
|
234
|
+
partitions = [np.uint64(0)]
|
|
235
|
+
for i in range(num_partitions - 1):
|
|
236
|
+
max_key_partition = np.uint64(max_key * (i + 1) // num_partitions)
|
|
237
|
+
partition_shifted = max_key_partition << (_64 - self.key_bits)
|
|
238
|
+
idx, is_match = galloping_search(encoded, partition_shifted, self.key_mask, start=last_partition)
|
|
239
|
+
last_partition = idx
|
|
240
|
+
partitions.append(idx)
|
|
241
|
+
# Append last index
|
|
242
|
+
partitions.append(np.uint64(len(encoded)))
|
|
243
|
+
return np.asarray(partitions, dtype=np.uint64)
|
|
244
|
+
|
|
245
|
+
def slice(self,
|
|
246
|
+
encoded: np.ndarray,
|
|
247
|
+
keys: Optional[np.ndarray] = None,
|
|
248
|
+
header: Optional[np.ndarray] = None,
|
|
249
|
+
max_payload: Optional[int] = None,
|
|
250
|
+
min_payload: Optional[int] = None) -> np.ndarray:
|
|
251
|
+
"""Get list of encoded that have values in keys."""
|
|
252
|
+
# encoded_keys = encoded.view(np.uint64) >> (_64 - self.key_bits)
|
|
253
|
+
if header is not None:
|
|
254
|
+
if keys is not None:
|
|
255
|
+
raise ValueError("Can't specify both keys and header")
|
|
256
|
+
encoded_header = self.header(encoded)
|
|
257
|
+
idx_docs, idx_enc = intersect(header.view(np.uint64),
|
|
258
|
+
encoded_header.view(np.uint64),
|
|
259
|
+
drop_duplicates=False)
|
|
260
|
+
encoded = encoded[idx_enc]
|
|
261
|
+
if keys is not None:
|
|
262
|
+
encoded_keys = self.keys(encoded)
|
|
263
|
+
idx_docs, idx_enc = intersect(keys.view(np.uint64),
|
|
264
|
+
encoded_keys.view(np.uint64),
|
|
265
|
+
drop_duplicates=False)
|
|
266
|
+
encoded = encoded[idx_enc]
|
|
267
|
+
if max_payload is None and min_payload is None:
|
|
268
|
+
return encoded
|
|
269
|
+
else:
|
|
270
|
+
if min_payload is not None and min_payload % self.payload_lsb_bits != 0:
|
|
271
|
+
raise ValueError(f"min_payload must be a multiple of {self.payload_lsb_bits}")
|
|
272
|
+
if max_payload is not None and max_payload % self.payload_lsb_bits != self.payload_lsb_bits - 1:
|
|
273
|
+
raise ValueError(f"max_payload must be a multiple of {self.payload_lsb_bits} - 1")
|
|
274
|
+
min_payload = 0 if min_payload is None else min_payload
|
|
275
|
+
max_payload = 0xFFFFFFFFFFFFFFFF if max_payload is None else max_payload
|
|
276
|
+
|
|
277
|
+
def payl_slice():
|
|
278
|
+
return payload_slice(encoded, self.payload_msb_mask,
|
|
279
|
+
min_payload // self.payload_lsb_bits,
|
|
280
|
+
max_payload // self.payload_lsb_bits)
|
|
281
|
+
|
|
282
|
+
return payl_slice()
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def convert_keys(keys) -> np.ndarray:
|
|
286
|
+
"""Convert keys to range or np.ndarray of uint64."""
|
|
287
|
+
if isinstance(keys, numbers.Number):
|
|
288
|
+
return np.asarray([keys], dtype=np.uint64)
|
|
289
|
+
elif isinstance(keys, list):
|
|
290
|
+
return np.asarray(keys, dtype=np.uint64)
|
|
291
|
+
elif isinstance(keys, np.ndarray):
|
|
292
|
+
return keys.astype(np.uint64)
|
|
293
|
+
elif isinstance(keys, range) and len(keys) > 0:
|
|
294
|
+
# UNFORTUNATE COPY
|
|
295
|
+
return np.arange(keys[0], keys[-1] + 1, dtype=np.uint64) + keys[0]
|
|
296
|
+
elif isinstance(keys, range):
|
|
297
|
+
return np.asarray([], dtype=np.uint64)
|
|
298
|
+
raise ValueError(f"Unknown type for keys: {type(keys)}")
|
|
Binary file
|