searcharray 0.0.73__cp313-cp313-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. searcharray/__init__.py +2 -0
  2. searcharray/bm25/__init__.py +1 -0
  3. searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
  4. searcharray/bm25/bm25.pyi +11 -0
  5. searcharray/bm25/bm25.pyx +41 -0
  6. searcharray/indexing.py +342 -0
  7. searcharray/phrase/__init__.py +0 -0
  8. searcharray/phrase/bigram_freqs.py +307 -0
  9. searcharray/phrase/memmap_arrays.py +208 -0
  10. searcharray/phrase/middle_out.py +553 -0
  11. searcharray/phrase/spans.py +187 -0
  12. searcharray/postings.py +708 -0
  13. searcharray/roaringish/__init__.py +7 -0
  14. searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
  15. searcharray/roaringish/intersect.pyi +25 -0
  16. searcharray/roaringish/intersect.pyx +390 -0
  17. searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
  18. searcharray/roaringish/merge.pyi +16 -0
  19. searcharray/roaringish/merge.pyx +232 -0
  20. searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
  21. searcharray/roaringish/popcount.pyi +36 -0
  22. searcharray/roaringish/popcount.pyx +278 -0
  23. searcharray/roaringish/roaringish.py +298 -0
  24. searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
  25. searcharray/roaringish/roaringish_ops.pyx +98 -0
  26. searcharray/roaringish/scatter_assign.h +31 -0
  27. searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
  28. searcharray/roaringish/search.pyi +25 -0
  29. searcharray/roaringish/search.pyx +140 -0
  30. searcharray/roaringish/snp_ops.pxd +13 -0
  31. searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
  32. searcharray/roaringish/spans.pyx +330 -0
  33. searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
  34. searcharray/roaringish/unique.pyi +6 -0
  35. searcharray/roaringish/unique.pyx +145 -0
  36. searcharray/similarity.py +92 -0
  37. searcharray/solr.py +355 -0
  38. searcharray/term_dict.py +59 -0
  39. searcharray/utils/__init__.py +0 -0
  40. searcharray/utils/bitcount.py +34 -0
  41. searcharray/utils/mat_set.py +170 -0
  42. searcharray/utils/row_viewable_matrix.py +91 -0
  43. searcharray/utils/sort.py +45 -0
  44. searcharray-0.0.73.dist-info/METADATA +245 -0
  45. searcharray-0.0.73.dist-info/RECORD +48 -0
  46. searcharray-0.0.73.dist-info/WHEEL +5 -0
  47. searcharray-0.0.73.dist-info/entry_points.txt +2 -0
  48. searcharray-0.0.73.dist-info/top_level.txt +1 -0
@@ -0,0 +1,98 @@
1
+ # These are modified versions of sortednp:
2
+ # https://gitlab.sauerburger.com/frank/sortednp
3
+ # cython: boundscheck=False
4
+ # cython: wraparound=False
5
+ # cython: initializedcheck=False
6
+ # cython: cdivision=True
7
+ # cython: nonecheck=False
8
+ # cython: language_level=3
9
+ cimport numpy as np
10
+ import numpy as np
11
+
12
+
13
+ # example.pyx
14
+ cdef extern from "scatter_assign.h":
15
+ void scatter_naive(float *array,
16
+ const unsigned long long *indices,
17
+ const float *values, int n) nogil
18
+
19
+ cimport searcharray.roaringish.snp_ops
20
+ from searcharray.roaringish.snp_ops cimport DTYPE_t
21
+
22
+
23
+ cdef DTYPE_t ALL_BITS = 0xFFFFFFFFFFFFFFFF
24
+
25
+
26
+ cdef extern from *:
27
+ """
28
+ #if defined(_MSC_VER)
29
+ #include <intrin.h>
30
+ #pragma intrinsic(__popcnt64)
31
+ static int popcountll(unsigned long long x) {
32
+ return __popcnt64(x);
33
+ }
34
+ #else
35
+ #include <stddef.h>
36
+ static int popcountll(unsigned long long x) {
37
+ return __builtin_popcountll(x);
38
+ }
39
+ #endif
40
+ """
41
+
42
+ cdef extern from *:
43
+ int popcountll(unsigned long long x)
44
+
45
+
46
+ cdef _payload_slice(DTYPE_t[:] arr,
47
+ DTYPE_t payload_msb_mask,
48
+ DTYPE_t min_payload,
49
+ DTYPE_t max_payload):
50
+ cdef DTYPE_t[:] sliced = np.zeros(arr.shape[0], dtype=np.uint64)
51
+ cdef DTYPE_t* sliced_ptr = &sliced[0]
52
+ cdef DTYPE_t* arr_ptr = &arr[0]
53
+
54
+ while arr_ptr < &arr[arr.shape[0]]:
55
+ if (arr_ptr[0] & payload_msb_mask) >= min_payload and (arr_ptr[0] & payload_msb_mask) <= max_payload:
56
+ sliced_ptr[0] = arr_ptr[0]
57
+ sliced_ptr += 1
58
+ arr_ptr += 1
59
+
60
+ return sliced, sliced_ptr - &sliced[0]
61
+
62
+
63
+ def payload_slice(arr, payload_msb_mask,
64
+ min_payload=0, max_payload=0xFFFFFFFFFFFFFFFF):
65
+ cdef DTYPE_t[:] arr_view = arr
66
+ sliced, sliced_len = _payload_slice(arr_view, payload_msb_mask,
67
+ min_payload, max_payload)
68
+ return np.array(sliced[:sliced_len])
69
+
70
+
71
+ cdef void _as_dense_array(DTYPE_t[:] indices, # Its likely these indices are sorted, if that helps
72
+ float[:] values,
73
+ float[:] arr_out) nogil:
74
+
75
+ cdef DTYPE_t* indices_ptr = &indices[0]
76
+ cdef float* values_ptr = &values[0]
77
+
78
+ while indices_ptr < &indices[indices.shape[0]]:
79
+ arr_out[indices_ptr[0]] = values_ptr[0]
80
+ indices_ptr += 1
81
+ values_ptr += 1
82
+
83
+
84
+ def as_dense(indices, values, size):
85
+ cdef float[:] arr_out = np.zeros(size, dtype=np.float32)
86
+ cdef DTYPE_t[:] indices_view = indices
87
+ cdef DTYPE_t len_indices = indices.shape[0]
88
+ cdef float[:] values_view = values
89
+ if len(indices) != len(values):
90
+ raise ValueError("indices and values must have the same length")
91
+ # _as_dense_array(indices_view, values_view,
92
+ # arr_out)
93
+ with nogil:
94
+ scatter_naive(&arr_out[0],
95
+ &indices_view[0],
96
+ &values_view[0],
97
+ len_indices)
98
+ return np.array(arr_out)
@@ -0,0 +1,31 @@
1
+ #ifndef SCATTER_ASSIGN_H
2
+ #define SCATTER_ASSIGN_H
3
+
4
+ #include <stdio.h>
5
+
6
+ void scatter_naive(float *array,
7
+ const unsigned long long *indices,
8
+ const float *values,
9
+ int n) {
10
+
11
+ const unsigned long long *end = indices + n;
12
+ const unsigned long long *unroll_end = indices + (n / 10) * 10;
13
+ while (indices < unroll_end) {
14
+ array[*indices++] = *values++;
15
+ array[*indices++] = *values++;
16
+ array[*indices++] = *values++;
17
+ array[*indices++] = *values++;
18
+ array[*indices++] = *values++;
19
+ array[*indices++] = *values++;
20
+ array[*indices++] = *values++;
21
+ array[*indices++] = *values++;
22
+ array[*indices++] = *values++;
23
+ array[*indices++] = *values++;
24
+ }
25
+ while (indices < end) {
26
+ array[*indices++] = *values++;
27
+ }
28
+
29
+ }
30
+
31
+ #endif // SCATTER_ASSIGN_H
@@ -0,0 +1,25 @@
1
+ import numpy as np
2
+ from numpy.typing import NDArray
3
+ from typing import Tuple
4
+
5
+
6
+ ALL_BITS = np.uint64(-1)
7
+
8
+
9
+ def binary_search(arr: NDArray[np.uint64],
10
+ target: np.uint64,
11
+ mask: np.uint64 = ALL_BITS,
12
+ start: np.uint64 = np.uint64(0)) -> Tuple[np.uint64, bool]:
13
+ ...
14
+
15
+
16
+ def galloping_search(arr: NDArray[np.uint64],
17
+ target: np.uint64,
18
+ mask: np.uint64 = ALL_BITS,
19
+ start: np.uint64 = np.uint64(0)) -> Tuple[np.uint64, bool]:
20
+ ...
21
+
22
+
23
+ def count_odds(lhs: NDArray[np.uint64],
24
+ rhs: NDArray[np.uint64]) -> np.uint64:
25
+ ...
@@ -0,0 +1,140 @@
1
+ # These are modified versions of sortednp:
2
+ # https://gitlab.sauerburger.com/frank/sortednp
3
+ # cython: boundscheck=False
4
+ # cython: wraparound=False
5
+ # cython: initializedcheck=False
6
+ # cython: cdivision=True
7
+ # cython: nonecheck=False
8
+ # cython: language_level=3
9
+ cimport numpy as np
10
+ import numpy as np
11
+
12
+
13
+ # cimport snp_ops
14
+ # from snp_ops cimport _galloping_search, DTYPE_t, ALL_BITS
15
+ cimport searcharray.roaringish.snp_ops
16
+ from searcharray.roaringish.snp_ops cimport DTYPE_t
17
+
18
+ cdef DTYPE_t ALL_BITS = 0xFFFFFFFFFFFFFFFF
19
+ cdef void _binary_search(DTYPE_t[:] array,
20
+ DTYPE_t target,
21
+ DTYPE_t mask,
22
+ np.uint64_t* idx_out,
23
+ np.uint64_t len):
24
+ """Write to idx_out the index of the first element in array that is
25
+ greater or equal to target (masked)."""
26
+ cdef DTYPE_t value = array[idx_out[0]]
27
+ target &= mask
28
+
29
+ # If already at correct location or beyond
30
+ if target <= value & mask:
31
+ return
32
+
33
+ cdef np.intp_t i_right = len - 1 # is always GREATER OR EQUAL
34
+ cdef np.intp_t i_left = idx_out[0] # is always LESS than value
35
+
36
+ if array[i_right] & mask < target:
37
+ idx_out[0] = i_right
38
+ return # indicate target value too large
39
+
40
+ while i_left + 1 < i_right:
41
+ idx_out[0] = (i_right + i_left) // 2 # midpoint
42
+ value = array[idx_out[0]]
43
+
44
+ if target <= value & mask:
45
+ i_right = idx_out[0]
46
+ else:
47
+ i_left = idx_out[0]
48
+
49
+ idx_out[0] = i_right
50
+
51
+
52
+ # Python wrapper for binary search
53
+ def binary_search(np.ndarray[DTYPE_t, ndim=1] array,
54
+ DTYPE_t target,
55
+ DTYPE_t mask=ALL_BITS,
56
+ start=0):
57
+ cdef np.uint64_t i = start
58
+ cdef np.uint64_t len = array.shape[0]
59
+ _binary_search(array, target, mask, &i, len)
60
+ return i, (array[i] & mask) == (target & mask)
61
+
62
+
63
+ cdef inline void _galloping_search(DTYPE_t[:] array,
64
+ DTYPE_t target,
65
+ DTYPE_t mask,
66
+ np.uint64_t* idx_out,
67
+ np.uint64_t len):
68
+ cdef DTYPE_t value = array[idx_out[0]] & mask
69
+ target &= mask
70
+
71
+ # If already at correct location or beyond
72
+ if target <= value:
73
+ return
74
+
75
+ cdef np.intp_t delta = 1
76
+ cdef DTYPE_t end = len - 1
77
+ cdef np.intp_t i_prev = idx_out[0]
78
+
79
+ while value < target:
80
+ i_prev = idx_out[0]
81
+ idx_out[0] += delta
82
+ if len <= idx_out[0]:
83
+ # Gallop jump reached end of array.
84
+ idx_out[0] = end
85
+ value = array[idx_out[0]] & mask
86
+ break
87
+
88
+ value = array[idx_out[0]] & mask
89
+ # Increase step size.
90
+ delta *= 2
91
+
92
+ cdef np.intp_t i_right = idx_out[0] + 1 # Convert pointer position to length.
93
+ idx_out[0] = i_prev # This is the lower boundary and the active counter.
94
+
95
+ # i_prev ~ i_left to save a variable
96
+ #
97
+ # _binary_search(array, target, mask, i, higher)
98
+ # Inline binary search without the checks
99
+ # length is one past current posn
100
+ # left is i_prev
101
+ while i_prev + 1 < i_right :
102
+ idx_out[0] = (i_right + i_prev) // 2 # midpoint
103
+ value = array[idx_out[0]] & mask
104
+ if target <= value:
105
+ i_right = idx_out[0]
106
+ else:
107
+ i_prev = idx_out[0]
108
+ idx_out[0] = i_right
109
+
110
+
111
+ # This is just a baseline function for measuring how
112
+ # fast we can expect a naive collection
113
+ cdef _count_odds(np.ndarray[DTYPE_t, ndim=1] lhs, np.ndarray[DTYPE_t, ndim=1] rhs):
114
+ cdef int i = 0
115
+ cdef int count = 0
116
+ for i in range(rhs.shape[0]):
117
+ if (rhs[i] & 1) or (i < lhs.shape[0] and (lhs[i] & 1)):
118
+ count += 1
119
+ return count
120
+
121
+
122
+ def count_odds(np.ndarray[DTYPE_t, ndim=1] lhs, np.ndarray[DTYPE_t, ndim=1] rhs):
123
+ # Make sure lhs is smallest
124
+ if lhs.shape[0] > rhs.shape[0]:
125
+ lhs, rhs = rhs, lhs
126
+ return _count_odds(lhs, rhs)
127
+
128
+
129
+ def galloping_search(np.ndarray[DTYPE_t, ndim=1] array,
130
+ DTYPE_t target,
131
+ DTYPE_t mask=ALL_BITS,
132
+ start=0):
133
+ cdef np.uint64_t i = start
134
+ cdef np.uint64_t len = array.shape[0]
135
+ _galloping_search(array, target, mask, &i, len)
136
+ return i, (array[i] & mask) == (target & mask)
137
+
138
+
139
+ def _u64(lst) -> np.ndarray:
140
+ return np.array(lst, dtype=np.uint64)
@@ -0,0 +1,13 @@
1
+ # cython: boundscheck=False
2
+ # cython: wraparound=False
3
+ # cython: initializedcheck=False
4
+ # cython: cdivision=True
5
+ # cython: nonecheck=False
6
+ # cython: language_level=3
7
+
8
+ cdef extern from "stdint.h":
9
+ ctypedef unsigned long long uint64_t
10
+ ctypedef long long int64_t
11
+
12
+
13
+ ctypedef unsigned long long DTYPE_t
@@ -0,0 +1,330 @@
1
+ # cython: boundscheck=False
2
+ # cython: wraparound=False
3
+ # cython: initializedcheck=False
4
+ # cython: cdivision=True
5
+ # cython: nonecheck=False
6
+ # cython: language_level=3
7
+ """Utilities for computing spans for position aware search with slop > 0."""
8
+ cimport numpy as np
9
+ import numpy as np
10
+
11
+ cimport searcharray.roaringish.snp_ops
12
+ from searcharray.roaringish.snp_ops cimport DTYPE_t
13
+
14
+ cdef extern from *:
15
+ """
16
+ #if defined(_MSC_VER)
17
+ #include <intrin.h>
18
+ #pragma intrinsic(__popcnt64)
19
+ #pragma intrinsic(_BitScanForward64)
20
+ #pragma intrinsic(_BitScanReverse64)
21
+
22
+ static int popcountll(unsigned long long x) {
23
+ return __popcnt64(x);
24
+ }
25
+
26
+ static int ctzll(unsigned long long x) {
27
+ unsigned long index;
28
+ if (x == 0) return 64;
29
+ _BitScanForward64(&index, x);
30
+ return index;
31
+ }
32
+
33
+ static int clzll(unsigned long long x) {
34
+ unsigned long index;
35
+ if (x == 0) return 64;
36
+ _BitScanReverse64(&index, x);
37
+ return 63 - index;
38
+ }
39
+
40
+ #else
41
+ #include <stddef.h>
42
+ static int popcountll(unsigned long long x) {
43
+ return __builtin_popcountll(x);
44
+ }
45
+
46
+ static int ctzll(unsigned long long x) {
47
+ return __builtin_ctzll(x);
48
+ }
49
+
50
+ static int clzll(unsigned long long x) {
51
+ return __builtin_clzll(x);
52
+ }
53
+ #endif
54
+ """
55
+
56
+ cdef extern from *:
57
+ int popcountll(unsigned long long x)
58
+ int ctzll(unsigned long long x)
59
+ int clzll(unsigned long long x)
60
+
61
+
62
+ cdef _count_spans_of_slop(DTYPE_t[:] posns, DTYPE_t slop):
63
+ # DTYPE_t mask = 0xFFFFFFFFFFFFFFFF
64
+ for posn in posns:
65
+ if posn == 0:
66
+ return 0
67
+ return 1
68
+
69
+
70
+ cdef struct ActiveSpans:
71
+ DTYPE_t[512] terms
72
+ DTYPE_t[512] posns
73
+ np.int64_t[512] beg
74
+ np.int64_t[512] end
75
+ np.uint64_t cursor
76
+
77
+
78
+ cdef ActiveSpans _new_active_spans():
79
+ cdef ActiveSpans active_spans
80
+ active_spans.terms = np.zeros(512, dtype=np.uint64)
81
+ active_spans.posns = np.zeros(512, dtype=np.uint64)
82
+ active_spans.beg = np.zeros(512, dtype=np.int64)
83
+ active_spans.end = np.zeros(512, dtype=np.int64)
84
+ active_spans.cursor = 0
85
+ return active_spans
86
+
87
+
88
+ cdef void _clear_span(ActiveSpans* spans, DTYPE_t span_idx):
89
+ spans[0].terms[span_idx] = 0
90
+ spans[0].posns[span_idx] = 0
91
+ spans[0].beg[span_idx] = 0
92
+ spans[0].end[span_idx] = 0
93
+
94
+
95
+ cdef np.int64_t _span_width(ActiveSpans* spans, DTYPE_t span_idx):
96
+ return abs(spans[0].end[span_idx] - spans[0].beg[span_idx])
97
+
98
+
99
+ cdef DTYPE_t _consume_lsb(DTYPE_t* term):
100
+ """Get lowest set bit, clear it, and return the position it occured in."""
101
+ lsb = ctzll(term[0])
102
+ # Clear LSB
103
+ term[0] = (term[0] & (term[0] - 1))
104
+ return lsb
105
+
106
+
107
+ cdef DTYPE_t _posn_mask(np.int64_t curr_posn):
108
+ return 1 << (curr_posn % 64)
109
+
110
+
111
+ cdef DTYPE_t _num_terms(ActiveSpans* spans, DTYPE_t span_idx):
112
+ return popcountll(spans[0].terms[span_idx])
113
+
114
+
115
+ cdef DTYPE_t _num_posns(ActiveSpans* spans, DTYPE_t span_idx):
116
+ return popcountll(spans[0].posns[span_idx])
117
+
118
+
119
+ cdef bint _do_spans_overlap(ActiveSpans* spans_lhs, DTYPE_t span_idx_lhs,
120
+ ActiveSpans* spans_rhs, DTYPE_t span_idx_rhs):
121
+ return ((spans_lhs.beg[span_idx_lhs] <= spans_rhs.end[span_idx_rhs])
122
+ and (spans_lhs.end[span_idx_lhs] >= spans_rhs.beg[span_idx_rhs]))
123
+
124
+
125
+ cdef bint _is_span_complete(ActiveSpans* spans, DTYPE_t span_idx, DTYPE_t num_terms):
126
+ cdef DTYPE_t num_terms_visited = _num_terms(spans, span_idx)
127
+ cdef DTYPE_t num_posns_visited = _num_posns(spans, span_idx)
128
+ return (num_terms_visited == num_terms) or (num_posns_visited == num_terms)
129
+
130
+
131
+ cdef bint _is_span_unsalvagable(ActiveSpans* spans, DTYPE_t span_idx, DTYPE_t max_width):
132
+ return (spans[0].beg[span_idx] < spans[0].end[span_idx]) and (_span_width(spans, span_idx) > max_width)
133
+
134
+
135
+ cdef void _print_span(ActiveSpans* spans, DTYPE_t span_idx):
136
+ cdef np.uint64_t terms = spans[0].terms[span_idx]
137
+ cdef np.uint64_t posns = spans[0].posns[span_idx]
138
+ print(f"{span_idx}: term:{terms:b} posns:{posns:b} beg-end:{spans[0].beg[span_idx]}-{spans[0].end[span_idx]}")
139
+
140
+
141
+ cdef ActiveSpans _compact_spans(ActiveSpans* spans, DTYPE_t max_width):
142
+ """Copy only active spans into new spans."""
143
+ cdef ActiveSpans new_spans = _new_active_spans()
144
+ cdef span_idx = 0
145
+ for span_idx in range(spans[0].cursor):
146
+ if _span_width(spans, span_idx) > max_width:
147
+ continue
148
+ if _num_terms(spans, span_idx) > 0:
149
+ new_spans.terms[new_spans.cursor] = spans[0].terms[span_idx]
150
+ new_spans.posns[new_spans.cursor] = spans[0].posns[span_idx]
151
+ new_spans.beg[new_spans.cursor] = spans[0].beg[span_idx]
152
+ new_spans.end[new_spans.cursor] = spans[0].end[span_idx]
153
+ new_spans.cursor += 1
154
+ return new_spans
155
+
156
+
157
+ cdef ActiveSpans _collect_spans(ActiveSpans* spans, DTYPE_t num_terms, DTYPE_t max_width):
158
+ """Sort so shortest spans are first."""
159
+ # TODO - if spans were a heap, this might be faster
160
+ cdef span_idx = 0
161
+ cdef coll_span_idx = 0
162
+ cdef bint overlaps = False
163
+ cdef ActiveSpans collected_spans = _new_active_spans()
164
+ for span_idx in range(spans[0].cursor):
165
+ if _is_span_complete(spans, span_idx, num_terms) and _span_width(spans, span_idx) < max_width:
166
+ new_width = abs(spans[0].end[span_idx] - spans[0].beg[span_idx])
167
+ overlaps = False
168
+ for coll_span_idx in range(collected_spans.cursor):
169
+ if _do_spans_overlap(spans, span_idx,
170
+ &collected_spans, coll_span_idx):
171
+ coll_width = abs(collected_spans.end[coll_span_idx] - collected_spans.beg[coll_span_idx])
172
+ if new_width < coll_width:
173
+ # Replace
174
+ collected_spans.terms[coll_span_idx] = spans[0].terms[span_idx]
175
+ collected_spans.posns[coll_span_idx] = spans[0].posns[span_idx]
176
+ collected_spans.beg[coll_span_idx] = spans[0].beg[span_idx]
177
+ collected_spans.end[coll_span_idx] = spans[0].end[span_idx]
178
+ overlaps = True
179
+ break
180
+ if not overlaps:
181
+ collected_spans.terms[collected_spans.cursor] = spans[0].terms[span_idx]
182
+ collected_spans.posns[collected_spans.cursor] = spans[0].posns[span_idx]
183
+ collected_spans.beg[collected_spans.cursor] = spans[0].beg[span_idx]
184
+ collected_spans.end[collected_spans.cursor] = spans[0].end[span_idx]
185
+ collected_spans.cursor += 1
186
+ return collected_spans
187
+
188
+
189
+ cdef _span_freqs(DTYPE_t[:] posns, # Flattened all terms in one array
190
+ DTYPE_t[:] lengths,
191
+ phrase_freqs,
192
+ DTYPE_t slop,
193
+ DTYPE_t key_mask,
194
+ DTYPE_t header_mask,
195
+ DTYPE_t key_bits,
196
+ DTYPE_t lsb_bits):
197
+ """Get unscored spans, within 64 bits."""
198
+
199
+ cdef np.int64_t set_idx = 0
200
+ cdef np.int64_t curr_posn = 0
201
+ cdef DTYPE_t payload_mask = ~header_mask
202
+ cdef DTYPE_t payload_msb_mask = header_mask & ~key_mask
203
+
204
+ cdef ActiveSpans spans = _new_active_spans()
205
+
206
+ cdef np.uint64_t[:] curr_idx = np.zeros(64, dtype=np.uint64)
207
+ cdef np.uint64_t[:] sum_popcount = np.zeros(64, dtype=np.uint64)
208
+ cdef np.uint64_t curr_term_mask = 0
209
+ cdef np.uint64_t posn_mask = 0
210
+ cdef np.uint64_t num_terms = len(lengths) - 1
211
+ cdef np.uint64_t term_ord = 0
212
+ cdef np.uint64_t term = 0
213
+ cdef np.uint64_t curr_key = 0
214
+ cdef np.uint64_t last_key = 0
215
+ cdef np.uint64_t payload_base = 0
216
+ cdef np.uint64_t max_span_width = num_terms + slop
217
+ cdef ActiveSpans collected_spans
218
+ last_set_idx = 0
219
+ cdef bint full = False
220
+
221
+ for i in range(num_terms):
222
+ curr_idx[i] = lengths[i]
223
+
224
+ while curr_idx[0] < lengths[1]:
225
+ # Read each term up to the next doc
226
+ for term_ord in range(num_terms):
227
+ curr_key = ((posns[curr_idx[term_ord]] & key_mask) >> (64 - key_bits))
228
+ sum_popcount[term_ord] = 0
229
+ payload_base = 0
230
+ while curr_idx[term_ord] < lengths[term_ord+1]:
231
+ last_key = curr_key
232
+ term = posns[curr_idx[term_ord]]
233
+ payload_base = ((term & payload_msb_mask) >> lsb_bits) * lsb_bits
234
+ term &= payload_mask
235
+ curr_term_mask = 0x1 << term_ord
236
+ sum_popcount[term_ord] += popcountll(posns[curr_idx[term_ord]] & payload_mask)
237
+
238
+ # Consume every position into every possible span
239
+ while term != 0:
240
+ set_idx = _consume_lsb(&term)
241
+ curr_posn = set_idx + payload_base
242
+ posn_mask = _posn_mask(curr_posn)
243
+
244
+ spans.terms[spans.cursor] = curr_term_mask
245
+ spans.posns[spans.cursor] = posn_mask
246
+ spans.beg[spans.cursor] = curr_posn
247
+ spans.end[spans.cursor] = curr_posn
248
+
249
+ # Update existing spans
250
+ end = spans.cursor
251
+ spans.cursor += 1
252
+ for span_idx in range(end):
253
+ # Continue active spans
254
+ num_terms_visited = _num_terms(&spans, span_idx)
255
+ num_posns_visited = _num_posns(&spans, span_idx)
256
+ if num_terms_visited < num_terms and num_posns_visited == num_terms:
257
+ continue
258
+ spans.terms[span_idx] |= curr_term_mask
259
+ num_terms_visited_now = _num_terms(&spans, span_idx)
260
+ # New term
261
+ if num_terms_visited_now > num_terms_visited:
262
+ # Add position for new unique term
263
+ spans.posns[span_idx] |= posn_mask
264
+
265
+ new_unique_posns = _num_posns(&spans, span_idx)
266
+ proposed_width = abs(curr_posn - spans.beg[span_idx])
267
+ if (num_posns_visited == new_unique_posns) or proposed_width > max_span_width:
268
+ # Clear curr_term_mask and cancel this position, we've seen it before
269
+ spans.terms[span_idx] &= ~curr_term_mask
270
+ continue
271
+ if spans.cursor < 512:
272
+ spans.terms[spans.cursor] = spans.terms[span_idx]
273
+ spans.posns[spans.cursor] = (spans.posns[span_idx] & ~posn_mask)
274
+ spans.beg[spans.cursor] = spans.beg[span_idx]
275
+ spans.end[spans.cursor] = spans.end[span_idx]
276
+ spans.cursor += 1
277
+ full = False
278
+ else:
279
+ full = True
280
+
281
+ spans.end[span_idx] = curr_posn
282
+ span_width = _span_width(&spans, span_idx)
283
+ if span_width > max_span_width:
284
+ continue
285
+
286
+ if spans.cursor >= 512:
287
+ break
288
+ last_set_idx = set_idx
289
+ curr_idx[term_ord] += 1
290
+ if curr_idx[term_ord] < lengths[term_ord+1]:
291
+ curr_key = (posns[curr_idx[term_ord]] & key_mask) >> (64 - key_bits)
292
+ if spans.cursor >= 512:
293
+ spans = _compact_spans(&spans, max_span_width)
294
+ if spans.cursor >= 512:
295
+ # Give up
296
+ # Read until key change
297
+ for i in range(curr_idx[term_ord], lengths[term_ord+1]):
298
+ term = posns[i]
299
+ curr_key = (term & key_mask) >> (64 - key_bits)
300
+ if curr_key != last_key:
301
+ curr_idx[term_ord] = i
302
+ break
303
+ if curr_key != last_key:
304
+ break
305
+
306
+ if full:
307
+ min_popcount = 0
308
+ for term_ord in range(num_terms):
309
+ if min_popcount == 0 or sum_popcount[term_ord] < min_popcount:
310
+ min_popcount = sum_popcount[term_ord]
311
+ phrase_freqs[last_key] += min_popcount
312
+ else:
313
+ # All terms consumed for doc
314
+ collected_spans = _collect_spans(&spans, num_terms, max_span_width)
315
+ phrase_freqs[last_key] += collected_spans.cursor
316
+
317
+ # Reset
318
+ spans = _new_active_spans()
319
+ full = False
320
+
321
+
322
+ def span_search(np.ndarray[DTYPE_t, ndim=1] posns,
323
+ np.ndarray[DTYPE_t, ndim=1] lengths,
324
+ phrase_freqs, # dictionary
325
+ DTYPE_t slop,
326
+ DTYPE_t key_mask,
327
+ DTYPE_t header_mask,
328
+ DTYPE_t key_bits,
329
+ DTYPE_t lsb_bits):
330
+ _span_freqs(posns, lengths, phrase_freqs, slop, key_mask, header_mask, key_bits, lsb_bits)
@@ -0,0 +1,6 @@
1
+ import numpy as np
2
+ from numpy.typing import NDArray
3
+
4
+
5
+ def unique(arr: NDArray[np.uint64], rshift: np.uint64 = np.uint64(0)) -> np.ndarray:
6
+ ...