searcharray 0.0.73__cp313-cp313-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- searcharray/__init__.py +2 -0
- searcharray/bm25/__init__.py +1 -0
- searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
- searcharray/bm25/bm25.pyi +11 -0
- searcharray/bm25/bm25.pyx +41 -0
- searcharray/indexing.py +342 -0
- searcharray/phrase/__init__.py +0 -0
- searcharray/phrase/bigram_freqs.py +307 -0
- searcharray/phrase/memmap_arrays.py +208 -0
- searcharray/phrase/middle_out.py +553 -0
- searcharray/phrase/spans.py +187 -0
- searcharray/postings.py +708 -0
- searcharray/roaringish/__init__.py +7 -0
- searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/intersect.pyi +25 -0
- searcharray/roaringish/intersect.pyx +390 -0
- searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/merge.pyi +16 -0
- searcharray/roaringish/merge.pyx +232 -0
- searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/popcount.pyi +36 -0
- searcharray/roaringish/popcount.pyx +278 -0
- searcharray/roaringish/roaringish.py +298 -0
- searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/roaringish_ops.pyx +98 -0
- searcharray/roaringish/scatter_assign.h +31 -0
- searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/search.pyi +25 -0
- searcharray/roaringish/search.pyx +140 -0
- searcharray/roaringish/snp_ops.pxd +13 -0
- searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/spans.pyx +330 -0
- searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/unique.pyi +6 -0
- searcharray/roaringish/unique.pyx +145 -0
- searcharray/similarity.py +92 -0
- searcharray/solr.py +355 -0
- searcharray/term_dict.py +59 -0
- searcharray/utils/__init__.py +0 -0
- searcharray/utils/bitcount.py +34 -0
- searcharray/utils/mat_set.py +170 -0
- searcharray/utils/row_viewable_matrix.py +91 -0
- searcharray/utils/sort.py +45 -0
- searcharray-0.0.73.dist-info/METADATA +245 -0
- searcharray-0.0.73.dist-info/RECORD +48 -0
- searcharray-0.0.73.dist-info/WHEEL +5 -0
- searcharray-0.0.73.dist-info/entry_points.txt +2 -0
- searcharray-0.0.73.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# These are modified versions of sortednp:
|
|
2
|
+
# https://gitlab.sauerburger.com/frank/sortednp
|
|
3
|
+
# cython: boundscheck=False
|
|
4
|
+
# cython: wraparound=False
|
|
5
|
+
# cython: initializedcheck=False
|
|
6
|
+
# cython: cdivision=True
|
|
7
|
+
# cython: nonecheck=False
|
|
8
|
+
# cython: language_level=3
|
|
9
|
+
cimport numpy as np
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# example.pyx
|
|
14
|
+
cdef extern from "scatter_assign.h":
|
|
15
|
+
void scatter_naive(float *array,
|
|
16
|
+
const unsigned long long *indices,
|
|
17
|
+
const float *values, int n) nogil
|
|
18
|
+
|
|
19
|
+
cimport searcharray.roaringish.snp_ops
|
|
20
|
+
from searcharray.roaringish.snp_ops cimport DTYPE_t
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
cdef DTYPE_t ALL_BITS = 0xFFFFFFFFFFFFFFFF
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
cdef extern from *:
|
|
27
|
+
"""
|
|
28
|
+
#if defined(_MSC_VER)
|
|
29
|
+
#include <intrin.h>
|
|
30
|
+
#pragma intrinsic(__popcnt64)
|
|
31
|
+
static int popcountll(unsigned long long x) {
|
|
32
|
+
return __popcnt64(x);
|
|
33
|
+
}
|
|
34
|
+
#else
|
|
35
|
+
#include <stddef.h>
|
|
36
|
+
static int popcountll(unsigned long long x) {
|
|
37
|
+
return __builtin_popcountll(x);
|
|
38
|
+
}
|
|
39
|
+
#endif
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
cdef extern from *:
|
|
43
|
+
int popcountll(unsigned long long x)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
cdef _payload_slice(DTYPE_t[:] arr,
|
|
47
|
+
DTYPE_t payload_msb_mask,
|
|
48
|
+
DTYPE_t min_payload,
|
|
49
|
+
DTYPE_t max_payload):
|
|
50
|
+
cdef DTYPE_t[:] sliced = np.zeros(arr.shape[0], dtype=np.uint64)
|
|
51
|
+
cdef DTYPE_t* sliced_ptr = &sliced[0]
|
|
52
|
+
cdef DTYPE_t* arr_ptr = &arr[0]
|
|
53
|
+
|
|
54
|
+
while arr_ptr < &arr[arr.shape[0]]:
|
|
55
|
+
if (arr_ptr[0] & payload_msb_mask) >= min_payload and (arr_ptr[0] & payload_msb_mask) <= max_payload:
|
|
56
|
+
sliced_ptr[0] = arr_ptr[0]
|
|
57
|
+
sliced_ptr += 1
|
|
58
|
+
arr_ptr += 1
|
|
59
|
+
|
|
60
|
+
return sliced, sliced_ptr - &sliced[0]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def payload_slice(arr, payload_msb_mask,
|
|
64
|
+
min_payload=0, max_payload=0xFFFFFFFFFFFFFFFF):
|
|
65
|
+
cdef DTYPE_t[:] arr_view = arr
|
|
66
|
+
sliced, sliced_len = _payload_slice(arr_view, payload_msb_mask,
|
|
67
|
+
min_payload, max_payload)
|
|
68
|
+
return np.array(sliced[:sliced_len])
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
cdef void _as_dense_array(DTYPE_t[:] indices, # Its likely these indices are sorted, if that helps
|
|
72
|
+
float[:] values,
|
|
73
|
+
float[:] arr_out) nogil:
|
|
74
|
+
|
|
75
|
+
cdef DTYPE_t* indices_ptr = &indices[0]
|
|
76
|
+
cdef float* values_ptr = &values[0]
|
|
77
|
+
|
|
78
|
+
while indices_ptr < &indices[indices.shape[0]]:
|
|
79
|
+
arr_out[indices_ptr[0]] = values_ptr[0]
|
|
80
|
+
indices_ptr += 1
|
|
81
|
+
values_ptr += 1
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def as_dense(indices, values, size):
|
|
85
|
+
cdef float[:] arr_out = np.zeros(size, dtype=np.float32)
|
|
86
|
+
cdef DTYPE_t[:] indices_view = indices
|
|
87
|
+
cdef DTYPE_t len_indices = indices.shape[0]
|
|
88
|
+
cdef float[:] values_view = values
|
|
89
|
+
if len(indices) != len(values):
|
|
90
|
+
raise ValueError("indices and values must have the same length")
|
|
91
|
+
# _as_dense_array(indices_view, values_view,
|
|
92
|
+
# arr_out)
|
|
93
|
+
with nogil:
|
|
94
|
+
scatter_naive(&arr_out[0],
|
|
95
|
+
&indices_view[0],
|
|
96
|
+
&values_view[0],
|
|
97
|
+
len_indices)
|
|
98
|
+
return np.array(arr_out)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
#ifndef SCATTER_ASSIGN_H
|
|
2
|
+
#define SCATTER_ASSIGN_H
|
|
3
|
+
|
|
4
|
+
#include <stdio.h>
|
|
5
|
+
|
|
6
|
+
void scatter_naive(float *array,
|
|
7
|
+
const unsigned long long *indices,
|
|
8
|
+
const float *values,
|
|
9
|
+
int n) {
|
|
10
|
+
|
|
11
|
+
const unsigned long long *end = indices + n;
|
|
12
|
+
const unsigned long long *unroll_end = indices + (n / 10) * 10;
|
|
13
|
+
while (indices < unroll_end) {
|
|
14
|
+
array[*indices++] = *values++;
|
|
15
|
+
array[*indices++] = *values++;
|
|
16
|
+
array[*indices++] = *values++;
|
|
17
|
+
array[*indices++] = *values++;
|
|
18
|
+
array[*indices++] = *values++;
|
|
19
|
+
array[*indices++] = *values++;
|
|
20
|
+
array[*indices++] = *values++;
|
|
21
|
+
array[*indices++] = *values++;
|
|
22
|
+
array[*indices++] = *values++;
|
|
23
|
+
array[*indices++] = *values++;
|
|
24
|
+
}
|
|
25
|
+
while (indices < end) {
|
|
26
|
+
array[*indices++] = *values++;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
#endif // SCATTER_ASSIGN_H
|
|
Binary file
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from numpy.typing import NDArray
|
|
3
|
+
from typing import Tuple
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
ALL_BITS = np.uint64(-1)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def binary_search(arr: NDArray[np.uint64],
|
|
10
|
+
target: np.uint64,
|
|
11
|
+
mask: np.uint64 = ALL_BITS,
|
|
12
|
+
start: np.uint64 = np.uint64(0)) -> Tuple[np.uint64, bool]:
|
|
13
|
+
...
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def galloping_search(arr: NDArray[np.uint64],
|
|
17
|
+
target: np.uint64,
|
|
18
|
+
mask: np.uint64 = ALL_BITS,
|
|
19
|
+
start: np.uint64 = np.uint64(0)) -> Tuple[np.uint64, bool]:
|
|
20
|
+
...
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def count_odds(lhs: NDArray[np.uint64],
|
|
24
|
+
rhs: NDArray[np.uint64]) -> np.uint64:
|
|
25
|
+
...
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# These are modified versions of sortednp:
|
|
2
|
+
# https://gitlab.sauerburger.com/frank/sortednp
|
|
3
|
+
# cython: boundscheck=False
|
|
4
|
+
# cython: wraparound=False
|
|
5
|
+
# cython: initializedcheck=False
|
|
6
|
+
# cython: cdivision=True
|
|
7
|
+
# cython: nonecheck=False
|
|
8
|
+
# cython: language_level=3
|
|
9
|
+
cimport numpy as np
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# cimport snp_ops
|
|
14
|
+
# from snp_ops cimport _galloping_search, DTYPE_t, ALL_BITS
|
|
15
|
+
cimport searcharray.roaringish.snp_ops
|
|
16
|
+
from searcharray.roaringish.snp_ops cimport DTYPE_t
|
|
17
|
+
|
|
18
|
+
cdef DTYPE_t ALL_BITS = 0xFFFFFFFFFFFFFFFF
|
|
19
|
+
cdef void _binary_search(DTYPE_t[:] array,
|
|
20
|
+
DTYPE_t target,
|
|
21
|
+
DTYPE_t mask,
|
|
22
|
+
np.uint64_t* idx_out,
|
|
23
|
+
np.uint64_t len):
|
|
24
|
+
"""Write to idx_out the index of the first element in array that is
|
|
25
|
+
greater or equal to target (masked)."""
|
|
26
|
+
cdef DTYPE_t value = array[idx_out[0]]
|
|
27
|
+
target &= mask
|
|
28
|
+
|
|
29
|
+
# If already at correct location or beyond
|
|
30
|
+
if target <= value & mask:
|
|
31
|
+
return
|
|
32
|
+
|
|
33
|
+
cdef np.intp_t i_right = len - 1 # is always GREATER OR EQUAL
|
|
34
|
+
cdef np.intp_t i_left = idx_out[0] # is always LESS than value
|
|
35
|
+
|
|
36
|
+
if array[i_right] & mask < target:
|
|
37
|
+
idx_out[0] = i_right
|
|
38
|
+
return # indicate target value too large
|
|
39
|
+
|
|
40
|
+
while i_left + 1 < i_right:
|
|
41
|
+
idx_out[0] = (i_right + i_left) // 2 # midpoint
|
|
42
|
+
value = array[idx_out[0]]
|
|
43
|
+
|
|
44
|
+
if target <= value & mask:
|
|
45
|
+
i_right = idx_out[0]
|
|
46
|
+
else:
|
|
47
|
+
i_left = idx_out[0]
|
|
48
|
+
|
|
49
|
+
idx_out[0] = i_right
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# Python wrapper for binary search
|
|
53
|
+
def binary_search(np.ndarray[DTYPE_t, ndim=1] array,
|
|
54
|
+
DTYPE_t target,
|
|
55
|
+
DTYPE_t mask=ALL_BITS,
|
|
56
|
+
start=0):
|
|
57
|
+
cdef np.uint64_t i = start
|
|
58
|
+
cdef np.uint64_t len = array.shape[0]
|
|
59
|
+
_binary_search(array, target, mask, &i, len)
|
|
60
|
+
return i, (array[i] & mask) == (target & mask)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
cdef inline void _galloping_search(DTYPE_t[:] array,
|
|
64
|
+
DTYPE_t target,
|
|
65
|
+
DTYPE_t mask,
|
|
66
|
+
np.uint64_t* idx_out,
|
|
67
|
+
np.uint64_t len):
|
|
68
|
+
cdef DTYPE_t value = array[idx_out[0]] & mask
|
|
69
|
+
target &= mask
|
|
70
|
+
|
|
71
|
+
# If already at correct location or beyond
|
|
72
|
+
if target <= value:
|
|
73
|
+
return
|
|
74
|
+
|
|
75
|
+
cdef np.intp_t delta = 1
|
|
76
|
+
cdef DTYPE_t end = len - 1
|
|
77
|
+
cdef np.intp_t i_prev = idx_out[0]
|
|
78
|
+
|
|
79
|
+
while value < target:
|
|
80
|
+
i_prev = idx_out[0]
|
|
81
|
+
idx_out[0] += delta
|
|
82
|
+
if len <= idx_out[0]:
|
|
83
|
+
# Gallop jump reached end of array.
|
|
84
|
+
idx_out[0] = end
|
|
85
|
+
value = array[idx_out[0]] & mask
|
|
86
|
+
break
|
|
87
|
+
|
|
88
|
+
value = array[idx_out[0]] & mask
|
|
89
|
+
# Increase step size.
|
|
90
|
+
delta *= 2
|
|
91
|
+
|
|
92
|
+
cdef np.intp_t i_right = idx_out[0] + 1 # Convert pointer position to length.
|
|
93
|
+
idx_out[0] = i_prev # This is the lower boundary and the active counter.
|
|
94
|
+
|
|
95
|
+
# i_prev ~ i_left to save a variable
|
|
96
|
+
#
|
|
97
|
+
# _binary_search(array, target, mask, i, higher)
|
|
98
|
+
# Inline binary search without the checks
|
|
99
|
+
# length is one past current posn
|
|
100
|
+
# left is i_prev
|
|
101
|
+
while i_prev + 1 < i_right :
|
|
102
|
+
idx_out[0] = (i_right + i_prev) // 2 # midpoint
|
|
103
|
+
value = array[idx_out[0]] & mask
|
|
104
|
+
if target <= value:
|
|
105
|
+
i_right = idx_out[0]
|
|
106
|
+
else:
|
|
107
|
+
i_prev = idx_out[0]
|
|
108
|
+
idx_out[0] = i_right
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# This is just a baseline function for measuring how
|
|
112
|
+
# fast we can expect a naive collection
|
|
113
|
+
cdef _count_odds(np.ndarray[DTYPE_t, ndim=1] lhs, np.ndarray[DTYPE_t, ndim=1] rhs):
|
|
114
|
+
cdef int i = 0
|
|
115
|
+
cdef int count = 0
|
|
116
|
+
for i in range(rhs.shape[0]):
|
|
117
|
+
if (rhs[i] & 1) or (i < lhs.shape[0] and (lhs[i] & 1)):
|
|
118
|
+
count += 1
|
|
119
|
+
return count
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def count_odds(np.ndarray[DTYPE_t, ndim=1] lhs, np.ndarray[DTYPE_t, ndim=1] rhs):
|
|
123
|
+
# Make sure lhs is smallest
|
|
124
|
+
if lhs.shape[0] > rhs.shape[0]:
|
|
125
|
+
lhs, rhs = rhs, lhs
|
|
126
|
+
return _count_odds(lhs, rhs)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def galloping_search(np.ndarray[DTYPE_t, ndim=1] array,
|
|
130
|
+
DTYPE_t target,
|
|
131
|
+
DTYPE_t mask=ALL_BITS,
|
|
132
|
+
start=0):
|
|
133
|
+
cdef np.uint64_t i = start
|
|
134
|
+
cdef np.uint64_t len = array.shape[0]
|
|
135
|
+
_galloping_search(array, target, mask, &i, len)
|
|
136
|
+
return i, (array[i] & mask) == (target & mask)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _u64(lst) -> np.ndarray:
|
|
140
|
+
return np.array(lst, dtype=np.uint64)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# cython: boundscheck=False
|
|
2
|
+
# cython: wraparound=False
|
|
3
|
+
# cython: initializedcheck=False
|
|
4
|
+
# cython: cdivision=True
|
|
5
|
+
# cython: nonecheck=False
|
|
6
|
+
# cython: language_level=3
|
|
7
|
+
|
|
8
|
+
cdef extern from "stdint.h":
|
|
9
|
+
ctypedef unsigned long long uint64_t
|
|
10
|
+
ctypedef long long int64_t
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
ctypedef unsigned long long DTYPE_t
|
|
Binary file
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
# cython: boundscheck=False
|
|
2
|
+
# cython: wraparound=False
|
|
3
|
+
# cython: initializedcheck=False
|
|
4
|
+
# cython: cdivision=True
|
|
5
|
+
# cython: nonecheck=False
|
|
6
|
+
# cython: language_level=3
|
|
7
|
+
"""Utilities for computing spans for position aware search with slop > 0."""
|
|
8
|
+
cimport numpy as np
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
cimport searcharray.roaringish.snp_ops
|
|
12
|
+
from searcharray.roaringish.snp_ops cimport DTYPE_t
|
|
13
|
+
|
|
14
|
+
cdef extern from *:
|
|
15
|
+
"""
|
|
16
|
+
#if defined(_MSC_VER)
|
|
17
|
+
#include <intrin.h>
|
|
18
|
+
#pragma intrinsic(__popcnt64)
|
|
19
|
+
#pragma intrinsic(_BitScanForward64)
|
|
20
|
+
#pragma intrinsic(_BitScanReverse64)
|
|
21
|
+
|
|
22
|
+
static int popcountll(unsigned long long x) {
|
|
23
|
+
return __popcnt64(x);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
static int ctzll(unsigned long long x) {
|
|
27
|
+
unsigned long index;
|
|
28
|
+
if (x == 0) return 64;
|
|
29
|
+
_BitScanForward64(&index, x);
|
|
30
|
+
return index;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
static int clzll(unsigned long long x) {
|
|
34
|
+
unsigned long index;
|
|
35
|
+
if (x == 0) return 64;
|
|
36
|
+
_BitScanReverse64(&index, x);
|
|
37
|
+
return 63 - index;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
#else
|
|
41
|
+
#include <stddef.h>
|
|
42
|
+
static int popcountll(unsigned long long x) {
|
|
43
|
+
return __builtin_popcountll(x);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
static int ctzll(unsigned long long x) {
|
|
47
|
+
return __builtin_ctzll(x);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
static int clzll(unsigned long long x) {
|
|
51
|
+
return __builtin_clzll(x);
|
|
52
|
+
}
|
|
53
|
+
#endif
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
cdef extern from *:
|
|
57
|
+
int popcountll(unsigned long long x)
|
|
58
|
+
int ctzll(unsigned long long x)
|
|
59
|
+
int clzll(unsigned long long x)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
cdef _count_spans_of_slop(DTYPE_t[:] posns, DTYPE_t slop):
|
|
63
|
+
# DTYPE_t mask = 0xFFFFFFFFFFFFFFFF
|
|
64
|
+
for posn in posns:
|
|
65
|
+
if posn == 0:
|
|
66
|
+
return 0
|
|
67
|
+
return 1
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
cdef struct ActiveSpans:
|
|
71
|
+
DTYPE_t[512] terms
|
|
72
|
+
DTYPE_t[512] posns
|
|
73
|
+
np.int64_t[512] beg
|
|
74
|
+
np.int64_t[512] end
|
|
75
|
+
np.uint64_t cursor
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
cdef ActiveSpans _new_active_spans():
|
|
79
|
+
cdef ActiveSpans active_spans
|
|
80
|
+
active_spans.terms = np.zeros(512, dtype=np.uint64)
|
|
81
|
+
active_spans.posns = np.zeros(512, dtype=np.uint64)
|
|
82
|
+
active_spans.beg = np.zeros(512, dtype=np.int64)
|
|
83
|
+
active_spans.end = np.zeros(512, dtype=np.int64)
|
|
84
|
+
active_spans.cursor = 0
|
|
85
|
+
return active_spans
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
cdef void _clear_span(ActiveSpans* spans, DTYPE_t span_idx):
|
|
89
|
+
spans[0].terms[span_idx] = 0
|
|
90
|
+
spans[0].posns[span_idx] = 0
|
|
91
|
+
spans[0].beg[span_idx] = 0
|
|
92
|
+
spans[0].end[span_idx] = 0
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
cdef np.int64_t _span_width(ActiveSpans* spans, DTYPE_t span_idx):
|
|
96
|
+
return abs(spans[0].end[span_idx] - spans[0].beg[span_idx])
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
cdef DTYPE_t _consume_lsb(DTYPE_t* term):
|
|
100
|
+
"""Get lowest set bit, clear it, and return the position it occured in."""
|
|
101
|
+
lsb = ctzll(term[0])
|
|
102
|
+
# Clear LSB
|
|
103
|
+
term[0] = (term[0] & (term[0] - 1))
|
|
104
|
+
return lsb
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
cdef DTYPE_t _posn_mask(np.int64_t curr_posn):
|
|
108
|
+
return 1 << (curr_posn % 64)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
cdef DTYPE_t _num_terms(ActiveSpans* spans, DTYPE_t span_idx):
|
|
112
|
+
return popcountll(spans[0].terms[span_idx])
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
cdef DTYPE_t _num_posns(ActiveSpans* spans, DTYPE_t span_idx):
|
|
116
|
+
return popcountll(spans[0].posns[span_idx])
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
cdef bint _do_spans_overlap(ActiveSpans* spans_lhs, DTYPE_t span_idx_lhs,
|
|
120
|
+
ActiveSpans* spans_rhs, DTYPE_t span_idx_rhs):
|
|
121
|
+
return ((spans_lhs.beg[span_idx_lhs] <= spans_rhs.end[span_idx_rhs])
|
|
122
|
+
and (spans_lhs.end[span_idx_lhs] >= spans_rhs.beg[span_idx_rhs]))
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
cdef bint _is_span_complete(ActiveSpans* spans, DTYPE_t span_idx, DTYPE_t num_terms):
|
|
126
|
+
cdef DTYPE_t num_terms_visited = _num_terms(spans, span_idx)
|
|
127
|
+
cdef DTYPE_t num_posns_visited = _num_posns(spans, span_idx)
|
|
128
|
+
return (num_terms_visited == num_terms) or (num_posns_visited == num_terms)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
cdef bint _is_span_unsalvagable(ActiveSpans* spans, DTYPE_t span_idx, DTYPE_t max_width):
|
|
132
|
+
return (spans[0].beg[span_idx] < spans[0].end[span_idx]) and (_span_width(spans, span_idx) > max_width)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
cdef void _print_span(ActiveSpans* spans, DTYPE_t span_idx):
|
|
136
|
+
cdef np.uint64_t terms = spans[0].terms[span_idx]
|
|
137
|
+
cdef np.uint64_t posns = spans[0].posns[span_idx]
|
|
138
|
+
print(f"{span_idx}: term:{terms:b} posns:{posns:b} beg-end:{spans[0].beg[span_idx]}-{spans[0].end[span_idx]}")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
cdef ActiveSpans _compact_spans(ActiveSpans* spans, DTYPE_t max_width):
|
|
142
|
+
"""Copy only active spans into new spans."""
|
|
143
|
+
cdef ActiveSpans new_spans = _new_active_spans()
|
|
144
|
+
cdef span_idx = 0
|
|
145
|
+
for span_idx in range(spans[0].cursor):
|
|
146
|
+
if _span_width(spans, span_idx) > max_width:
|
|
147
|
+
continue
|
|
148
|
+
if _num_terms(spans, span_idx) > 0:
|
|
149
|
+
new_spans.terms[new_spans.cursor] = spans[0].terms[span_idx]
|
|
150
|
+
new_spans.posns[new_spans.cursor] = spans[0].posns[span_idx]
|
|
151
|
+
new_spans.beg[new_spans.cursor] = spans[0].beg[span_idx]
|
|
152
|
+
new_spans.end[new_spans.cursor] = spans[0].end[span_idx]
|
|
153
|
+
new_spans.cursor += 1
|
|
154
|
+
return new_spans
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
cdef ActiveSpans _collect_spans(ActiveSpans* spans, DTYPE_t num_terms, DTYPE_t max_width):
|
|
158
|
+
"""Sort so shortest spans are first."""
|
|
159
|
+
# TODO - if spans were a heap, this might be faster
|
|
160
|
+
cdef span_idx = 0
|
|
161
|
+
cdef coll_span_idx = 0
|
|
162
|
+
cdef bint overlaps = False
|
|
163
|
+
cdef ActiveSpans collected_spans = _new_active_spans()
|
|
164
|
+
for span_idx in range(spans[0].cursor):
|
|
165
|
+
if _is_span_complete(spans, span_idx, num_terms) and _span_width(spans, span_idx) < max_width:
|
|
166
|
+
new_width = abs(spans[0].end[span_idx] - spans[0].beg[span_idx])
|
|
167
|
+
overlaps = False
|
|
168
|
+
for coll_span_idx in range(collected_spans.cursor):
|
|
169
|
+
if _do_spans_overlap(spans, span_idx,
|
|
170
|
+
&collected_spans, coll_span_idx):
|
|
171
|
+
coll_width = abs(collected_spans.end[coll_span_idx] - collected_spans.beg[coll_span_idx])
|
|
172
|
+
if new_width < coll_width:
|
|
173
|
+
# Replace
|
|
174
|
+
collected_spans.terms[coll_span_idx] = spans[0].terms[span_idx]
|
|
175
|
+
collected_spans.posns[coll_span_idx] = spans[0].posns[span_idx]
|
|
176
|
+
collected_spans.beg[coll_span_idx] = spans[0].beg[span_idx]
|
|
177
|
+
collected_spans.end[coll_span_idx] = spans[0].end[span_idx]
|
|
178
|
+
overlaps = True
|
|
179
|
+
break
|
|
180
|
+
if not overlaps:
|
|
181
|
+
collected_spans.terms[collected_spans.cursor] = spans[0].terms[span_idx]
|
|
182
|
+
collected_spans.posns[collected_spans.cursor] = spans[0].posns[span_idx]
|
|
183
|
+
collected_spans.beg[collected_spans.cursor] = spans[0].beg[span_idx]
|
|
184
|
+
collected_spans.end[collected_spans.cursor] = spans[0].end[span_idx]
|
|
185
|
+
collected_spans.cursor += 1
|
|
186
|
+
return collected_spans
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
cdef _span_freqs(DTYPE_t[:] posns, # Flattened all terms in one array
|
|
190
|
+
DTYPE_t[:] lengths,
|
|
191
|
+
phrase_freqs,
|
|
192
|
+
DTYPE_t slop,
|
|
193
|
+
DTYPE_t key_mask,
|
|
194
|
+
DTYPE_t header_mask,
|
|
195
|
+
DTYPE_t key_bits,
|
|
196
|
+
DTYPE_t lsb_bits):
|
|
197
|
+
"""Get unscored spans, within 64 bits."""
|
|
198
|
+
|
|
199
|
+
cdef np.int64_t set_idx = 0
|
|
200
|
+
cdef np.int64_t curr_posn = 0
|
|
201
|
+
cdef DTYPE_t payload_mask = ~header_mask
|
|
202
|
+
cdef DTYPE_t payload_msb_mask = header_mask & ~key_mask
|
|
203
|
+
|
|
204
|
+
cdef ActiveSpans spans = _new_active_spans()
|
|
205
|
+
|
|
206
|
+
cdef np.uint64_t[:] curr_idx = np.zeros(64, dtype=np.uint64)
|
|
207
|
+
cdef np.uint64_t[:] sum_popcount = np.zeros(64, dtype=np.uint64)
|
|
208
|
+
cdef np.uint64_t curr_term_mask = 0
|
|
209
|
+
cdef np.uint64_t posn_mask = 0
|
|
210
|
+
cdef np.uint64_t num_terms = len(lengths) - 1
|
|
211
|
+
cdef np.uint64_t term_ord = 0
|
|
212
|
+
cdef np.uint64_t term = 0
|
|
213
|
+
cdef np.uint64_t curr_key = 0
|
|
214
|
+
cdef np.uint64_t last_key = 0
|
|
215
|
+
cdef np.uint64_t payload_base = 0
|
|
216
|
+
cdef np.uint64_t max_span_width = num_terms + slop
|
|
217
|
+
cdef ActiveSpans collected_spans
|
|
218
|
+
last_set_idx = 0
|
|
219
|
+
cdef bint full = False
|
|
220
|
+
|
|
221
|
+
for i in range(num_terms):
|
|
222
|
+
curr_idx[i] = lengths[i]
|
|
223
|
+
|
|
224
|
+
while curr_idx[0] < lengths[1]:
|
|
225
|
+
# Read each term up to the next doc
|
|
226
|
+
for term_ord in range(num_terms):
|
|
227
|
+
curr_key = ((posns[curr_idx[term_ord]] & key_mask) >> (64 - key_bits))
|
|
228
|
+
sum_popcount[term_ord] = 0
|
|
229
|
+
payload_base = 0
|
|
230
|
+
while curr_idx[term_ord] < lengths[term_ord+1]:
|
|
231
|
+
last_key = curr_key
|
|
232
|
+
term = posns[curr_idx[term_ord]]
|
|
233
|
+
payload_base = ((term & payload_msb_mask) >> lsb_bits) * lsb_bits
|
|
234
|
+
term &= payload_mask
|
|
235
|
+
curr_term_mask = 0x1 << term_ord
|
|
236
|
+
sum_popcount[term_ord] += popcountll(posns[curr_idx[term_ord]] & payload_mask)
|
|
237
|
+
|
|
238
|
+
# Consume every position into every possible span
|
|
239
|
+
while term != 0:
|
|
240
|
+
set_idx = _consume_lsb(&term)
|
|
241
|
+
curr_posn = set_idx + payload_base
|
|
242
|
+
posn_mask = _posn_mask(curr_posn)
|
|
243
|
+
|
|
244
|
+
spans.terms[spans.cursor] = curr_term_mask
|
|
245
|
+
spans.posns[spans.cursor] = posn_mask
|
|
246
|
+
spans.beg[spans.cursor] = curr_posn
|
|
247
|
+
spans.end[spans.cursor] = curr_posn
|
|
248
|
+
|
|
249
|
+
# Update existing spans
|
|
250
|
+
end = spans.cursor
|
|
251
|
+
spans.cursor += 1
|
|
252
|
+
for span_idx in range(end):
|
|
253
|
+
# Continue active spans
|
|
254
|
+
num_terms_visited = _num_terms(&spans, span_idx)
|
|
255
|
+
num_posns_visited = _num_posns(&spans, span_idx)
|
|
256
|
+
if num_terms_visited < num_terms and num_posns_visited == num_terms:
|
|
257
|
+
continue
|
|
258
|
+
spans.terms[span_idx] |= curr_term_mask
|
|
259
|
+
num_terms_visited_now = _num_terms(&spans, span_idx)
|
|
260
|
+
# New term
|
|
261
|
+
if num_terms_visited_now > num_terms_visited:
|
|
262
|
+
# Add position for new unique term
|
|
263
|
+
spans.posns[span_idx] |= posn_mask
|
|
264
|
+
|
|
265
|
+
new_unique_posns = _num_posns(&spans, span_idx)
|
|
266
|
+
proposed_width = abs(curr_posn - spans.beg[span_idx])
|
|
267
|
+
if (num_posns_visited == new_unique_posns) or proposed_width > max_span_width:
|
|
268
|
+
# Clear curr_term_mask and cancel this position, we've seen it before
|
|
269
|
+
spans.terms[span_idx] &= ~curr_term_mask
|
|
270
|
+
continue
|
|
271
|
+
if spans.cursor < 512:
|
|
272
|
+
spans.terms[spans.cursor] = spans.terms[span_idx]
|
|
273
|
+
spans.posns[spans.cursor] = (spans.posns[span_idx] & ~posn_mask)
|
|
274
|
+
spans.beg[spans.cursor] = spans.beg[span_idx]
|
|
275
|
+
spans.end[spans.cursor] = spans.end[span_idx]
|
|
276
|
+
spans.cursor += 1
|
|
277
|
+
full = False
|
|
278
|
+
else:
|
|
279
|
+
full = True
|
|
280
|
+
|
|
281
|
+
spans.end[span_idx] = curr_posn
|
|
282
|
+
span_width = _span_width(&spans, span_idx)
|
|
283
|
+
if span_width > max_span_width:
|
|
284
|
+
continue
|
|
285
|
+
|
|
286
|
+
if spans.cursor >= 512:
|
|
287
|
+
break
|
|
288
|
+
last_set_idx = set_idx
|
|
289
|
+
curr_idx[term_ord] += 1
|
|
290
|
+
if curr_idx[term_ord] < lengths[term_ord+1]:
|
|
291
|
+
curr_key = (posns[curr_idx[term_ord]] & key_mask) >> (64 - key_bits)
|
|
292
|
+
if spans.cursor >= 512:
|
|
293
|
+
spans = _compact_spans(&spans, max_span_width)
|
|
294
|
+
if spans.cursor >= 512:
|
|
295
|
+
# Give up
|
|
296
|
+
# Read until key change
|
|
297
|
+
for i in range(curr_idx[term_ord], lengths[term_ord+1]):
|
|
298
|
+
term = posns[i]
|
|
299
|
+
curr_key = (term & key_mask) >> (64 - key_bits)
|
|
300
|
+
if curr_key != last_key:
|
|
301
|
+
curr_idx[term_ord] = i
|
|
302
|
+
break
|
|
303
|
+
if curr_key != last_key:
|
|
304
|
+
break
|
|
305
|
+
|
|
306
|
+
if full:
|
|
307
|
+
min_popcount = 0
|
|
308
|
+
for term_ord in range(num_terms):
|
|
309
|
+
if min_popcount == 0 or sum_popcount[term_ord] < min_popcount:
|
|
310
|
+
min_popcount = sum_popcount[term_ord]
|
|
311
|
+
phrase_freqs[last_key] += min_popcount
|
|
312
|
+
else:
|
|
313
|
+
# All terms consumed for doc
|
|
314
|
+
collected_spans = _collect_spans(&spans, num_terms, max_span_width)
|
|
315
|
+
phrase_freqs[last_key] += collected_spans.cursor
|
|
316
|
+
|
|
317
|
+
# Reset
|
|
318
|
+
spans = _new_active_spans()
|
|
319
|
+
full = False
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def span_search(np.ndarray[DTYPE_t, ndim=1] posns,
|
|
323
|
+
np.ndarray[DTYPE_t, ndim=1] lengths,
|
|
324
|
+
phrase_freqs, # dictionary
|
|
325
|
+
DTYPE_t slop,
|
|
326
|
+
DTYPE_t key_mask,
|
|
327
|
+
DTYPE_t header_mask,
|
|
328
|
+
DTYPE_t key_bits,
|
|
329
|
+
DTYPE_t lsb_bits):
|
|
330
|
+
_span_freqs(posns, lengths, phrase_freqs, slop, key_mask, header_mask, key_bits, lsb_bits)
|
|
Binary file
|