searcharray 0.0.73__cp313-cp313-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- searcharray/__init__.py +2 -0
- searcharray/bm25/__init__.py +1 -0
- searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
- searcharray/bm25/bm25.pyi +11 -0
- searcharray/bm25/bm25.pyx +41 -0
- searcharray/indexing.py +342 -0
- searcharray/phrase/__init__.py +0 -0
- searcharray/phrase/bigram_freqs.py +307 -0
- searcharray/phrase/memmap_arrays.py +208 -0
- searcharray/phrase/middle_out.py +553 -0
- searcharray/phrase/spans.py +187 -0
- searcharray/postings.py +708 -0
- searcharray/roaringish/__init__.py +7 -0
- searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/intersect.pyi +25 -0
- searcharray/roaringish/intersect.pyx +390 -0
- searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/merge.pyi +16 -0
- searcharray/roaringish/merge.pyx +232 -0
- searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/popcount.pyi +36 -0
- searcharray/roaringish/popcount.pyx +278 -0
- searcharray/roaringish/roaringish.py +298 -0
- searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/roaringish_ops.pyx +98 -0
- searcharray/roaringish/scatter_assign.h +31 -0
- searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/search.pyi +25 -0
- searcharray/roaringish/search.pyx +140 -0
- searcharray/roaringish/snp_ops.pxd +13 -0
- searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/spans.pyx +330 -0
- searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
- searcharray/roaringish/unique.pyi +6 -0
- searcharray/roaringish/unique.pyx +145 -0
- searcharray/similarity.py +92 -0
- searcharray/solr.py +355 -0
- searcharray/term_dict.py +59 -0
- searcharray/utils/__init__.py +0 -0
- searcharray/utils/bitcount.py +34 -0
- searcharray/utils/mat_set.py +170 -0
- searcharray/utils/row_viewable_matrix.py +91 -0
- searcharray/utils/sort.py +45 -0
- searcharray-0.0.73.dist-info/METADATA +245 -0
- searcharray-0.0.73.dist-info/RECORD +48 -0
- searcharray-0.0.73.dist-info/WHEEL +5 -0
- searcharray-0.0.73.dist-info/entry_points.txt +2 -0
- searcharray-0.0.73.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Find span matches up to PAYLOAD_LSB bits span distance."""
|
|
2
|
+
# Colab notebooks
|
|
3
|
+
# https://colab.research.google.com/drive/1H4g7eeaYaHT1dJ5dmcmZUcwKNT-14tYs?authuser=1
|
|
4
|
+
# Chat GPT convo
|
|
5
|
+
# https://chat.openai.com/share/b97c61e6-95d0-40e7-a8cc-447dc1495314
|
|
6
|
+
import numpy as np
|
|
7
|
+
from typing import List
|
|
8
|
+
from searcharray.roaringish import intersect, adjacent, merge, span_search as r_span_search
|
|
9
|
+
# from searcharary.roaringish import intersect_all
|
|
10
|
+
from searcharray.roaringish.roaringish import RoaringishEncoder
|
|
11
|
+
from typing import Tuple
|
|
12
|
+
from collections import Counter
|
|
13
|
+
|
|
14
|
+
encoder = RoaringishEncoder()
|
|
15
|
+
|
|
16
|
+
# To not constantly type coerce
|
|
17
|
+
_64 = np.uint64(64)
|
|
18
|
+
_2 = np.uint64(2)
|
|
19
|
+
_1 = np.uint64(1)
|
|
20
|
+
_0 = np.uint64(0)
|
|
21
|
+
_neg1 = np.int64(-1)
|
|
22
|
+
_upper_bit = _1 << (encoder.payload_lsb_bits - _1)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# ---
|
|
26
|
+
# How to find max spans in roaringish
|
|
27
|
+
# MSB
|
|
28
|
+
# term A |10| | 12| 13| | 18 * Intersect A+B -> 10,12
|
|
29
|
+
# term B |10| | 12| | 14 15 17 | * RHS intersect -> 13 -> 14
|
|
30
|
+
# term C | 11 | 14
|
|
31
|
+
# term D 10 | 15
|
|
32
|
+
# term E | 11 12
|
|
33
|
+
# ------
|
|
34
|
+
#
|
|
35
|
+
#
|
|
36
|
+
# term A |10| 11 | 12| 13| * Intersect A+B -> 10,12
|
|
37
|
+
# term B |10| 11 | 12| 13| * Intersect A+B -> 10,12
|
|
38
|
+
# term C 9 | | 11 |
|
|
39
|
+
#
|
|
40
|
+
# ----
|
|
41
|
+
#
|
|
42
|
+
# Intersecting
|
|
43
|
+
# term A |10| | 12| 13 18
|
|
44
|
+
# term B |10| | 12| 14 15 17
|
|
45
|
+
# Adjacent
|
|
46
|
+
# term A 10 12| 13| 18
|
|
47
|
+
# term B 10 12| | 14 15 17 |
|
|
48
|
+
# Giving 9.5, 10.5 11.5, 12.5, 13.5 , 17.5
|
|
49
|
+
# ------
|
|
50
|
+
# Next pair, intersecting
|
|
51
|
+
# term B 10 12 | 14 | 15 17 * RHS intersect -> 13 -> 14
|
|
52
|
+
# term C 11 | 14 |
|
|
53
|
+
# Adjacent
|
|
54
|
+
# term B 10 12 14 15 17
|
|
55
|
+
# term C |11 14
|
|
56
|
+
# Giving 10.5, 13.5, 14.5
|
|
57
|
+
# w/ above 10.5, 13.5
|
|
58
|
+
# ------
|
|
59
|
+
#
|
|
60
|
+
#
|
|
61
|
+
# Finally: X X X X <- 4 words selected from each to find max span
|
|
62
|
+
#
|
|
63
|
+
# Does it work to compute any intersection, and then attach any adjecent that exist
|
|
64
|
+
#
|
|
65
|
+
#
|
|
66
|
+
|
|
67
|
+
def hdr(arr):
|
|
68
|
+
return encoder.header(arr) >> (_64 - encoder.header_bits)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _intersect_all(posns_encoded: List[np.ndarray]):
|
|
72
|
+
"""Intersect all encoded positions at roaringish MSBs."""
|
|
73
|
+
if len(posns_encoded) < 2:
|
|
74
|
+
raise ValueError("Need at least two positions to intersect")
|
|
75
|
+
last_lhs_headers = None
|
|
76
|
+
last_rhs_headers = None
|
|
77
|
+
curr = posns_encoded[0]
|
|
78
|
+
for term_idx, posns_next in enumerate(posns_encoded[1:]):
|
|
79
|
+
lhs_int_idx, rhs_int = intersect(curr, posns_next, mask=encoder.header_mask)
|
|
80
|
+
int_headers = encoder.header(curr[lhs_int_idx])
|
|
81
|
+
|
|
82
|
+
# What is adjacent on LHS / RHS of interserction
|
|
83
|
+
# Next to left
|
|
84
|
+
# 0 -> 2
|
|
85
|
+
# <- 1 (curr to right)
|
|
86
|
+
# 1 -> (keep
|
|
87
|
+
# 0 <- 2
|
|
88
|
+
curr_to_right, next_to_left = adjacent(curr, posns_next, mask=encoder.header_mask)
|
|
89
|
+
lhs_headers = merge(int_headers, posns_next[next_to_left])
|
|
90
|
+
rhs_headers = merge(int_headers, curr[curr_to_right])
|
|
91
|
+
next_to_right, curr_to_left = adjacent(posns_next, curr, mask=encoder.header_mask)
|
|
92
|
+
lhs_headers = merge(lhs_headers, curr[curr_to_left])
|
|
93
|
+
rhs_headers = merge(rhs_headers, posns_next[next_to_right])
|
|
94
|
+
|
|
95
|
+
if last_lhs_headers is not None:
|
|
96
|
+
lhs, _ = intersect(last_lhs_headers, lhs_headers, mask=encoder.header_mask)
|
|
97
|
+
rhs, _ = intersect(last_rhs_headers, rhs_headers, mask=encoder.header_mask)
|
|
98
|
+
last_lhs_headers = last_lhs_headers[lhs]
|
|
99
|
+
last_rhs_headers = last_rhs_headers[rhs]
|
|
100
|
+
else:
|
|
101
|
+
last_lhs_headers = lhs_headers
|
|
102
|
+
last_rhs_headers = rhs_headers
|
|
103
|
+
|
|
104
|
+
# Update by intersecting with current working lhs / rhs headers
|
|
105
|
+
|
|
106
|
+
assert last_rhs_headers is not None
|
|
107
|
+
assert last_lhs_headers is not None
|
|
108
|
+
to_rhs = last_rhs_headers + (_1 << (_64 - encoder.header_bits))
|
|
109
|
+
to_lhs = last_lhs_headers - (_1 << (_64 - encoder.header_bits))
|
|
110
|
+
all_headers = merge(to_rhs, to_lhs, drop_duplicates=True)
|
|
111
|
+
all_headers = merge(last_lhs_headers, all_headers, drop_duplicates=True)
|
|
112
|
+
all_headers = merge(last_rhs_headers, all_headers, drop_duplicates=True)
|
|
113
|
+
all_headers = all_headers & encoder.header_mask
|
|
114
|
+
# Get active MSBs now
|
|
115
|
+
# Merge all the rest to grab them
|
|
116
|
+
|
|
117
|
+
# Slice only matches at header MSBs
|
|
118
|
+
new_posns_encoded = posns_encoded.copy()
|
|
119
|
+
for i in range(len(posns_encoded)):
|
|
120
|
+
new_posns_encoded[i] = encoder.slice(posns_encoded[i], header=all_headers)
|
|
121
|
+
lengths = np.cumsum([0] + [len(posns) for posns in new_posns_encoded], dtype=np.uint64)
|
|
122
|
+
concatted = np.concatenate(new_posns_encoded, dtype=np.uint64)
|
|
123
|
+
return concatted, lengths
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# Picking up from intersections:
|
|
127
|
+
#
|
|
128
|
+
# Finally: X X X X X X <- 6 words selected from each to find max span
|
|
129
|
+
#
|
|
130
|
+
# Each X may be adjacent or not
|
|
131
|
+
#
|
|
132
|
+
# Looking at LSBs:
|
|
133
|
+
# MSB: 9 10 13 14 15 20
|
|
134
|
+
# Term A 01001 00000 00000 01000 00001 00100
|
|
135
|
+
# Term B 00010 00010 00010 00000 01000 00010
|
|
136
|
+
# Term C 10000 00001 00000 00001 00000 00001
|
|
137
|
+
#
|
|
138
|
+
# Limit to two adj words at a time (slop must be < LSB bits)
|
|
139
|
+
#
|
|
140
|
+
# MSB: 9 10 13 14 15 20
|
|
141
|
+
# Term A 01001 00000 00000 01000 00001 00100
|
|
142
|
+
# Term B 00010 00010 00010 00000 01000 00010
|
|
143
|
+
# Term C 10000 00001 00000 00001 00000 00001
|
|
144
|
+
# -----------
|
|
145
|
+
# (has span?)
|
|
146
|
+
# -----------
|
|
147
|
+
# (has span?)
|
|
148
|
+
# -----------
|
|
149
|
+
# (has span?)
|
|
150
|
+
# -----
|
|
151
|
+
# (has span?)
|
|
152
|
+
#
|
|
153
|
+
# Has span algo, really finding minimum sized spans
|
|
154
|
+
#
|
|
155
|
+
# Concat to single 64 bit word
|
|
156
|
+
# 0100100000
|
|
157
|
+
# 0001000010
|
|
158
|
+
# 1000000001
|
|
159
|
+
#
|
|
160
|
+
# Search w/ mask
|
|
161
|
+
# ffff000000
|
|
162
|
+
# And'd, now check if > 0:
|
|
163
|
+
# 0100000000 > 0 -> 1 0 0
|
|
164
|
+
# 0001000000 > 0 -> 1 1 0
|
|
165
|
+
# 1000000000 > 0 -> 1 1 0
|
|
166
|
+
#
|
|
167
|
+
# has_span (ANDED): 1
|
|
168
|
+
#
|
|
169
|
+
# Can we shrink the mask?
|
|
170
|
+
# If any hase leading or trailing 0s in mask, then shrink
|
|
171
|
+
def span_search(posns_encoded: List[np.ndarray],
|
|
172
|
+
slop: int) -> Tuple[np.ndarray, np.ndarray]:
|
|
173
|
+
"""Find span matches up to PAYLOAD_LSB bits span distance."""
|
|
174
|
+
# Find posns to check for span candidates
|
|
175
|
+
posns, lengths = _intersect_all(posns_encoded)
|
|
176
|
+
phrase_freqs: Counter = Counter()
|
|
177
|
+
|
|
178
|
+
# Populate phrase freqs with matches of slop
|
|
179
|
+
r_span_search(posns, lengths,
|
|
180
|
+
phrase_freqs,
|
|
181
|
+
slop,
|
|
182
|
+
encoder.key_mask,
|
|
183
|
+
encoder.header_mask,
|
|
184
|
+
encoder.key_bits,
|
|
185
|
+
encoder.payload_lsb_bits)
|
|
186
|
+
return (np.array(list(phrase_freqs.keys()), dtype=np.uint64),
|
|
187
|
+
np.array(list(phrase_freqs.values()), dtype=np.float32))
|