searcharray 0.0.73__cp313-cp313-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. searcharray/__init__.py +2 -0
  2. searcharray/bm25/__init__.py +1 -0
  3. searcharray/bm25/bm25.cp313-win_amd64.pyd +0 -0
  4. searcharray/bm25/bm25.pyi +11 -0
  5. searcharray/bm25/bm25.pyx +41 -0
  6. searcharray/indexing.py +342 -0
  7. searcharray/phrase/__init__.py +0 -0
  8. searcharray/phrase/bigram_freqs.py +307 -0
  9. searcharray/phrase/memmap_arrays.py +208 -0
  10. searcharray/phrase/middle_out.py +553 -0
  11. searcharray/phrase/spans.py +187 -0
  12. searcharray/postings.py +708 -0
  13. searcharray/roaringish/__init__.py +7 -0
  14. searcharray/roaringish/intersect.cp313-win_amd64.pyd +0 -0
  15. searcharray/roaringish/intersect.pyi +25 -0
  16. searcharray/roaringish/intersect.pyx +390 -0
  17. searcharray/roaringish/merge.cp313-win_amd64.pyd +0 -0
  18. searcharray/roaringish/merge.pyi +16 -0
  19. searcharray/roaringish/merge.pyx +232 -0
  20. searcharray/roaringish/popcount.cp313-win_amd64.pyd +0 -0
  21. searcharray/roaringish/popcount.pyi +36 -0
  22. searcharray/roaringish/popcount.pyx +278 -0
  23. searcharray/roaringish/roaringish.py +298 -0
  24. searcharray/roaringish/roaringish_ops.cp313-win_amd64.pyd +0 -0
  25. searcharray/roaringish/roaringish_ops.pyx +98 -0
  26. searcharray/roaringish/scatter_assign.h +31 -0
  27. searcharray/roaringish/search.cp313-win_amd64.pyd +0 -0
  28. searcharray/roaringish/search.pyi +25 -0
  29. searcharray/roaringish/search.pyx +140 -0
  30. searcharray/roaringish/snp_ops.pxd +13 -0
  31. searcharray/roaringish/spans.cp313-win_amd64.pyd +0 -0
  32. searcharray/roaringish/spans.pyx +330 -0
  33. searcharray/roaringish/unique.cp313-win_amd64.pyd +0 -0
  34. searcharray/roaringish/unique.pyi +6 -0
  35. searcharray/roaringish/unique.pyx +145 -0
  36. searcharray/similarity.py +92 -0
  37. searcharray/solr.py +355 -0
  38. searcharray/term_dict.py +59 -0
  39. searcharray/utils/__init__.py +0 -0
  40. searcharray/utils/bitcount.py +34 -0
  41. searcharray/utils/mat_set.py +170 -0
  42. searcharray/utils/row_viewable_matrix.py +91 -0
  43. searcharray/utils/sort.py +45 -0
  44. searcharray-0.0.73.dist-info/METADATA +245 -0
  45. searcharray-0.0.73.dist-info/RECORD +48 -0
  46. searcharray-0.0.73.dist-info/WHEEL +5 -0
  47. searcharray-0.0.73.dist-info/entry_points.txt +2 -0
  48. searcharray-0.0.73.dist-info/top_level.txt +1 -0
@@ -0,0 +1,187 @@
1
+ """Find span matches up to PAYLOAD_LSB bits span distance."""
2
+ # Colab notebooks
3
+ # https://colab.research.google.com/drive/1H4g7eeaYaHT1dJ5dmcmZUcwKNT-14tYs?authuser=1
4
+ # Chat GPT convo
5
+ # https://chat.openai.com/share/b97c61e6-95d0-40e7-a8cc-447dc1495314
6
+ import numpy as np
7
+ from typing import List
8
+ from searcharray.roaringish import intersect, adjacent, merge, span_search as r_span_search
9
+ # from searcharary.roaringish import intersect_all
10
+ from searcharray.roaringish.roaringish import RoaringishEncoder
11
+ from typing import Tuple
12
+ from collections import Counter
13
+
14
+ encoder = RoaringishEncoder()
15
+
16
+ # To not constantly type coerce
17
+ _64 = np.uint64(64)
18
+ _2 = np.uint64(2)
19
+ _1 = np.uint64(1)
20
+ _0 = np.uint64(0)
21
+ _neg1 = np.int64(-1)
22
+ _upper_bit = _1 << (encoder.payload_lsb_bits - _1)
23
+
24
+
25
+ # ---
26
+ # How to find max spans in roaringish
27
+ # MSB
28
+ # term A |10| | 12| 13| | 18 * Intersect A+B -> 10,12
29
+ # term B |10| | 12| | 14 15 17 | * RHS intersect -> 13 -> 14
30
+ # term C | 11 | 14
31
+ # term D 10 | 15
32
+ # term E | 11 12
33
+ # ------
34
+ #
35
+ #
36
+ # term A |10| 11 | 12| 13| * Intersect A+B -> 10,12
37
+ # term B |10| 11 | 12| 13| * Intersect A+B -> 10,12
38
+ # term C 9 | | 11 |
39
+ #
40
+ # ----
41
+ #
42
+ # Intersecting
43
+ # term A |10| | 12| 13 18
44
+ # term B |10| | 12| 14 15 17
45
+ # Adjacent
46
+ # term A 10 12| 13| 18
47
+ # term B 10 12| | 14 15 17 |
48
+ # Giving 9.5, 10.5 11.5, 12.5, 13.5 , 17.5
49
+ # ------
50
+ # Next pair, intersecting
51
+ # term B 10 12 | 14 | 15 17 * RHS intersect -> 13 -> 14
52
+ # term C 11 | 14 |
53
+ # Adjacent
54
+ # term B 10 12 14 15 17
55
+ # term C |11 14
56
+ # Giving 10.5, 13.5, 14.5
57
+ # w/ above 10.5, 13.5
58
+ # ------
59
+ #
60
+ #
61
+ # Finally: X X X X <- 4 words selected from each to find max span
62
+ #
63
+ # Does it work to compute any intersection, and then attach any adjecent that exist
64
+ #
65
+ #
66
+
67
+ def hdr(arr):
68
+ return encoder.header(arr) >> (_64 - encoder.header_bits)
69
+
70
+
71
+ def _intersect_all(posns_encoded: List[np.ndarray]):
72
+ """Intersect all encoded positions at roaringish MSBs."""
73
+ if len(posns_encoded) < 2:
74
+ raise ValueError("Need at least two positions to intersect")
75
+ last_lhs_headers = None
76
+ last_rhs_headers = None
77
+ curr = posns_encoded[0]
78
+ for term_idx, posns_next in enumerate(posns_encoded[1:]):
79
+ lhs_int_idx, rhs_int = intersect(curr, posns_next, mask=encoder.header_mask)
80
+ int_headers = encoder.header(curr[lhs_int_idx])
81
+
82
+ # What is adjacent on LHS / RHS of interserction
83
+ # Next to left
84
+ # 0 -> 2
85
+ # <- 1 (curr to right)
86
+ # 1 -> (keep
87
+ # 0 <- 2
88
+ curr_to_right, next_to_left = adjacent(curr, posns_next, mask=encoder.header_mask)
89
+ lhs_headers = merge(int_headers, posns_next[next_to_left])
90
+ rhs_headers = merge(int_headers, curr[curr_to_right])
91
+ next_to_right, curr_to_left = adjacent(posns_next, curr, mask=encoder.header_mask)
92
+ lhs_headers = merge(lhs_headers, curr[curr_to_left])
93
+ rhs_headers = merge(rhs_headers, posns_next[next_to_right])
94
+
95
+ if last_lhs_headers is not None:
96
+ lhs, _ = intersect(last_lhs_headers, lhs_headers, mask=encoder.header_mask)
97
+ rhs, _ = intersect(last_rhs_headers, rhs_headers, mask=encoder.header_mask)
98
+ last_lhs_headers = last_lhs_headers[lhs]
99
+ last_rhs_headers = last_rhs_headers[rhs]
100
+ else:
101
+ last_lhs_headers = lhs_headers
102
+ last_rhs_headers = rhs_headers
103
+
104
+ # Update by intersecting with current working lhs / rhs headers
105
+
106
+ assert last_rhs_headers is not None
107
+ assert last_lhs_headers is not None
108
+ to_rhs = last_rhs_headers + (_1 << (_64 - encoder.header_bits))
109
+ to_lhs = last_lhs_headers - (_1 << (_64 - encoder.header_bits))
110
+ all_headers = merge(to_rhs, to_lhs, drop_duplicates=True)
111
+ all_headers = merge(last_lhs_headers, all_headers, drop_duplicates=True)
112
+ all_headers = merge(last_rhs_headers, all_headers, drop_duplicates=True)
113
+ all_headers = all_headers & encoder.header_mask
114
+ # Get active MSBs now
115
+ # Merge all the rest to grab them
116
+
117
+ # Slice only matches at header MSBs
118
+ new_posns_encoded = posns_encoded.copy()
119
+ for i in range(len(posns_encoded)):
120
+ new_posns_encoded[i] = encoder.slice(posns_encoded[i], header=all_headers)
121
+ lengths = np.cumsum([0] + [len(posns) for posns in new_posns_encoded], dtype=np.uint64)
122
+ concatted = np.concatenate(new_posns_encoded, dtype=np.uint64)
123
+ return concatted, lengths
124
+
125
+
126
+ # Picking up from intersections:
127
+ #
128
+ # Finally: X X X X X X <- 6 words selected from each to find max span
129
+ #
130
+ # Each X may be adjacent or not
131
+ #
132
+ # Looking at LSBs:
133
+ # MSB: 9 10 13 14 15 20
134
+ # Term A 01001 00000 00000 01000 00001 00100
135
+ # Term B 00010 00010 00010 00000 01000 00010
136
+ # Term C 10000 00001 00000 00001 00000 00001
137
+ #
138
+ # Limit to two adj words at a time (slop must be < LSB bits)
139
+ #
140
+ # MSB: 9 10 13 14 15 20
141
+ # Term A 01001 00000 00000 01000 00001 00100
142
+ # Term B 00010 00010 00010 00000 01000 00010
143
+ # Term C 10000 00001 00000 00001 00000 00001
144
+ # -----------
145
+ # (has span?)
146
+ # -----------
147
+ # (has span?)
148
+ # -----------
149
+ # (has span?)
150
+ # -----
151
+ # (has span?)
152
+ #
153
+ # Has span algo, really finding minimum sized spans
154
+ #
155
+ # Concat to single 64 bit word
156
+ # 0100100000
157
+ # 0001000010
158
+ # 1000000001
159
+ #
160
+ # Search w/ mask
161
+ # ffff000000
162
+ # And'd, now check if > 0:
163
+ # 0100000000 > 0 -> 1 0 0
164
+ # 0001000000 > 0 -> 1 1 0
165
+ # 1000000000 > 0 -> 1 1 0
166
+ #
167
+ # has_span (ANDED): 1
168
+ #
169
+ # Can we shrink the mask?
170
+ # If any hase leading or trailing 0s in mask, then shrink
171
+ def span_search(posns_encoded: List[np.ndarray],
172
+ slop: int) -> Tuple[np.ndarray, np.ndarray]:
173
+ """Find span matches up to PAYLOAD_LSB bits span distance."""
174
+ # Find posns to check for span candidates
175
+ posns, lengths = _intersect_all(posns_encoded)
176
+ phrase_freqs: Counter = Counter()
177
+
178
+ # Populate phrase freqs with matches of slop
179
+ r_span_search(posns, lengths,
180
+ phrase_freqs,
181
+ slop,
182
+ encoder.key_mask,
183
+ encoder.header_mask,
184
+ encoder.key_bits,
185
+ encoder.payload_lsb_bits)
186
+ return (np.array(list(phrase_freqs.keys()), dtype=np.uint64),
187
+ np.array(list(phrase_freqs.values()), dtype=np.float32))