searcharray 0.0.15__tar.gz → 0.0.17__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {searcharray-0.0.15 → searcharray-0.0.17}/PKG-INFO +1 -2
  2. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/phrase/middle_out.py +30 -122
  3. searcharray-0.0.17/searcharray/phrase/scan_merge.py +75 -0
  4. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/postings.py +69 -204
  5. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/term_dict.py +14 -0
  6. searcharray-0.0.17/searcharray/utils/bitcount.py +30 -0
  7. searcharray-0.0.17/searcharray/utils/mat_set.py +143 -0
  8. searcharray-0.0.17/searcharray/utils/roaringish.py +139 -0
  9. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/utils/row_viewable_matrix.py +21 -9
  10. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/PKG-INFO +1 -2
  11. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/SOURCES.txt +3 -1
  12. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/requires.txt +0 -1
  13. {searcharray-0.0.15 → searcharray-0.0.17}/setup.py +2 -2
  14. {searcharray-0.0.15 → searcharray-0.0.17}/test/test_extension_array.py +10 -1
  15. {searcharray-0.0.15 → searcharray-0.0.17}/test/test_msmarco.py +19 -5
  16. {searcharray-0.0.15 → searcharray-0.0.17}/test/test_phrase_matches.py +3 -23
  17. {searcharray-0.0.15 → searcharray-0.0.17}/test/test_tmdb.py +29 -2
  18. {searcharray-0.0.15 → searcharray-0.0.17}/test/test_utils.py +11 -1
  19. searcharray-0.0.15/searcharray/phrase/scan_merge.py +0 -230
  20. searcharray-0.0.15/searcharray/phrase/wide_spans.py +0 -183
  21. {searcharray-0.0.15 → searcharray-0.0.17}/README.md +0 -0
  22. {searcharray-0.0.15 → searcharray-0.0.17}/pyproject.toml +0 -0
  23. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/__init__.py +0 -0
  24. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/phrase/__init__.py +0 -0
  25. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/phrase/padded_arrays.py +0 -0
  26. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/phrase/posn_diffs.py +0 -0
  27. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/stemmers.py +0 -0
  28. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray/utils/__init__.py +0 -0
  29. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/dependency_links.txt +0 -0
  30. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/entry_points.txt +0 -0
  31. {searcharray-0.0.15 → searcharray-0.0.17}/searcharray.egg-info/top_level.txt +0 -0
  32. {searcharray-0.0.15 → searcharray-0.0.17}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.15
3
+ Version: 0.0.17
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -21,7 +21,6 @@ Classifier: Programming Language :: Python :: 3 :: Only
21
21
  Requires-Python: >=3.7, <4
22
22
  Description-Content-Type: text/markdown
23
23
  Requires-Dist: pandas>=2.0.0
24
- Requires-Dist: scipy>=1.0.0
25
24
  Requires-Dist: sortednp
26
25
  Provides-Extra: dev
27
26
  Requires-Dist: check-manifest; extra == "dev"
@@ -8,10 +8,12 @@ https://colab.research.google.com/drive/10tIEkdlCE_1J_CcgEcV0jkLfBc-0H4am?authus
8
8
  import numpy as np
9
9
  import sortednp as snp
10
10
  from copy import deepcopy
11
- from typing import List, Optional
11
+ from typing import List
12
+ from searcharray.utils.roaringish import RoaringishEncoder, convert_keys
12
13
  import numbers
13
14
  import logging
14
- from time import perf_counter
15
+
16
+ from searcharray.utils.bitcount import bit_count64
15
17
 
16
18
 
17
19
  logger = logging.getLogger(__name__)
@@ -26,129 +28,21 @@ logger.addHandler(handler)
26
28
  logger.setLevel(logging.ERROR)
27
29
 
28
30
 
29
- DOC_ID_MASK = 0xFFFFFFF000000000
30
- DOC_ID_BITS = 28
31
- POSN_MSB_MASK = 0x0000000FFFFC0000
32
- POSN_MSB_BITS = 18
33
- POSN_LSB_MASK = 0x000000000003FFFF
34
- POSN_LSB_BITS = 18
35
-
36
- assert DOC_ID_BITS + POSN_MSB_BITS + POSN_LSB_BITS == 64
37
- assert DOC_ID_MASK | POSN_MSB_MASK | POSN_LSB_MASK == 0xFFFFFFFFFFFFFFFF
38
-
39
-
40
- def validate_posns(posns: np.ndarray):
41
- if not np.all(posns < 2**POSN_LSB_BITS):
42
- raise ValueError(f"Positions must be less than {2**POSN_LSB_BITS}")
43
-
44
-
45
- def encode_posns(posns: np.ndarray, doc_ids: Optional[np.ndarray] = None):
46
- """Pack a sorted array of positions into compact bit array.
47
-
48
- each returned array represents a single term, with doc_id as 32 MSBs
49
-
50
- | 32 MSBs | 16 LSBs | 16 LSBs |
51
- doc_id bits msbs posns
52
-
53
- for later easy intersection of 32+16 msbs, then checking for adjacent
54
- positions
55
-
56
- """
57
- validate_posns(posns)
58
- cols = posns // POSN_LSB_BITS # Header of bit to use
59
- cols = cols.astype(np.uint64) << POSN_MSB_BITS
60
- if doc_ids is not None:
61
- cols |= doc_ids.astype(np.uint64) << (64 - DOC_ID_BITS)
62
- values = posns % POSN_LSB_BITS # Value to encode
63
-
64
- change_indices = np.nonzero(np.diff(cols))[0] + 1
65
- change_indices = np.insert(change_indices, 0, 0)
66
-
67
- encoded = cols | (1 << values)
68
- if len(encoded) == 0:
69
- return encoded
70
- return np.bitwise_or.reduceat(encoded, change_indices)
71
-
72
-
73
- def decode_posns(encoded, get_doc_ids=True):
74
- start = perf_counter()
75
- doc_ids = (encoded & DOC_ID_MASK) >> (64 - DOC_ID_BITS)
76
- msbs = (encoded & POSN_MSB_MASK) >> POSN_MSB_BITS
77
- to_concat = []
78
- for bit in range(POSN_LSB_BITS):
79
- mask = 1 << bit
80
- lsbs = encoded & mask
81
- set_lsbs = (lsbs != 0)
82
- this_doc_ids = doc_ids[set_lsbs]
83
- posns = bit + (msbs[set_lsbs] * POSN_LSB_BITS)
84
- doc_with_posn = np.dstack([this_doc_ids, posns])[0]
85
- to_concat.append(doc_with_posn)
86
-
87
- logger.debug(f"loop took {perf_counter() - start:.2f} seconds")
88
- stacked = np.vstack(to_concat)
89
- # Sort by doc_id, then posn
90
- sorted_posns = stacked[np.lexsort((stacked[:, 1], stacked[:, 0]))]
91
- doc_ids, idx = np.unique(sorted_posns[:, 0], return_index=True)
92
- grouped = np.split(sorted_posns[:, 1], idx[1:])
93
- if get_doc_ids:
94
- as_list = list(zip(doc_ids, grouped))
95
- else:
96
- as_list = grouped
97
-
98
- logger.debug(f"groupby took {perf_counter() - start:.2f} seconds | got {len(as_list)}")
99
- return as_list
100
-
101
-
102
- def intersect_msbs(lhs: np.ndarray, rhs: np.ndarray):
103
- """Return the MSBs that are common to both lhs and rhs."""
104
- # common = np.intersect1d(lhs_msbs, rhs_msbs)
105
- _, (lhs_idx, rhs_idx) = snp.intersect(lhs >> POSN_LSB_BITS, rhs >> POSN_LSB_BITS, indices=True)
106
- # With large arrays np.isin becomes a bottleneck
107
- return lhs[lhs_idx], rhs[rhs_idx]
108
-
109
-
110
- def convert_doc_ids(doc_ids):
111
- if isinstance(doc_ids, numbers.Number):
112
- return np.asarray([doc_ids], dtype=np.uint64)
113
- elif isinstance(doc_ids, list):
114
- return np.asarray(doc_ids, dtype=np.uint64)
115
- elif isinstance(doc_ids, np.ndarray):
116
- return doc_ids.astype(np.uint64)
117
- elif isinstance(doc_ids, range) and len(doc_ids) > 0:
118
- # return np.asarray(doc_ids, dtype=np.uint64) # UNFORTUNATE COPY
119
- return np.arange(doc_ids[0], doc_ids[-1] + 1, dtype=np.uint64) + doc_ids[0]
120
- elif isinstance(doc_ids, range):
121
- return np.asarray([], dtype=np.uint64)
122
-
123
-
124
- def get_docs(encoded: np.ndarray, doc_ids: np.ndarray):
125
- """Get list of encoded that have positions in doc_ids."""
126
- doc_ids = convert_doc_ids(doc_ids)
127
- assert len(doc_ids.shape) == 1
128
- assert len(encoded.shape) == 1
129
- encoded_doc_ids = encoded.astype(np.uint64) >> (64 - DOC_ID_BITS)
130
- empty = doc_ids << (64 - DOC_ID_BITS)
131
- _, (idx_docs, idx_enc) = snp.intersect(doc_ids, encoded_doc_ids, indices=True,
132
- duplicates=snp.KEEP_MAX_N)
133
-
134
- found = encoded[idx_enc]
135
-
136
- merged = snp.merge(found, empty, duplicates=snp.DROP)
137
- return merged
31
+ encoder = RoaringishEncoder()
138
32
 
139
33
 
140
34
  def inner_bigram_match(lhs, rhs):
141
35
  """Count bigram matches between two encoded arrays."""
142
- lhs, rhs = intersect_msbs(lhs, rhs)
36
+ lhs, rhs = encoder.intersect(lhs, rhs)
143
37
  if len(lhs) != len(rhs):
144
38
  raise ValueError("Encoding error, MSBs apparently are duplicated among your encoded posn arrays.")
145
- rhs_next = (rhs & DOC_ID_MASK)
39
+ rhs_next = (rhs & encoder.key_mask)
146
40
  rhs_next_mask = np.zeros(len(rhs), dtype=bool)
147
41
  counts = []
148
42
  # With popcount soon to be in numpy, this could potentially
149
43
  # be simply a left shift of the RHS LSB poppcount, and and a popcount
150
44
  # to count the overlaps
151
- for bit in range(1, POSN_LSB_BITS - 1):
45
+ for bit in range(1, encoder.payload_lsb_bits - 1):
152
46
  lhs_mask = 1 << bit
153
47
  rhs_mask = 1 << (bit + 1)
154
48
  lhs_set = (lhs & lhs_mask) != 0
@@ -194,9 +88,9 @@ class PosnBitArrayBuilder:
194
88
  doc_ids = self.term_posn_doc_ids[term_id]
195
89
  if isinstance(doc_ids, list):
196
90
  doc_ids = np.asarray(doc_ids, dtype=np.uint32)
197
- encoded = encode_posns(doc_ids=doc_ids, posns=posns)
91
+ encoded = encoder.encode(keys=doc_ids, payload=posns)
198
92
  if check:
199
- decode_again = decode_posns(encoded)
93
+ decode_again = encoder.decode(encoded)
200
94
  docs_to_posns = dict(decode_again)
201
95
  doc_ids_again = []
202
96
  posns_again = []
@@ -264,10 +158,11 @@ class PosnBitArray:
264
158
  def slice(self, key):
265
159
  sliced_term_posns = {}
266
160
  doc_ids = index_range(self.doc_ids, key)
161
+ np_doc_ids = convert_keys(doc_ids)
267
162
  for term_id, posns in self.encoded_term_posns.items():
268
163
  encoded = self.encoded_term_posns[term_id]
269
164
  assert len(encoded.shape) == 1
270
- sliced_term_posns[term_id] = get_docs(encoded, doc_ids=doc_ids)
165
+ sliced_term_posns[term_id] = encoder.slice(encoded, keys=np_doc_ids)
271
166
 
272
167
  return PosnBitArray(sliced_term_posns, doc_ids=doc_ids)
273
168
 
@@ -289,8 +184,8 @@ class PosnBitArray:
289
184
  self.encoded_term_posns[term_id] = snp.merge(posns_self, posns_other)
290
185
 
291
186
  def doc_encoded_posns(self, term_id: int, doc_id: int) -> List:
292
- term_posns = get_docs(self.encoded_term_posns[term_id],
293
- doc_ids=doc_id)
187
+ term_posns = encoder.slice(self.encoded_term_posns[term_id],
188
+ keys=np.asarray([doc_id], dtype=np.uint64))
294
189
  return term_posns
295
190
 
296
191
  def positions(self, term_id: int, key) -> List:
@@ -300,15 +195,16 @@ class PosnBitArray:
300
195
  doc_ids = np.asarray([doc_ids])
301
196
 
302
197
  try:
303
- term_posns = get_docs(self.encoded_term_posns[term_id],
304
- doc_ids=doc_ids)
198
+ np_doc_ids = convert_keys(doc_ids)
199
+ term_posns = encoder.slice(self.encoded_term_posns[term_id],
200
+ keys=np_doc_ids)
305
201
  except KeyError:
306
202
  r_val = [np.array([], dtype=np.uint32) for doc_id in doc_ids]
307
203
  if len(r_val) == 1 and isinstance(key, numbers.Number):
308
204
  r_val = r_val[0]
309
205
  return r_val
310
206
 
311
- decoded = decode_posns(term_posns, get_doc_ids=True)
207
+ decoded = encoder.decode(encoded=term_posns, get_keys=True)
312
208
 
313
209
  if len(decoded) == 0:
314
210
  return np.array([], dtype=np.uint32)
@@ -328,6 +224,18 @@ class PosnBitArray:
328
224
  decs = decs[0]
329
225
  return decs
330
226
 
227
+ def termfreqs(self, term_id: int) -> np.ndarray:
228
+ """Count term freqs using unique positions."""
229
+ encoded = self.encoded_term_posns[term_id]
230
+ doc_ids = encoder.keys(encoded)
231
+ change_indices = np.nonzero(np.diff(doc_ids))[0]
232
+ change_indices = np.concatenate(([0], change_indices + 1))
233
+ posns = encoded & encoder.payload_lsb_mask
234
+ bit_counts = bit_count64(posns)
235
+
236
+ term_freqs = np.add.reduceat(bit_counts, change_indices)
237
+ return np.unique(doc_ids), term_freqs
238
+
331
239
  def insert(self, key, term_ids_to_posns, is_encoded=False):
332
240
  new_posns = PosnBitArrayBuilder()
333
241
  if is_encoded:
@@ -0,0 +1,75 @@
1
+ import numpy as np
2
+ from typing import List
3
+
4
+
5
+ def _self_adjs(prior_posns, next_posns):
6
+ """Given two arrays of positions, return the self adjacencies.
7
+
8
+ Prior is a subset of next, find all the locations in next in prior,
9
+ then get all the adjacent positions, returning a numpy array.
10
+ """
11
+ in1d = np.in1d(next_posns, prior_posns)
12
+ if len(in1d) == 0:
13
+ return np.array([])
14
+
15
+ # Where there's an intersection
16
+ start_indices = np.argwhere(in1d).flatten()
17
+ # The adjacent to the intersections
18
+ adj_indices = np.argwhere(in1d).flatten() + 1
19
+ adj_indices = adj_indices[adj_indices < len(next_posns)]
20
+
21
+ arr = np.union1d(next_posns[start_indices], next_posns[adj_indices])
22
+ arr.sort()
23
+ return arr
24
+
25
+
26
+ def scan_merge_ins(term_posns: List[np.ndarray],
27
+ phrase_freqs: np.ndarray, slop=1):
28
+ """Merge bigram, by bigram, using np.searchsorted to find if insert posns match slop.
29
+
30
+ See https://colab.research.google.com/drive/1EeqHYuCiqyptd-awS67Re78pqVdTfH4A
31
+ """
32
+ # Any identical terms, default to shitty algo for now
33
+ # if len(tokens) != len(set(tokens)):
34
+ # return self.phrase_freq_shitty(tokens, slop=slop)
35
+
36
+ # Iterate each phrase with its next term
37
+ prior_posns = term_posns[0]
38
+ for term_cnt, curr_posns in enumerate(term_posns[1:]):
39
+ assert len(prior_posns) == len(curr_posns)
40
+ bigram_freqs = np.zeros(len(curr_posns))
41
+ cont_posns = []
42
+ for idx in range(len(curr_posns)):
43
+ # Find insert position of every next term in prior term's positions
44
+ # Intuition:
45
+ # https://colab.research.google.com/drive/1EeqHYuCiqyptd-awS67Re78pqVdTfH4A
46
+ if len(prior_posns[idx]) == 0:
47
+ bigram_freqs[idx] = 0
48
+ cont_posns.append([])
49
+ continue
50
+ priors_in_self = _self_adjs(prior_posns[idx], curr_posns[idx])
51
+ takeaway = 0
52
+ satisfies_slop = None
53
+ cont_indices = None
54
+ # Different term
55
+ if len(priors_in_self) == 0:
56
+ ins_posns = np.searchsorted(prior_posns[idx], curr_posns[idx], side='right')
57
+ prior_adjacents = prior_posns[idx][ins_posns - 1]
58
+ adjacents = curr_posns[idx] - prior_adjacents
59
+ satisfies_slop = (adjacents <= slop) & ~(ins_posns == 0)
60
+ cont_indices = np.argwhere(satisfies_slop).flatten()
61
+ # Overlapping term
62
+ else:
63
+ adjacents = np.diff(priors_in_self)
64
+ satisfies_slop = adjacents <= slop
65
+ consecutive_slops = satisfies_slop[1:] & satisfies_slop[:-1]
66
+ sum_consecutive = np.sum(consecutive_slops)
67
+ takeaway = -np.floor_divide(sum_consecutive, -2) # ceiling divide
68
+ cont_indices = np.argwhere(satisfies_slop).flatten() + 1
69
+
70
+ bigram_freqs[idx] = np.sum(satisfies_slop) - takeaway
71
+ cont_posn = curr_posns[idx][cont_indices]
72
+ cont_posns.append(cont_posn)
73
+ phrase_freqs = bigram_freqs
74
+ prior_posns = cont_posns
75
+ return phrase_freqs