searcharray 0.0.58__tar.gz → 0.0.60__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {searcharray-0.0.58 → searcharray-0.0.60}/PKG-INFO +7 -6
  2. {searcharray-0.0.58 → searcharray-0.0.60}/README.md +6 -5
  3. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/phrase/bigram_freqs.py +9 -10
  4. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/phrase/middle_out.py +2 -6
  5. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/phrase/spans.py +73 -22
  6. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/postings.py +5 -1
  7. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/__init__.py +1 -1
  8. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/intersect.c +216 -160
  9. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/intersect.pyx +1 -1
  10. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/merge.c +149 -149
  11. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/popcount.c +1509 -819
  12. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/popcount.pyi +18 -0
  13. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/popcount.pyx +43 -0
  14. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/roaringish.py +10 -0
  15. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/roaringish_ops.c +149 -149
  16. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/search.c +149 -149
  17. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/spans.c +10817 -6953
  18. searcharray-0.0.60/searcharray/roaringish/spans.pyx +266 -0
  19. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/unique.c +149 -149
  20. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/solr.py +12 -6
  21. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray.egg-info/PKG-INFO +7 -6
  22. {searcharray-0.0.58 → searcharray-0.0.60}/setup.py +1 -1
  23. {searcharray-0.0.58 → searcharray-0.0.60}/test/test_msmarco.py +33 -1
  24. {searcharray-0.0.58 → searcharray-0.0.60}/test/test_phrase_matches.py +73 -1
  25. searcharray-0.0.60/test/test_slop_matches.py +88 -0
  26. {searcharray-0.0.58 → searcharray-0.0.60}/test/test_tmdb.py +13 -3
  27. {searcharray-0.0.58 → searcharray-0.0.60}/test/test_utils.py +1 -0
  28. searcharray-0.0.58/searcharray/roaringish/spans.pyx +0 -77
  29. searcharray-0.0.58/test/test_slop_matches.py +0 -52
  30. {searcharray-0.0.58 → searcharray-0.0.60}/pyproject.toml +0 -0
  31. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/__init__.py +0 -0
  32. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/indexing.py +0 -0
  33. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/phrase/__init__.py +0 -0
  34. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/intersect.pyi +0 -0
  35. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/merge.pyi +0 -0
  36. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/merge.pyx +0 -0
  37. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/roaringish_ops.pyx +0 -0
  38. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/search.pyi +0 -0
  39. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/search.pyx +0 -0
  40. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/snp_ops.pxd +0 -0
  41. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/unique.pyi +0 -0
  42. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/roaringish/unique.pyx +0 -0
  43. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/similarity.py +0 -0
  44. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/term_dict.py +0 -0
  45. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/utils/__init__.py +0 -0
  46. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/utils/bitcount.py +0 -0
  47. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/utils/mat_set.py +0 -0
  48. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/utils/row_viewable_matrix.py +0 -0
  49. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray/utils/sort.py +0 -0
  50. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray.egg-info/SOURCES.txt +0 -0
  51. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray.egg-info/dependency_links.txt +0 -0
  52. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray.egg-info/entry_points.txt +0 -0
  53. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray.egg-info/requires.txt +0 -0
  54. {searcharray-0.0.58 → searcharray-0.0.60}/searcharray.egg-info/top_level.txt +0 -0
  55. {searcharray-0.0.58 → searcharray-0.0.60}/setup.cfg +0 -0
  56. {searcharray-0.0.58 → searcharray-0.0.60}/test/test_bitcount64.py +0 -0
  57. {searcharray-0.0.58 → searcharray-0.0.60}/test/test_extension_array.py +0 -0
  58. {searcharray-0.0.58 → searcharray-0.0.60}/test/test_minmax_posns.py +0 -0
  59. {searcharray-0.0.58 → searcharray-0.0.60}/test/test_search.py +0 -0
  60. {searcharray-0.0.58 → searcharray-0.0.60}/test/test_similarity.py +0 -0
  61. {searcharray-0.0.58 → searcharray-0.0.60}/test/test_snp_ops.py +0 -0
  62. {searcharray-0.0.58 → searcharray-0.0.60}/test/test_solr.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.58
3
+ Version: 0.0.60
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -36,13 +36,14 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
36
36
  Think Lucene, but as a Pandas column.
37
37
 
38
38
  ```python
39
- In[2]: from searcharray import SearchArray
39
+ from searcharray import SearchArray
40
+ import pandas as pd
40
41
 
41
- In[3]: df['title_indexed'] = SearchArray.index(df['title'])
42
- np.sort(df['title_indexed'].array.score('Cat'))
42
+ df['title_indexed'] = SearchArray.index(df['title'])
43
+ np.sort(df['title_indexed'].array.score('Cat'))
43
44
 
44
- Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
45
- 15.84568033, 15.84568033])
45
+ > BM25 scores:
46
+ > array([ 0. , 0. , 0. , ..., 15.84568033, 15.84568033, 15.84568033])
46
47
  ```
47
48
 
48
49
  ## Docs | Guide
@@ -7,13 +7,14 @@ SearchArray turns Pandas string columns into a term index. It alows efficient BM
7
7
  Think Lucene, but as a Pandas column.
8
8
 
9
9
  ```python
10
- In[2]: from searcharray import SearchArray
10
+ from searcharray import SearchArray
11
+ import pandas as pd
11
12
 
12
- In[3]: df['title_indexed'] = SearchArray.index(df['title'])
13
- np.sort(df['title_indexed'].array.score('Cat'))
13
+ df['title_indexed'] = SearchArray.index(df['title'])
14
+ np.sort(df['title_indexed'].array.score('Cat'))
14
15
 
15
- Out[3]: array([ 0. , 0. , 0. , ..., 15.84568033,
16
- 15.84568033, 15.84568033])
16
+ > BM25 scores:
17
+ > array([ 0. , 0. , 0. , ..., 15.84568033, 15.84568033, 15.84568033])
17
18
  ```
18
19
 
19
20
  ## Docs | Guide
@@ -5,7 +5,7 @@ from searcharray.roaringish import RoaringishEncoder
5
5
  import logging
6
6
  from enum import Enum
7
7
 
8
- from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at
8
+ from searcharray.roaringish import intersect, popcount64, merge, popcount_reduce_at, key_sum_over
9
9
 
10
10
 
11
11
  logger = logging.getLogger(__name__)
@@ -82,7 +82,8 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
82
82
  adj_count = encoder.payload_lsb(overlap)
83
83
  adjacents = popcount64(adj_count).view(np.int64)
84
84
 
85
- phrase_freqs[lhs_doc_ids] += _adj_to_phrase_freq(overlap, adjacents)
85
+ adjusted = _adj_to_phrase_freq(overlap, adjacents).astype(np.uint64)
86
+ key_sum_over(lhs_doc_ids, adjusted, phrase_freqs)
86
87
  # Continue with ?? ?? foo foo
87
88
  # term_int without lsbs
88
89
  term_int_msbs = lhs_int & ~encoder.payload_lsb_mask
@@ -90,12 +91,13 @@ def _inner_bigram_same_term(lhs_int: np.ndarray, rhs_int: np.ndarray,
90
91
  lhs_cont = None
91
92
  # rhs_cont = term_int_msbs | encoder.payload_lsb(rhs_shift)
92
93
  rhs_cont = encoder.payload_lsb(rhs_shift & rhs_int) | term_int_msbs
93
-
94
94
  lhs_cont = term_int_msbs | encoder.payload_lsb(lhs_int & (lhs_int >> _1))
95
+
95
96
  if cont not in [Continuation.RHS, Continuation.BOTH]:
96
97
  rhs_cont = None
97
98
  if cont not in [Continuation.LHS, Continuation.BOTH]:
98
99
  lhs_cont = None
100
+
99
101
  return phrase_freqs, (lhs_cont, rhs_cont)
100
102
 
101
103
 
@@ -177,14 +179,11 @@ def _adjacent_bigram_freqs(lhs: np.ndarray, rhs: np.ndarray,
177
179
  if cont in [Continuation.RHS, Continuation.BOTH]:
178
180
  rhs_next = rhs_adj[matches]
179
181
  assert rhs_next is not None
180
- rhs_next = encoder.header(rhs_adj) | _1
181
- elif cont in [Continuation.LHS, Continuation.BOTH]:
182
- rhs_next = rhs_adj[matches]
183
- assert rhs_next is not None
184
- rhs_next |= _1
185
- rhs_next = encoder.header(rhs_adj) | _1
182
+ rhs_next = encoder.header(rhs_next) | _1
183
+ if cont in [Continuation.LHS, Continuation.BOTH]:
186
184
  lhs_next = lhs_adj[matches]
187
- lhs_next = encoder.header(lhs_adj) | _upper_bit
185
+ assert lhs_next is not None
186
+ lhs_next = encoder.header(lhs_next) | _upper_bit
188
187
  rhs_next = None
189
188
  return phrase_freqs, (lhs_next, rhs_next)
190
189
 
@@ -89,12 +89,10 @@ def _compute_phrase_freqs_left_to_right(encoded_posns: List[np.ndarray],
89
89
  phrase_freqs[mask] = 0
90
90
  phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
91
91
  cont=Continuation.RHS)
92
+
92
93
  assert conts[1] is not None
93
94
  lhs = conts[1]
94
95
  mask &= (phrase_freqs > 0)
95
- # print("-- pf", phrase_freqs)
96
- # dec = encoder.decode(lhs)
97
- # print("-- co", dec)
98
96
  phrase_freqs[~mask] = 0
99
97
  return phrase_freqs
100
98
 
@@ -118,12 +116,10 @@ def _compute_phrase_freqs_right_to_left(encoded_posns: List[np.ndarray],
118
116
  phrase_freqs[mask] = 0
119
117
  phrase_freqs, conts = bigram_freqs(lhs, rhs, phrase_freqs,
120
118
  cont=Continuation.LHS)
119
+
121
120
  assert conts[0] is not None
122
121
  rhs = conts[0]
123
122
  mask &= (phrase_freqs > 0)
124
- # print("-- pf", phrase_freqs)
125
- # dec = encoder.decode(lhs)
126
- # print("-- co", dec)
127
123
  phrase_freqs[~mask] = 0
128
124
  return phrase_freqs
129
125
 
@@ -25,7 +25,7 @@ _upper_bit = _1 << (encoder.payload_lsb_bits - _1)
25
25
  # MSB
26
26
  # term A |10| | 12| 13| | 18 * Intersect A+B -> 10,12
27
27
  # term B |10| | 12| | 14 15 17 | * RHS intersect -> 13 -> 14
28
- # term C | 11 14
28
+ # term C | 11 | 14
29
29
  # term D 10 | 15
30
30
  # term E | 11 12
31
31
  # ------
@@ -37,39 +37,88 @@ _upper_bit = _1 << (encoder.payload_lsb_bits - _1)
37
37
  #
38
38
  # ----
39
39
  #
40
+ # Intersecting
41
+ # term A |10| | 12| 13 18
42
+ # term B |10| | 12| 14 15 17
43
+ # Adjacent
44
+ # term A 10 12| 13| 18
45
+ # term B 10 12| | 14 15 17 |
46
+ # Giving 9.5, 10.5 11.5, 12.5, 13.5 , 17.5
47
+ # ------
48
+ # Next pair, intersecting
49
+ # term B 10 12 | 14 | 15 17 * RHS intersect -> 13 -> 14
50
+ # term C 11 | 14 |
51
+ # Adjacent
52
+ # term B 10 12 14 15 17
53
+ # term C |11 14
54
+ # Giving 10.5, 13.5, 14.5
55
+ # w/ above 10.5, 13.5
56
+ # ------
57
+ #
58
+ #
40
59
  # Finally: X X X X <- 4 words selected from each to find max span
41
60
  #
61
+ # Does it work to compute any intersection, and then attach any adjecent that exist
62
+ #
42
63
  #
64
+
65
+ def hdr(arr):
66
+ return encoder.header(arr) >> (_64 - encoder.header_bits)
67
+
68
+
43
69
  def _intersect_all(posns_encoded: List[np.ndarray]):
44
70
  """Intersect all encoded positions at roaringish MSBs."""
45
71
  if len(posns_encoded) < 2:
46
72
  raise ValueError("Need at least two positions to intersect")
73
+ last_lhs_headers = None
74
+ last_rhs_headers = None
75
+ curr = posns_encoded[0]
76
+ for term_idx, posns_next in enumerate(posns_encoded[1:]):
77
+ lhs_int_idx, rhs_int = intersect(curr, posns_next, mask=encoder.header_mask)
78
+ int_headers = encoder.header(curr[lhs_int_idx])
47
79
 
48
- lhs_int = posns_encoded[0]
49
- lhs_to_left = posns_encoded[0]
50
- lhs_to_right = posns_encoded[0]
80
+ # What is adjacent on LHS / RHS of interserction
81
+ # Next to left
82
+ # 0 -> 2
83
+ # <- 1 (curr to right)
84
+ # 1 -> (keep
85
+ # 0 <- 2
86
+ curr_to_right, next_to_left = adjacent(curr, posns_next, mask=encoder.header_mask)
87
+ lhs_headers = merge(int_headers, posns_next[next_to_left])
88
+ rhs_headers = merge(int_headers, curr[curr_to_right])
89
+ next_to_right, curr_to_left = adjacent(posns_next, curr, mask=encoder.header_mask)
90
+ lhs_headers = merge(lhs_headers, curr[curr_to_left])
91
+ rhs_headers = merge(rhs_headers, posns_next[next_to_right])
51
92
 
52
- for term_idx, posns_next in enumerate(posns_encoded[1:]):
53
- _, rhs_int = intersect(lhs_int, posns_next, mask=encoder.header_mask)
54
- _, rhs_ls = adjacent(lhs_to_left, posns_next, mask=encoder.header_mask)
55
- rhs_rs, _ = adjacent(lhs_to_right, posns_next, mask=encoder.header_mask)
93
+ if last_lhs_headers is not None:
94
+ lhs, _ = intersect(last_lhs_headers, lhs_headers, mask=encoder.header_mask)
95
+ rhs, _ = intersect(last_rhs_headers, rhs_headers, mask=encoder.header_mask)
96
+ last_lhs_headers = last_lhs_headers[lhs]
97
+ last_rhs_headers = last_rhs_headers[rhs]
98
+ else:
99
+ last_lhs_headers = lhs_headers
100
+ last_rhs_headers = rhs_headers
56
101
 
57
- # Update LHS to rhs_int + rhs_ls + rhs_rs indices
58
- lhs_int = posns_next[rhs_int]
59
- lhs_to_left = posns_next[rhs_ls]
60
- lhs_to_right = posns_next[rhs_rs]
102
+ # Update by intersecting with current working lhs / rhs headers
61
103
 
104
+ assert last_rhs_headers is not None
105
+ assert last_lhs_headers is not None
106
+ to_rhs = last_rhs_headers + (_1 << (_64 - encoder.header_bits))
107
+ to_lhs = last_lhs_headers - (_1 << (_64 - encoder.header_bits))
108
+ all_headers = merge(to_rhs, to_lhs, drop_duplicates=True)
109
+ all_headers = merge(last_lhs_headers, all_headers, drop_duplicates=True)
110
+ all_headers = merge(last_rhs_headers, all_headers, drop_duplicates=True)
111
+ all_headers = all_headers & encoder.header_mask
62
112
  # Get active MSBs now
63
113
  # Merge all the rest to grab them
64
- int_header = encoder.header(lhs_int)
65
- to_left_header = encoder.header(lhs_to_left)
66
- to_right_header = encoder.header(lhs_to_right)
67
- merged = merge(int_header, to_left_header, drop_duplicates=True)
68
- merged = merge(merged, to_right_header, drop_duplicates=True)
69
114
 
70
- # Slice only matches at MSBs
115
+ # Slice only matches at header MSBs
116
+ new_posns_encoded = posns_encoded.copy()
71
117
  for i in range(len(posns_encoded)):
72
- posns_encoded[i] = encoder.slice(posns_encoded[i], merged)
118
+ new_posns_encoded[i] = encoder.slice(posns_encoded[i], header=all_headers)
119
+ lengths = np.cumsum([0] + [len(posns) for posns in new_posns_encoded], dtype=np.uint64)
120
+ concatted = np.concatenate(new_posns_encoded, dtype=np.uint64)
121
+ return concatted, lengths
73
122
 
74
123
 
75
124
  # Picking up from intersections:
@@ -121,12 +170,14 @@ def span_search(posns_encoded: List[np.ndarray],
121
170
  phrase_freqs: np.ndarray,
122
171
  slop: int) -> np.ndarray:
123
172
  """Find span matches up to PAYLOAD_LSB bits span distance."""
124
- # Find inner span candidates
125
- _intersect_all(posns_encoded)
173
+ # Find posns to check for span candidates
174
+ posns, lengths = _intersect_all(posns_encoded)
126
175
 
127
176
  # Populate phrase freqs with matches of slop
128
- r_span_search(posns_encoded, phrase_freqs, slop,
177
+ r_span_search(posns, lengths,
178
+ phrase_freqs, slop,
129
179
  encoder.key_mask,
130
180
  encoder.header_mask,
181
+ encoder.key_bits,
131
182
  encoder.payload_lsb_bits)
132
183
  return phrase_freqs
@@ -619,6 +619,7 @@ class SearchArray(ExtensionArray):
619
619
 
620
620
  def score(self, token: Union[str, List[str]],
621
621
  similarity: Similarity = default_bm25,
622
+ slop: int = 0,
622
623
  min_posn: Optional[int] = None,
623
624
  max_posn: Optional[int] = None) -> np.ndarray:
624
625
  """Score each doc using a similarity function.
@@ -638,7 +639,8 @@ class SearchArray(ExtensionArray):
638
639
  tokens_l = [token] if isinstance(token, str) else token
639
640
  all_dfs = np.asarray([self.docfreq(token) for token in tokens_l])
640
641
 
641
- tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn)
642
+ tfs = self.termfreqs(token, min_posn=min_posn, max_posn=max_posn,
643
+ slop=slop)
642
644
  token = self._check_token_arg(token)
643
645
  doc_lens = self.doclengths()
644
646
 
@@ -656,6 +658,8 @@ class SearchArray(ExtensionArray):
656
658
  slop=0,
657
659
  min_posn: Optional[int] = None,
658
660
  max_posn: Optional[int] = None) -> np.ndarray:
661
+ if slop > 0:
662
+ logger.warning("!! Slop is experimental and may be slow, crash, or inaccurate etc")
659
663
  try:
660
664
  # Decide how/if we need to filter doc ids
661
665
  term_ids = [self.term_dict.get_term_id(token) for token in tokens]
@@ -1,6 +1,6 @@
1
1
  from searcharray.roaringish.roaringish import RoaringishEncoder, convert_keys # noqa
2
2
  from searcharray.roaringish.search import count_odds # noqa
3
- from searcharray.roaringish.popcount import popcount_reduce_at, popcount64 # noqa
3
+ from searcharray.roaringish.popcount import popcount_reduce_at, popcount64, key_sum_over # noqa
4
4
  from searcharray.roaringish.intersect import intersect, adjacent # noqa
5
5
  from searcharray.roaringish.unique import unique # noqa
6
6
  from searcharray.roaringish.merge import merge # noqa