searcharray 0.0.24__tar.gz → 0.0.26__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. {searcharray-0.0.24 → searcharray-0.0.26}/PKG-INFO +1 -1
  2. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/phrase/middle_out.py +1 -1
  3. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/solr.py +7 -6
  4. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/PKG-INFO +1 -1
  5. {searcharray-0.0.24 → searcharray-0.0.26}/setup.py +1 -1
  6. {searcharray-0.0.24 → searcharray-0.0.26}/test/test_solr.py +14 -0
  7. {searcharray-0.0.24 → searcharray-0.0.26}/test/test_tmdb.py +13 -0
  8. {searcharray-0.0.24 → searcharray-0.0.26}/README.md +0 -0
  9. {searcharray-0.0.24 → searcharray-0.0.26}/pyproject.toml +0 -0
  10. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/__init__.py +0 -0
  11. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/phrase/__init__.py +0 -0
  12. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/phrase/posn_diffs.py +0 -0
  13. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/phrase/scan_merge.py +0 -0
  14. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/postings.py +0 -0
  15. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/term_dict.py +0 -0
  16. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/utils/__init__.py +0 -0
  17. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/utils/bitcount.py +0 -0
  18. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/utils/mat_set.py +0 -0
  19. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/utils/roaringish.py +0 -0
  20. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/utils/row_viewable_matrix.py +0 -0
  21. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/SOURCES.txt +0 -0
  22. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/dependency_links.txt +0 -0
  23. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/entry_points.txt +0 -0
  24. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/requires.txt +0 -0
  25. {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/top_level.txt +0 -0
  26. {searcharray-0.0.24 → searcharray-0.0.26}/setup.cfg +0 -0
  27. {searcharray-0.0.24 → searcharray-0.0.26}/test/test_extension_array.py +0 -0
  28. {searcharray-0.0.24 → searcharray-0.0.26}/test/test_msmarco.py +0 -0
  29. {searcharray-0.0.24 → searcharray-0.0.26}/test/test_phrase_matches.py +0 -0
  30. {searcharray-0.0.24 → searcharray-0.0.26}/test/test_search.py +0 -0
  31. {searcharray-0.0.24 → searcharray-0.0.26}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.24
3
+ Version: 0.0.26
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -316,7 +316,7 @@ class PosnBitArray:
316
316
  doc_ids = encoder.keys(term_posns)
317
317
  change_indices = np.nonzero(np.diff(doc_ids))[0]
318
318
  change_indices = np.concatenate((np.asarray([0]), change_indices + 1))
319
- posns = encoded & encoder.payload_lsb_mask
319
+ posns = term_posns & encoder.payload_lsb_mask
320
320
  bit_counts = bit_count64(posns)
321
321
 
322
322
  term_freqs = np.add.reduceat(bit_counts, change_indices)
@@ -111,7 +111,7 @@ def _edismax_term_centric(frame: pd.DataFrame,
111
111
  query_fields: Dict[str, float],
112
112
  num_search_terms: int,
113
113
  search_terms: Dict[str, List[str]],
114
- min_should_match: int) -> Tuple[np.ndarray, str]:
114
+ mm: str) -> Tuple[np.ndarray, str]:
115
115
  explain = []
116
116
  term_scores = []
117
117
  for term_posn in range(num_search_terms):
@@ -127,6 +127,7 @@ def _edismax_term_centric(frame: pd.DataFrame,
127
127
  term_scores.append(max_scores)
128
128
  explain.append("(" + " | ".join(term_explain) + ")")
129
129
 
130
+ min_should_match = parse_min_should_match(num_search_terms, spec=mm)
130
131
  qf_scores = np.asarray(term_scores)
131
132
  matches_gt_mm = np.sum(qf_scores > 0, axis=0) >= min_should_match
132
133
  qf_scores = np.sum(term_scores, axis=0)
@@ -138,17 +139,18 @@ def _edismax_field_centric(frame: pd.DataFrame,
138
139
  query_fields: Dict[str, float],
139
140
  num_search_terms: int,
140
141
  search_terms: Dict[str, List[str]],
141
- min_should_match: int) -> Tuple[np.ndarray, str]:
142
+ mm: str) -> Tuple[np.ndarray, str]:
142
143
  field_scores = []
143
144
  explain = []
144
145
  for field, boost in query_fields.items():
145
146
  post_arr = get_field(frame, field)
146
147
  term_scores = np.array([post_arr.bm25(term) for term in search_terms[field]])
148
+ min_should_match = parse_min_should_match(len(search_terms[field]), spec=mm)
147
149
  exp = " ".join([f"{field}:{term}" for term in search_terms[field]])
148
150
  boost_exp = f"{boost}" if boost is not None else "1"
149
151
  exp = "(" + exp + f")~{min(min_should_match, len(search_terms[field]))}"
150
152
  exp = "(" + exp + f")^{boost_exp}"
151
- explain.append(exp)
153
+
152
154
  matches_gt_mm = np.sum(term_scores > 0, axis=0) >= min(min_should_match, len(search_terms[field]))
153
155
  sum_terms_bm25 = np.sum(term_scores, axis=0)
154
156
  sum_terms_bm25[~matches_gt_mm] = 0
@@ -206,11 +208,10 @@ def edismax(frame: pd.DataFrame,
206
208
  # trigram_fields = parse_field_boosts(pf3) if pf3 else {}
207
209
 
208
210
  num_search_terms, search_terms, term_centric = parse_query_terms(frame, q, list(query_fields.keys()))
209
- min_should_match = parse_min_should_match(num_search_terms, spec=mm)
210
211
  if term_centric:
211
- qf_scores, explain = _edismax_term_centric(frame, query_fields, num_search_terms, search_terms, min_should_match)
212
+ qf_scores, explain = _edismax_term_centric(frame, query_fields, num_search_terms, search_terms, mm)
212
213
  else:
213
- qf_scores, explain = _edismax_field_centric(frame, query_fields, num_search_terms, search_terms, min_should_match)
214
+ qf_scores, explain = _edismax_field_centric(frame, query_fields, num_search_terms, search_terms, mm)
214
215
 
215
216
  phrase_scores = []
216
217
  for field, boost in phrase_fields.items():
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.24
3
+ Version: 0.0.26
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -36,7 +36,7 @@ setup(
36
36
  # For a discussion on single-sourcing the version across setup.py and the
37
37
  # project code, see
38
38
  # https://packaging.python.org/guides/single-sourcing-package-version/
39
- version="0.0.24", # Required
39
+ version="0.0.26", # Required
40
40
  # This is a one-line description or tagline of what your project does. This
41
41
  # corresponds to the "Summary" metadata field:
42
42
  # https://packaging.python.org/specifications/core-metadata/#summary
@@ -121,6 +121,20 @@ edismax_scenarios = {
121
121
  0],
122
122
  "params": {'q': "foo bar", 'qf': ["title", "body"], 'mm': "2"},
123
123
  },
124
+ "field_centric_mm_opp": {
125
+ "frame": {
126
+ 'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
127
+ 'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
128
+ tokenizer=just_lowercasing_tokenizer)
129
+ },
130
+ "expected": [lambda frame: max(sum([frame['title'].array.bm25("foo")[0],
131
+ frame['title'].array.bm25("bar")[0]]),
132
+ frame['body'].array.bm25("foo bar")[0]),
133
+ 0,
134
+ 0,
135
+ 0],
136
+ "params": {'q': "foo bar", 'qf': ["body", "title"], 'mm': "2"},
137
+ },
124
138
  "boost_title": {
125
139
  "frame": {
126
140
  'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
@@ -91,6 +91,19 @@ def test_slice_then_search(tmdb_data):
91
91
  assert skywalker_bm25.shape[0] == 3
92
92
 
93
93
 
94
+ tmdb_term_matches = [
95
+ ("Star", ['11', '330459', '76180']),
96
+ ("Black", ['374430']),
97
+ ]
98
+
99
+
100
+ @pytest.mark.parametrize("term,expected_matches", tmdb_term_matches)
101
+ def test_term_freqs(tmdb_data, term, expected_matches):
102
+ sliced = tmdb_data[tmdb_data['doc_id'].isin(expected_matches)]
103
+ term_freqs = sliced['title_tokens'].array.term_freq(term)
104
+ assert np.all(term_freqs == 1)
105
+
106
+
94
107
  tmdb_phrase_matches = [
95
108
  (["Star", "Wars"], ['11', '330459', '76180']),
96
109
  (["Black", "Mirror:"], ['374430']),
File without changes
File without changes