searcharray 0.0.21__tar.gz → 0.0.23__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. {searcharray-0.0.21 → searcharray-0.0.23}/PKG-INFO +1 -1
  2. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/solr.py +52 -21
  3. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/PKG-INFO +1 -1
  4. {searcharray-0.0.21 → searcharray-0.0.23}/setup.py +1 -1
  5. {searcharray-0.0.21 → searcharray-0.0.23}/test/test_solr.py +46 -0
  6. {searcharray-0.0.21 → searcharray-0.0.23}/README.md +0 -0
  7. {searcharray-0.0.21 → searcharray-0.0.23}/pyproject.toml +0 -0
  8. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/__init__.py +0 -0
  9. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/phrase/__init__.py +0 -0
  10. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/phrase/middle_out.py +0 -0
  11. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/phrase/posn_diffs.py +0 -0
  12. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/phrase/scan_merge.py +0 -0
  13. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/postings.py +0 -0
  14. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/term_dict.py +0 -0
  15. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/utils/__init__.py +0 -0
  16. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/utils/bitcount.py +0 -0
  17. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/utils/mat_set.py +0 -0
  18. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/utils/roaringish.py +0 -0
  19. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/utils/row_viewable_matrix.py +0 -0
  20. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/SOURCES.txt +0 -0
  21. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/dependency_links.txt +0 -0
  22. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/entry_points.txt +0 -0
  23. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/requires.txt +0 -0
  24. {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/top_level.txt +0 -0
  25. {searcharray-0.0.21 → searcharray-0.0.23}/setup.cfg +0 -0
  26. {searcharray-0.0.21 → searcharray-0.0.23}/test/test_extension_array.py +0 -0
  27. {searcharray-0.0.21 → searcharray-0.0.23}/test/test_msmarco.py +0 -0
  28. {searcharray-0.0.21 → searcharray-0.0.23}/test/test_phrase_matches.py +0 -0
  29. {searcharray-0.0.21 → searcharray-0.0.23}/test/test_search.py +0 -0
  30. {searcharray-0.0.21 → searcharray-0.0.23}/test/test_tmdb.py +0 -0
  31. {searcharray-0.0.21 → searcharray-0.0.23}/test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.21
3
+ Version: 0.0.23
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -88,6 +88,7 @@ def parse_query_terms(frame: pd.DataFrame,
88
88
 
89
89
  search_terms: Dict[str, List[str]] = {}
90
90
  num_search_terms = 0
91
+ term_centric = True
91
92
 
92
93
  for field in query_fields:
93
94
  arr = get_field(frame, field)
@@ -101,9 +102,50 @@ def parse_query_terms(frame: pd.DataFrame,
101
102
  if num_search_terms == 0:
102
103
  num_search_terms = field_num_search_terms
103
104
  elif field_num_search_terms != num_search_terms:
104
- raise ValueError("All qf field tokenizers must emit the same number of terms")
105
+ term_centric = False
105
106
 
106
- return num_search_terms, search_terms
107
+ return num_search_terms, search_terms, term_centric
108
+
109
+
110
+ def _edismax_term_centric(frame: pd.DataFrame,
111
+ query_fields: Dict[str, float],
112
+ num_search_terms: int,
113
+ search_terms: Dict[str, List[str]],
114
+ min_should_match: int):
115
+ term_scores = []
116
+ for term_posn in range(num_search_terms):
117
+ max_scores = np.zeros(len(frame))
118
+ for field, boost in query_fields.items():
119
+ term = search_terms[field][term_posn]
120
+ post_arr = get_field(frame, field)
121
+ field_term_score = post_arr.bm25(term) * (1 if boost is None else boost)
122
+ max_scores = np.maximum(max_scores, field_term_score)
123
+ term_scores.append(max_scores)
124
+
125
+ qf_scores = np.asarray(term_scores)
126
+ matches_gt_mm = np.sum(qf_scores > 0, axis=0) >= min_should_match
127
+ qf_scores = np.sum(term_scores, axis=0)
128
+ qf_scores[~matches_gt_mm] = 0
129
+ return qf_scores
130
+
131
+
132
+ def _edismax_field_centric(frame: pd.DataFrame,
133
+ query_fields: Dict[str, float],
134
+ num_search_terms: int,
135
+ search_terms: Dict[str, List[str]],
136
+ min_should_match: int) -> np.ndarray:
137
+ field_scores = []
138
+ for field, boost in query_fields.items():
139
+ post_arr = get_field(frame, field)
140
+ term_scores = np.array([post_arr.bm25(term) for term in search_terms[field]])
141
+ matches_gt_mm = np.sum(term_scores > 0, axis=0) >= min(min_should_match, len(search_terms[field]))
142
+ sum_terms_bm25 = np.sum(term_scores, axis=0)
143
+ sum_terms_bm25[~matches_gt_mm] = 0
144
+ field_scores.append(sum_terms_bm25 * (1 if boost is None else boost))
145
+ # Take maximum field scores as qf
146
+ qf_scores = np.asarray(field_scores)
147
+ qf_scores = np.max(qf_scores, axis=0)
148
+ return qf_scores
107
149
 
108
150
 
109
151
  def edismax(frame: pd.DataFrame,
@@ -143,31 +185,20 @@ def edismax(frame: pd.DataFrame,
143
185
 
144
186
  query_fields = parse_field_boosts(listify(qf))
145
187
  phrase_fields = parse_field_boosts(listify(pf)) if pf else {}
146
-
147
- # bigram_fields = parse_field_boosts(pf2) if pf2 else {}
148
- # trigram_fields = parse_field_boosts(pf3) if pf3 else {}
149
-
150
- num_search_terms, search_terms = parse_query_terms(frame, q, qf)
151
-
152
- term_scores = []
153
- for term_posn in range(num_search_terms):
154
- max_scores = np.zeros(len(frame))
155
- for field, boost in query_fields.items():
156
- term = search_terms[field][term_posn]
157
- field_term_score = frame[field].array.bm25(term) * (1 if boost is None else boost)
158
- max_scores = np.maximum(max_scores, field_term_score)
159
- term_scores.append(max_scores)
160
-
161
188
  if mm is None:
162
189
  mm = "1"
163
190
  if q_op == "AND":
164
191
  mm = "100%"
165
192
 
193
+ # bigram_fields = parse_field_boosts(pf2) if pf2 else {}
194
+ # trigram_fields = parse_field_boosts(pf3) if pf3 else {}
195
+
196
+ num_search_terms, search_terms, term_centric = parse_query_terms(frame, q, list(query_fields.keys()))
166
197
  min_should_match = parse_min_should_match(num_search_terms, spec=mm)
167
- qf_scores = np.asarray(term_scores)
168
- matches_gt_mm = np.sum(qf_scores > 0, axis=0) >= min_should_match
169
- qf_scores = np.sum(term_scores, axis=0)
170
- qf_scores[~matches_gt_mm] = 0
198
+ if term_centric:
199
+ qf_scores = _edismax_term_centric(frame, query_fields, num_search_terms, search_terms, min_should_match)
200
+ else:
201
+ qf_scores = _edismax_field_centric(frame, query_fields, num_search_terms, search_terms, min_should_match)
171
202
 
172
203
  phrase_scores = []
173
204
  for field, boost in phrase_fields.items():
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: searcharray
3
- Version: 0.0.21
3
+ Version: 0.0.23
4
4
  Summary: Searchable pandas text extension arrays for prototyping search
5
5
  Home-page: https://github.com/softwaredoug/searcharray
6
6
  Author: Doug Turnbull
@@ -36,7 +36,7 @@ setup(
36
36
  # For a discussion on single-sourcing the version across setup.py and the
37
37
  # project code, see
38
38
  # https://packaging.python.org/guides/single-sourcing-package-version/
39
- version="0.0.21", # Required
39
+ version="0.0.23", # Required
40
40
  # This is a one-line description or tagline of what your project does. This
41
41
  # corresponds to the "Summary" metadata field:
42
42
  # https://packaging.python.org/specifications/core-metadata/#summary
@@ -74,6 +74,11 @@ def everythings_a_b_tokenizer(text: str) -> List[str]:
74
74
  return ["b"] * len(text.split())
75
75
 
76
76
 
77
+ def just_lowercasing_tokenizer(text: str) -> List[str]:
78
+ """Lowercase and return a list of tokens."""
79
+ return [text.lower()]
80
+
81
+
77
82
  edismax_scenarios = {
78
83
  "base": {
79
84
  "frame": {
@@ -88,6 +93,47 @@ edismax_scenarios = {
88
93
  0],
89
94
  "params": {'q': "foo bar", 'qf': ["title", "body"]},
90
95
  },
96
+ "field_centric": {
97
+ "frame": {
98
+ 'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
99
+ 'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
100
+ tokenizer=just_lowercasing_tokenizer)
101
+ },
102
+ "expected": [lambda frame: max(sum([frame['title'].array.bm25("foo")[0],
103
+ frame['title'].array.bm25("bar")[0]]),
104
+ frame['body'].array.bm25("foo bar")[0]),
105
+ 0,
106
+ lambda frame: frame['title'].array.bm25("bar")[2],
107
+ 0],
108
+ "params": {'q': "foo bar", 'qf': ["title", "body"]},
109
+ },
110
+ "field_centric_mm": {
111
+ "frame": {
112
+ 'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
113
+ 'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
114
+ tokenizer=just_lowercasing_tokenizer)
115
+ },
116
+ "expected": [lambda frame: max(sum([frame['title'].array.bm25("foo")[0],
117
+ frame['title'].array.bm25("bar")[0]]),
118
+ frame['body'].array.bm25("foo bar")[0]),
119
+ 0,
120
+ 0,
121
+ 0],
122
+ "params": {'q': "foo bar", 'qf': ["title", "body"], 'mm': "2"},
123
+ },
124
+ "boost_title": {
125
+ "frame": {
126
+ 'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
127
+ 'body': lambda: PostingsArray.index(["buzz", "data2", "data3 bar", "bunny funny wunny"])
128
+ },
129
+ "expected": [lambda frame: sum([frame['title'].array.bm25("foo")[0] * 10,
130
+ frame['title'].array.bm25("bar")[0] * 10]),
131
+ 0,
132
+ lambda frame: max(frame['title'].array.bm25("bar")[2] * 10,
133
+ frame['body'].array.bm25("bar")[2]),
134
+ 0],
135
+ "params": {'q': "foo bar", 'qf': ["title^10", "body"]},
136
+ },
91
137
  "pf_title": {
92
138
  "frame": {
93
139
  'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
File without changes
File without changes