searcharray 0.0.21__tar.gz → 0.0.23__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.21 → searcharray-0.0.23}/PKG-INFO +1 -1
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/solr.py +52 -21
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.21 → searcharray-0.0.23}/setup.py +1 -1
- {searcharray-0.0.21 → searcharray-0.0.23}/test/test_solr.py +46 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/README.md +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/pyproject.toml +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/__init__.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/phrase/middle_out.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/postings.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/utils/roaringish.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/setup.cfg +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/test/test_extension_array.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/test/test_msmarco.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/test/test_search.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/test/test_tmdb.py +0 -0
- {searcharray-0.0.21 → searcharray-0.0.23}/test/test_utils.py +0 -0
|
@@ -88,6 +88,7 @@ def parse_query_terms(frame: pd.DataFrame,
|
|
|
88
88
|
|
|
89
89
|
search_terms: Dict[str, List[str]] = {}
|
|
90
90
|
num_search_terms = 0
|
|
91
|
+
term_centric = True
|
|
91
92
|
|
|
92
93
|
for field in query_fields:
|
|
93
94
|
arr = get_field(frame, field)
|
|
@@ -101,9 +102,50 @@ def parse_query_terms(frame: pd.DataFrame,
|
|
|
101
102
|
if num_search_terms == 0:
|
|
102
103
|
num_search_terms = field_num_search_terms
|
|
103
104
|
elif field_num_search_terms != num_search_terms:
|
|
104
|
-
|
|
105
|
+
term_centric = False
|
|
105
106
|
|
|
106
|
-
return num_search_terms, search_terms
|
|
107
|
+
return num_search_terms, search_terms, term_centric
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _edismax_term_centric(frame: pd.DataFrame,
|
|
111
|
+
query_fields: Dict[str, float],
|
|
112
|
+
num_search_terms: int,
|
|
113
|
+
search_terms: Dict[str, List[str]],
|
|
114
|
+
min_should_match: int):
|
|
115
|
+
term_scores = []
|
|
116
|
+
for term_posn in range(num_search_terms):
|
|
117
|
+
max_scores = np.zeros(len(frame))
|
|
118
|
+
for field, boost in query_fields.items():
|
|
119
|
+
term = search_terms[field][term_posn]
|
|
120
|
+
post_arr = get_field(frame, field)
|
|
121
|
+
field_term_score = post_arr.bm25(term) * (1 if boost is None else boost)
|
|
122
|
+
max_scores = np.maximum(max_scores, field_term_score)
|
|
123
|
+
term_scores.append(max_scores)
|
|
124
|
+
|
|
125
|
+
qf_scores = np.asarray(term_scores)
|
|
126
|
+
matches_gt_mm = np.sum(qf_scores > 0, axis=0) >= min_should_match
|
|
127
|
+
qf_scores = np.sum(term_scores, axis=0)
|
|
128
|
+
qf_scores[~matches_gt_mm] = 0
|
|
129
|
+
return qf_scores
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _edismax_field_centric(frame: pd.DataFrame,
|
|
133
|
+
query_fields: Dict[str, float],
|
|
134
|
+
num_search_terms: int,
|
|
135
|
+
search_terms: Dict[str, List[str]],
|
|
136
|
+
min_should_match: int) -> np.ndarray:
|
|
137
|
+
field_scores = []
|
|
138
|
+
for field, boost in query_fields.items():
|
|
139
|
+
post_arr = get_field(frame, field)
|
|
140
|
+
term_scores = np.array([post_arr.bm25(term) for term in search_terms[field]])
|
|
141
|
+
matches_gt_mm = np.sum(term_scores > 0, axis=0) >= min(min_should_match, len(search_terms[field]))
|
|
142
|
+
sum_terms_bm25 = np.sum(term_scores, axis=0)
|
|
143
|
+
sum_terms_bm25[~matches_gt_mm] = 0
|
|
144
|
+
field_scores.append(sum_terms_bm25 * (1 if boost is None else boost))
|
|
145
|
+
# Take maximum field scores as qf
|
|
146
|
+
qf_scores = np.asarray(field_scores)
|
|
147
|
+
qf_scores = np.max(qf_scores, axis=0)
|
|
148
|
+
return qf_scores
|
|
107
149
|
|
|
108
150
|
|
|
109
151
|
def edismax(frame: pd.DataFrame,
|
|
@@ -143,31 +185,20 @@ def edismax(frame: pd.DataFrame,
|
|
|
143
185
|
|
|
144
186
|
query_fields = parse_field_boosts(listify(qf))
|
|
145
187
|
phrase_fields = parse_field_boosts(listify(pf)) if pf else {}
|
|
146
|
-
|
|
147
|
-
# bigram_fields = parse_field_boosts(pf2) if pf2 else {}
|
|
148
|
-
# trigram_fields = parse_field_boosts(pf3) if pf3 else {}
|
|
149
|
-
|
|
150
|
-
num_search_terms, search_terms = parse_query_terms(frame, q, qf)
|
|
151
|
-
|
|
152
|
-
term_scores = []
|
|
153
|
-
for term_posn in range(num_search_terms):
|
|
154
|
-
max_scores = np.zeros(len(frame))
|
|
155
|
-
for field, boost in query_fields.items():
|
|
156
|
-
term = search_terms[field][term_posn]
|
|
157
|
-
field_term_score = frame[field].array.bm25(term) * (1 if boost is None else boost)
|
|
158
|
-
max_scores = np.maximum(max_scores, field_term_score)
|
|
159
|
-
term_scores.append(max_scores)
|
|
160
|
-
|
|
161
188
|
if mm is None:
|
|
162
189
|
mm = "1"
|
|
163
190
|
if q_op == "AND":
|
|
164
191
|
mm = "100%"
|
|
165
192
|
|
|
193
|
+
# bigram_fields = parse_field_boosts(pf2) if pf2 else {}
|
|
194
|
+
# trigram_fields = parse_field_boosts(pf3) if pf3 else {}
|
|
195
|
+
|
|
196
|
+
num_search_terms, search_terms, term_centric = parse_query_terms(frame, q, list(query_fields.keys()))
|
|
166
197
|
min_should_match = parse_min_should_match(num_search_terms, spec=mm)
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
198
|
+
if term_centric:
|
|
199
|
+
qf_scores = _edismax_term_centric(frame, query_fields, num_search_terms, search_terms, min_should_match)
|
|
200
|
+
else:
|
|
201
|
+
qf_scores = _edismax_field_centric(frame, query_fields, num_search_terms, search_terms, min_should_match)
|
|
171
202
|
|
|
172
203
|
phrase_scores = []
|
|
173
204
|
for field, boost in phrase_fields.items():
|
|
@@ -36,7 +36,7 @@ setup(
|
|
|
36
36
|
# For a discussion on single-sourcing the version across setup.py and the
|
|
37
37
|
# project code, see
|
|
38
38
|
# https://packaging.python.org/guides/single-sourcing-package-version/
|
|
39
|
-
version="0.0.
|
|
39
|
+
version="0.0.23", # Required
|
|
40
40
|
# This is a one-line description or tagline of what your project does. This
|
|
41
41
|
# corresponds to the "Summary" metadata field:
|
|
42
42
|
# https://packaging.python.org/specifications/core-metadata/#summary
|
|
@@ -74,6 +74,11 @@ def everythings_a_b_tokenizer(text: str) -> List[str]:
|
|
|
74
74
|
return ["b"] * len(text.split())
|
|
75
75
|
|
|
76
76
|
|
|
77
|
+
def just_lowercasing_tokenizer(text: str) -> List[str]:
|
|
78
|
+
"""Lowercase and return a list of tokens."""
|
|
79
|
+
return [text.lower()]
|
|
80
|
+
|
|
81
|
+
|
|
77
82
|
edismax_scenarios = {
|
|
78
83
|
"base": {
|
|
79
84
|
"frame": {
|
|
@@ -88,6 +93,47 @@ edismax_scenarios = {
|
|
|
88
93
|
0],
|
|
89
94
|
"params": {'q': "foo bar", 'qf': ["title", "body"]},
|
|
90
95
|
},
|
|
96
|
+
"field_centric": {
|
|
97
|
+
"frame": {
|
|
98
|
+
'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
|
|
99
|
+
'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
|
|
100
|
+
tokenizer=just_lowercasing_tokenizer)
|
|
101
|
+
},
|
|
102
|
+
"expected": [lambda frame: max(sum([frame['title'].array.bm25("foo")[0],
|
|
103
|
+
frame['title'].array.bm25("bar")[0]]),
|
|
104
|
+
frame['body'].array.bm25("foo bar")[0]),
|
|
105
|
+
0,
|
|
106
|
+
lambda frame: frame['title'].array.bm25("bar")[2],
|
|
107
|
+
0],
|
|
108
|
+
"params": {'q': "foo bar", 'qf': ["title", "body"]},
|
|
109
|
+
},
|
|
110
|
+
"field_centric_mm": {
|
|
111
|
+
"frame": {
|
|
112
|
+
'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
|
|
113
|
+
'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
|
|
114
|
+
tokenizer=just_lowercasing_tokenizer)
|
|
115
|
+
},
|
|
116
|
+
"expected": [lambda frame: max(sum([frame['title'].array.bm25("foo")[0],
|
|
117
|
+
frame['title'].array.bm25("bar")[0]]),
|
|
118
|
+
frame['body'].array.bm25("foo bar")[0]),
|
|
119
|
+
0,
|
|
120
|
+
0,
|
|
121
|
+
0],
|
|
122
|
+
"params": {'q': "foo bar", 'qf': ["title", "body"], 'mm': "2"},
|
|
123
|
+
},
|
|
124
|
+
"boost_title": {
|
|
125
|
+
"frame": {
|
|
126
|
+
'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
|
|
127
|
+
'body': lambda: PostingsArray.index(["buzz", "data2", "data3 bar", "bunny funny wunny"])
|
|
128
|
+
},
|
|
129
|
+
"expected": [lambda frame: sum([frame['title'].array.bm25("foo")[0] * 10,
|
|
130
|
+
frame['title'].array.bm25("bar")[0] * 10]),
|
|
131
|
+
0,
|
|
132
|
+
lambda frame: max(frame['title'].array.bm25("bar")[2] * 10,
|
|
133
|
+
frame['body'].array.bm25("bar")[2]),
|
|
134
|
+
0],
|
|
135
|
+
"params": {'q': "foo bar", 'qf': ["title^10", "body"]},
|
|
136
|
+
},
|
|
91
137
|
"pf_title": {
|
|
92
138
|
"frame": {
|
|
93
139
|
'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|