searcharray 0.0.24__tar.gz → 0.0.26__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searcharray-0.0.24 → searcharray-0.0.26}/PKG-INFO +1 -1
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/phrase/middle_out.py +1 -1
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/solr.py +7 -6
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/PKG-INFO +1 -1
- {searcharray-0.0.24 → searcharray-0.0.26}/setup.py +1 -1
- {searcharray-0.0.24 → searcharray-0.0.26}/test/test_solr.py +14 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/test/test_tmdb.py +13 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/README.md +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/pyproject.toml +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/__init__.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/phrase/__init__.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/phrase/posn_diffs.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/phrase/scan_merge.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/postings.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/term_dict.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/utils/__init__.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/utils/bitcount.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/utils/mat_set.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/utils/roaringish.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray/utils/row_viewable_matrix.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/SOURCES.txt +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/dependency_links.txt +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/entry_points.txt +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/requires.txt +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/searcharray.egg-info/top_level.txt +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/setup.cfg +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/test/test_extension_array.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/test/test_msmarco.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/test/test_phrase_matches.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/test/test_search.py +0 -0
- {searcharray-0.0.24 → searcharray-0.0.26}/test/test_utils.py +0 -0
|
@@ -316,7 +316,7 @@ class PosnBitArray:
|
|
|
316
316
|
doc_ids = encoder.keys(term_posns)
|
|
317
317
|
change_indices = np.nonzero(np.diff(doc_ids))[0]
|
|
318
318
|
change_indices = np.concatenate((np.asarray([0]), change_indices + 1))
|
|
319
|
-
posns =
|
|
319
|
+
posns = term_posns & encoder.payload_lsb_mask
|
|
320
320
|
bit_counts = bit_count64(posns)
|
|
321
321
|
|
|
322
322
|
term_freqs = np.add.reduceat(bit_counts, change_indices)
|
|
@@ -111,7 +111,7 @@ def _edismax_term_centric(frame: pd.DataFrame,
|
|
|
111
111
|
query_fields: Dict[str, float],
|
|
112
112
|
num_search_terms: int,
|
|
113
113
|
search_terms: Dict[str, List[str]],
|
|
114
|
-
|
|
114
|
+
mm: str) -> Tuple[np.ndarray, str]:
|
|
115
115
|
explain = []
|
|
116
116
|
term_scores = []
|
|
117
117
|
for term_posn in range(num_search_terms):
|
|
@@ -127,6 +127,7 @@ def _edismax_term_centric(frame: pd.DataFrame,
|
|
|
127
127
|
term_scores.append(max_scores)
|
|
128
128
|
explain.append("(" + " | ".join(term_explain) + ")")
|
|
129
129
|
|
|
130
|
+
min_should_match = parse_min_should_match(num_search_terms, spec=mm)
|
|
130
131
|
qf_scores = np.asarray(term_scores)
|
|
131
132
|
matches_gt_mm = np.sum(qf_scores > 0, axis=0) >= min_should_match
|
|
132
133
|
qf_scores = np.sum(term_scores, axis=0)
|
|
@@ -138,17 +139,18 @@ def _edismax_field_centric(frame: pd.DataFrame,
|
|
|
138
139
|
query_fields: Dict[str, float],
|
|
139
140
|
num_search_terms: int,
|
|
140
141
|
search_terms: Dict[str, List[str]],
|
|
141
|
-
|
|
142
|
+
mm: str) -> Tuple[np.ndarray, str]:
|
|
142
143
|
field_scores = []
|
|
143
144
|
explain = []
|
|
144
145
|
for field, boost in query_fields.items():
|
|
145
146
|
post_arr = get_field(frame, field)
|
|
146
147
|
term_scores = np.array([post_arr.bm25(term) for term in search_terms[field]])
|
|
148
|
+
min_should_match = parse_min_should_match(len(search_terms[field]), spec=mm)
|
|
147
149
|
exp = " ".join([f"{field}:{term}" for term in search_terms[field]])
|
|
148
150
|
boost_exp = f"{boost}" if boost is not None else "1"
|
|
149
151
|
exp = "(" + exp + f")~{min(min_should_match, len(search_terms[field]))}"
|
|
150
152
|
exp = "(" + exp + f")^{boost_exp}"
|
|
151
|
-
|
|
153
|
+
|
|
152
154
|
matches_gt_mm = np.sum(term_scores > 0, axis=0) >= min(min_should_match, len(search_terms[field]))
|
|
153
155
|
sum_terms_bm25 = np.sum(term_scores, axis=0)
|
|
154
156
|
sum_terms_bm25[~matches_gt_mm] = 0
|
|
@@ -206,11 +208,10 @@ def edismax(frame: pd.DataFrame,
|
|
|
206
208
|
# trigram_fields = parse_field_boosts(pf3) if pf3 else {}
|
|
207
209
|
|
|
208
210
|
num_search_terms, search_terms, term_centric = parse_query_terms(frame, q, list(query_fields.keys()))
|
|
209
|
-
min_should_match = parse_min_should_match(num_search_terms, spec=mm)
|
|
210
211
|
if term_centric:
|
|
211
|
-
qf_scores, explain = _edismax_term_centric(frame, query_fields, num_search_terms, search_terms,
|
|
212
|
+
qf_scores, explain = _edismax_term_centric(frame, query_fields, num_search_terms, search_terms, mm)
|
|
212
213
|
else:
|
|
213
|
-
qf_scores, explain = _edismax_field_centric(frame, query_fields, num_search_terms, search_terms,
|
|
214
|
+
qf_scores, explain = _edismax_field_centric(frame, query_fields, num_search_terms, search_terms, mm)
|
|
214
215
|
|
|
215
216
|
phrase_scores = []
|
|
216
217
|
for field, boost in phrase_fields.items():
|
|
@@ -36,7 +36,7 @@ setup(
|
|
|
36
36
|
# For a discussion on single-sourcing the version across setup.py and the
|
|
37
37
|
# project code, see
|
|
38
38
|
# https://packaging.python.org/guides/single-sourcing-package-version/
|
|
39
|
-
version="0.0.
|
|
39
|
+
version="0.0.26", # Required
|
|
40
40
|
# This is a one-line description or tagline of what your project does. This
|
|
41
41
|
# corresponds to the "Summary" metadata field:
|
|
42
42
|
# https://packaging.python.org/specifications/core-metadata/#summary
|
|
@@ -121,6 +121,20 @@ edismax_scenarios = {
|
|
|
121
121
|
0],
|
|
122
122
|
"params": {'q': "foo bar", 'qf': ["title", "body"], 'mm': "2"},
|
|
123
123
|
},
|
|
124
|
+
"field_centric_mm_opp": {
|
|
125
|
+
"frame": {
|
|
126
|
+
'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
|
|
127
|
+
'body': lambda: PostingsArray.index(["foo bar", "data2", "data3 bar", "bunny funny wunny"],
|
|
128
|
+
tokenizer=just_lowercasing_tokenizer)
|
|
129
|
+
},
|
|
130
|
+
"expected": [lambda frame: max(sum([frame['title'].array.bm25("foo")[0],
|
|
131
|
+
frame['title'].array.bm25("bar")[0]]),
|
|
132
|
+
frame['body'].array.bm25("foo bar")[0]),
|
|
133
|
+
0,
|
|
134
|
+
0,
|
|
135
|
+
0],
|
|
136
|
+
"params": {'q': "foo bar", 'qf': ["body", "title"], 'mm': "2"},
|
|
137
|
+
},
|
|
124
138
|
"boost_title": {
|
|
125
139
|
"frame": {
|
|
126
140
|
'title': lambda: PostingsArray.index(["foo bar bar baz", "data2", "data3 bar", "bunny funny wunny"]),
|
|
@@ -91,6 +91,19 @@ def test_slice_then_search(tmdb_data):
|
|
|
91
91
|
assert skywalker_bm25.shape[0] == 3
|
|
92
92
|
|
|
93
93
|
|
|
94
|
+
tmdb_term_matches = [
|
|
95
|
+
("Star", ['11', '330459', '76180']),
|
|
96
|
+
("Black", ['374430']),
|
|
97
|
+
]
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@pytest.mark.parametrize("term,expected_matches", tmdb_term_matches)
|
|
101
|
+
def test_term_freqs(tmdb_data, term, expected_matches):
|
|
102
|
+
sliced = tmdb_data[tmdb_data['doc_id'].isin(expected_matches)]
|
|
103
|
+
term_freqs = sliced['title_tokens'].array.term_freq(term)
|
|
104
|
+
assert np.all(term_freqs == 1)
|
|
105
|
+
|
|
106
|
+
|
|
94
107
|
tmdb_phrase_matches = [
|
|
95
108
|
(["Star", "Wars"], ['11', '330459', '76180']),
|
|
96
109
|
(["Black", "Mirror:"], ['374430']),
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|